From 111aeb9e92263b72478414dfb84c62be21ee64a2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 16 Mar 2023 23:00:14 -0400 Subject: [PATCH 001/133] Testing overlayfs stores --- tests/init.sh | 2 +- tests/local.mk | 3 +- tests/overlay-local-store.sh | 3 ++ tests/overlay-local-store/inner.sh | 51 ++++++++++++++++++++++++++++++ 4 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 tests/overlay-local-store.sh create mode 100755 tests/overlay-local-store/inner.sh diff --git a/tests/init.sh b/tests/init.sh index c420e8c9f..2ec5fa1dd 100755 --- a/tests/init.sh +++ b/tests/init.sh @@ -3,7 +3,7 @@ source common/vars-and-functions.sh test -n "$TEST_ROOT" if test -d "$TEST_ROOT"; then - chmod -R u+w "$TEST_ROOT" + chmod -R u+rw "$TEST_ROOT" # We would delete any daemon socket, so let's stop the daemon first. killDaemon rm -rf "$TEST_ROOT" diff --git a/tests/local.mk b/tests/local.mk index 328f27e90..05894727c 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -124,7 +124,8 @@ nix_tests = \ flakes/show.sh \ impure-derivations.sh \ path-from-hash-part.sh \ - toString-path.sh + toString-path.sh \ + overlay-local-store.sh ifeq ($(HAVE_LIBCPUID), 1) nix_tests += compute-levels.sh diff --git a/tests/overlay-local-store.sh b/tests/overlay-local-store.sh new file mode 100644 index 000000000..6c0a5f96d --- /dev/null +++ b/tests/overlay-local-store.sh @@ -0,0 +1,3 @@ +source common.sh + +exec unshare --mount --map-root-user overlay-local-store/inner.sh diff --git a/tests/overlay-local-store/inner.sh b/tests/overlay-local-store/inner.sh new file mode 100755 index 000000000..fb690a85f --- /dev/null +++ b/tests/overlay-local-store/inner.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +set -x + +source common.sh + +export NIX_CONFIG='build-users-group = ' + +# Creating testing directories + +storeA="$TEST_ROOT/store_a" +storeB="$TEST_ROOT/store_b?real=$TEST_ROOT/merged-store" +storeBTop="$TEST_ROOT/store_b" + +mkdir -p "$TEST_ROOT"/{store_a,store_b,merged-store,workdir} + +# Mounting Overlay Store + +nix-store --store "$TEST_ROOT/store_a" --add dummy +nix-store --store "$TEST_ROOT/store_b" --add dummy + +mount -t overlay overlay \ + -o lowerdir="$TEST_ROOT/store_a/nix/store" \ + -o upperdir="$TEST_ROOT/store_b/nix/store" \ + -o workdir="$TEST_ROOT/workdir" \ + "$TEST_ROOT/merged-store" || skipTest "overlayfs is not supported" + +path_a=$(nix-build '' -A signal-desktop --store "$storeA") + +# Checking for Path in store_a +stat "$TEST_ROOT/store_a/$path_a" + +# Checking for Path in store_b +expect 1 stat "$TEST_ROOT/store_b/$path_a" + +# Checking for Path in merged-store +ls "$TEST_ROOT/merged-store/$(echo "$path_a" | sed 's|/nix/store/||g')" + + +# Verifying path in store_a +nix-store --verify-path --store "$storeA" "$path_a" + +# Verifiying path in merged-store (Should fail) +expect 1 nix-store --verify-path --store "$storeB" "$path_a" + +# Verifying path in store_b (Should fail) +expect 1 nix-store --verify-path --store "$storeBTop" "$path_a" + +path_b=$(nix-build '' -A signal-desktop --store "$storeB") From f0a176e2f1061475546ade22b81343e4b4967568 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 8 May 2023 10:20:06 -0400 Subject: [PATCH 002/133] Init local overlay store --- src/libstore/local-overlay-store.cc | 20 ++++++++ src/libstore/local-overlay-store.hh | 74 +++++++++++++++++++++++++++++ src/libstore/store-api.cc | 3 ++ 3 files changed, 97 insertions(+) create mode 100644 src/libstore/local-overlay-store.cc create mode 100644 src/libstore/local-overlay-store.hh diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc new file mode 100644 index 000000000..4cb50aef0 --- /dev/null +++ b/src/libstore/local-overlay-store.cc @@ -0,0 +1,20 @@ +#include "local-overlay-store.hh" + +namespace nix { + +LocalOverlayStore::LocalOverlayStore(const Params & params) + : StoreConfig(params) + , LocalFSStoreConfig(params) + , LocalStoreConfig(params) + , LocalOverlayStoreConfig(params) + , Store(params) + , LocalFSStore(params) + , LocalStore(params) + , lowerStore(openStore(lowerStoreUri).dynamic_pointer_cast()) +{ +} + + +static RegisterStoreImplementation regLocalOverlayStore; + +} diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh new file mode 100644 index 000000000..3129bb7a0 --- /dev/null +++ b/src/libstore/local-overlay-store.hh @@ -0,0 +1,74 @@ +#include "local-store.hh" + +namespace nix { + +/** + * Configuration for `LocalOverlayStore`. + */ +struct LocalOverlayStoreConfig : virtual LocalStoreConfig +{ + // FIXME why doesn't this work? + // using LocalStoreConfig::LocalStoreConfig; + + LocalOverlayStoreConfig(const StringMap & params) + : StoreConfig(params) + , LocalFSStoreConfig(params) + , LocalStoreConfig(params) + { } + + const Setting lowerStoreUri{(StoreConfig*) this, "", "lower-store", + R"( + [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + for the lower store. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly). + + Must be a store with a store dir on the file system. + )"}; + + const std::string name() override { return "Experimental Local Overlay Store"; } + + std::string doc() override + { + return + "" + // FIXME write docs + //#include "local-overlay-store.md" + ; + } +}; + +/** + * Variation of local store using overlayfs for the store dir. + */ +class LocalOverlayStore : public virtual LocalOverlayStoreConfig, public virtual LocalStore +{ + /** + * The store beneath us. + * + * Our store dir should be an overlay fs where the lower layer + * is that store's store dir, and the upper layer is some + * scratch storage just for us. + */ + ref lowerStore; + +public: + LocalOverlayStore(const Params & params); + + LocalOverlayStore(std::string scheme, std::string path, const Params & params) + : LocalOverlayStore(params) + { + throw UnimplementedError("LocalOverlayStore"); + } + + static std::set uriSchemes() + { return {}; } + + std::string getUri() override + { + return "local-overlay"; + } + +private: + // Overridden methods… +}; + +} diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 8fc3ed46a..c9ea03a1a 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -10,6 +10,7 @@ #include "archive.hh" #include "callback.hh" #include "remote-store.hh" +#include "local-overlay-store.hh" #include #include @@ -1391,6 +1392,8 @@ std::shared_ptr openFromNonUri(const std::string & uri, const Store::Para return std::make_shared(params); } else if (uri == "local") { return std::make_shared(params); + } else if (uri == "local-overlay") { + return std::make_shared(params); } else if (isNonUriPath(uri)) { Store::Params params2 = params; params2["root"] = absPath(uri); From f08754a97a22a97fe48da65517f8fb95292ab251 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 8 May 2023 14:47:39 -0400 Subject: [PATCH 003/133] Progress on tests --- tests/hermetic.nix | 56 ++++++++++++++++++++++++++++++ tests/overlay-local-store.sh | 4 +++ tests/overlay-local-store/inner.sh | 16 +++++++-- 3 files changed, 73 insertions(+), 3 deletions(-) create mode 100644 tests/hermetic.nix diff --git a/tests/hermetic.nix b/tests/hermetic.nix new file mode 100644 index 000000000..7ccd0fd72 --- /dev/null +++ b/tests/hermetic.nix @@ -0,0 +1,56 @@ +{ busybox }: + +with import ./config.nix; + +let + contentAddressedByDefault = builtins.getEnv "NIX_TESTS_CA_BY_DEFAULT" == "1"; + caArgs = if contentAddressedByDefault then { + __contentAddressed = true; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + } else {}; + + mkDerivation = args: + derivation ({ + inherit system; + builder = busybox; + args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; + } // removeAttrs args ["builder" "meta" "passthru"] + // caArgs) + // { meta = args.meta or {}; passthru = args.passthru or {}; }; + + input1 = mkDerivation { + shell = busybox; + name = "build-remote-input-1"; + buildCommand = "echo hi-input1; echo FOO > $out"; + }; + + input2 = mkDerivation { + shell = busybox; + name = "build-remote-input-2"; + buildCommand = "echo hi; echo BAR > $out"; + }; + + input3 = mkDerivation { + shell = busybox; + name = "build-remote-input-3"; + buildCommand = '' + echo hi-input3 + read x < ${input2} + echo $x BAZ > $out + ''; + }; + +in + + mkDerivation { + shell = busybox; + name = "build-remote"; + passthru = { inherit input1 input2 input3; }; + buildCommand = + '' + read x < ${input1} + read y < ${input3} + echo "$x $y" > $out + ''; + } diff --git a/tests/overlay-local-store.sh b/tests/overlay-local-store.sh index 6c0a5f96d..c98931118 100644 --- a/tests/overlay-local-store.sh +++ b/tests/overlay-local-store.sh @@ -1,3 +1,7 @@ source common.sh +requireSandboxSupport +[[ $busybox =~ busybox ]] || skipTest "no busybox" +if [[ $(uname) != Linux ]]; then skipTest "Need Linux for overlayfs"; fi + exec unshare --mount --map-root-user overlay-local-store/inner.sh diff --git a/tests/overlay-local-store/inner.sh b/tests/overlay-local-store/inner.sh index fb690a85f..5079dd0bd 100755 --- a/tests/overlay-local-store/inner.sh +++ b/tests/overlay-local-store/inner.sh @@ -11,13 +11,16 @@ export NIX_CONFIG='build-users-group = ' # Creating testing directories storeA="$TEST_ROOT/store_a" -storeB="$TEST_ROOT/store_b?real=$TEST_ROOT/merged-store" +storeB="local-overlay?root=$TEST_ROOT/store_b&lower-store=$TEST_ROOT/merged-store" storeBTop="$TEST_ROOT/store_b" mkdir -p "$TEST_ROOT"/{store_a,store_b,merged-store,workdir} # Mounting Overlay Store +## Restore normal, because we are using these chroot stores +#NIX_STORE_DIR=/nix/store + nix-store --store "$TEST_ROOT/store_a" --add dummy nix-store --store "$TEST_ROOT/store_b" --add dummy @@ -27,7 +30,14 @@ mount -t overlay overlay \ -o workdir="$TEST_ROOT/workdir" \ "$TEST_ROOT/merged-store" || skipTest "overlayfs is not supported" -path_a=$(nix-build '' -A signal-desktop --store "$storeA") +# Add in lower +NIX_REMOTE=$storeA source add.sh + +# Add in layered +NIX_REMOTE=$storeB source add.sh + +#busyboxExpr="\"\${$(dirname "$busybox")}/$(basename "$busybox")\"" +path_a=$(nix-build ./hermetic.nix --arg busybox "$busybox" --store "$storeA") # Checking for Path in store_a stat "$TEST_ROOT/store_a/$path_a" @@ -48,4 +58,4 @@ expect 1 nix-store --verify-path --store "$storeB" "$path_a" # Verifying path in store_b (Should fail) expect 1 nix-store --verify-path --store "$storeBTop" "$path_a" -path_b=$(nix-build '' -A signal-desktop --store "$storeB") +path_b=$(nix-build ./hermetic.nix --arg busybox $busybox --store "$storeB") From d80fc2ac1b39b4d81425d1aa614d4d06e5cd8640 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 8 May 2023 16:00:47 -0400 Subject: [PATCH 004/133] First round of testing, with todos --- tests/hermetic.nix | 12 ++--- tests/overlay-local-store.sh | 3 ++ tests/overlay-local-store/inner.sh | 85 +++++++++++++++++++----------- 3 files changed, 64 insertions(+), 36 deletions(-) diff --git a/tests/hermetic.nix b/tests/hermetic.nix index 7ccd0fd72..4c9d7a51f 100644 --- a/tests/hermetic.nix +++ b/tests/hermetic.nix @@ -1,4 +1,4 @@ -{ busybox }: +{ busybox, seed }: with import ./config.nix; @@ -21,19 +21,19 @@ let input1 = mkDerivation { shell = busybox; - name = "build-remote-input-1"; - buildCommand = "echo hi-input1; echo FOO > $out"; + name = "hermetic-input-1"; + buildCommand = "echo hi-input1 seed=${toString seed}; echo FOO > $out"; }; input2 = mkDerivation { shell = busybox; - name = "build-remote-input-2"; + name = "hermetic-input-2"; buildCommand = "echo hi; echo BAR > $out"; }; input3 = mkDerivation { shell = busybox; - name = "build-remote-input-3"; + name = "hermetic-input-3"; buildCommand = '' echo hi-input3 read x < ${input2} @@ -45,7 +45,7 @@ in mkDerivation { shell = busybox; - name = "build-remote"; + name = "hermetic"; passthru = { inherit input1 input2 input3; }; buildCommand = '' diff --git a/tests/overlay-local-store.sh b/tests/overlay-local-store.sh index c98931118..242c16e20 100644 --- a/tests/overlay-local-store.sh +++ b/tests/overlay-local-store.sh @@ -3,5 +3,8 @@ source common.sh requireSandboxSupport [[ $busybox =~ busybox ]] || skipTest "no busybox" if [[ $(uname) != Linux ]]; then skipTest "Need Linux for overlayfs"; fi +needLocalStore "The test uses --store always so we would just be bypassing the daemon" + +echo "drop-supplementary-groups = false" >> "$NIX_CONF_DIR"/nix.conf exec unshare --mount --map-root-user overlay-local-store/inner.sh diff --git a/tests/overlay-local-store/inner.sh b/tests/overlay-local-store/inner.sh index 5079dd0bd..524c801bd 100755 --- a/tests/overlay-local-store/inner.sh +++ b/tests/overlay-local-store/inner.sh @@ -10,52 +10,77 @@ export NIX_CONFIG='build-users-group = ' # Creating testing directories -storeA="$TEST_ROOT/store_a" -storeB="local-overlay?root=$TEST_ROOT/store_b&lower-store=$TEST_ROOT/merged-store" -storeBTop="$TEST_ROOT/store_b" +storeA="$TEST_ROOT/store-a" +storeBTop="$TEST_ROOT/store-b" +storeB="local-overlay?root=$TEST_ROOT/merged-store&lower-store=$TEST_ROOT/store-a" -mkdir -p "$TEST_ROOT"/{store_a,store_b,merged-store,workdir} +mkdir -p "$TEST_ROOT"/{store-a,store-b,merged-store/nix/store,workdir} # Mounting Overlay Store -## Restore normal, because we are using these chroot stores -#NIX_STORE_DIR=/nix/store +# Init lower store with some stuff +nix-store --store "$storeA" --add dummy -nix-store --store "$TEST_ROOT/store_a" --add dummy -nix-store --store "$TEST_ROOT/store_b" --add dummy +# Build something in lower store +path=$(nix-build ./hermetic.nix --arg busybox "$busybox" --arg seed 1 --store "$storeA") mount -t overlay overlay \ - -o lowerdir="$TEST_ROOT/store_a/nix/store" \ - -o upperdir="$TEST_ROOT/store_b/nix/store" \ + -o lowerdir="$storeA/nix/store" \ + -o upperdir="$storeBTop" \ -o workdir="$TEST_ROOT/workdir" \ - "$TEST_ROOT/merged-store" || skipTest "overlayfs is not supported" + "$TEST_ROOT/merged-store/nix/store" \ + || skipTest "overlayfs is not supported" -# Add in lower -NIX_REMOTE=$storeA source add.sh +cleanupOverlay () { + umount "$TEST_ROOT/merged-store/nix/store" + rm -r $TEST_ROOT/workdir +} +trap cleanupOverlay EXIT -# Add in layered -NIX_REMOTE=$storeB source add.sh +toRealPath () { + storeDir=$1; shift + storePath=$1; shift + echo $storeDir$(echo $storePath | sed "s^$NIX_STORE_DIR^^") +} -#busyboxExpr="\"\${$(dirname "$busybox")}/$(basename "$busybox")\"" -path_a=$(nix-build ./hermetic.nix --arg busybox "$busybox" --store "$storeA") +### Check status -# Checking for Path in store_a -stat "$TEST_ROOT/store_a/$path_a" +# Checking for path in lower layer +stat $(toRealPath "$storeA/nix/store" "$path") -# Checking for Path in store_b -expect 1 stat "$TEST_ROOT/store_b/$path_a" +# Checking for path in upper layer (should fail) +expect 1 stat $(toRealPath "$storeBTop" "$path") -# Checking for Path in merged-store -ls "$TEST_ROOT/merged-store/$(echo "$path_a" | sed 's|/nix/store/||g')" +# Checking for path in overlay store matching lower layer +diff $(toRealPath "$storeA/nix/store" "$path") $(toRealPath "$TEST_ROOT/merged-store/nix/store" "$path") +# Verifying path in lower layer +nix-store --verify-path --store "$storeA" "$path" -# Verifying path in store_a -nix-store --verify-path --store "$storeA" "$path_a" +# Verifying path in merged-store +# FIXME should succeed +expect 1 nix-store --verify-path --store "$storeB" "$path" -# Verifiying path in merged-store (Should fail) -expect 1 nix-store --verify-path --store "$storeB" "$path_a" +### Do a redundant add -# Verifying path in store_b (Should fail) -expect 1 nix-store --verify-path --store "$storeBTop" "$path_a" +path=$(nix-store --store "$storeB" --add dummy) -path_b=$(nix-build ./hermetic.nix --arg busybox $busybox --store "$storeB") +# lower store should have it from before +stat $(toRealPath "$storeA/nix/store" "$path") + +# upper store should not have redundant copy +# FIXME should fail +stat $(toRealPath "$storeA/nix/store" "$path") + +### Do a build in overlay store + +path=$(nix-build ./hermetic.nix --arg busybox $busybox --arg seed 2 --store "$storeB") + +# Checking for path in lower layer (should fail) +expect 1 stat $(toRealPath "$storeA/nix/store" "$path") + +# Checking for path in upper layer +stat $(toRealPath "$storeBTop" "$path") + +# Verifying path in overlay store +nix-store --verify-path --store "$storeB" "$path" From 0193c2abcd28cb4f5de73601360a1ba13de633d9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 8 May 2023 16:03:32 -0400 Subject: [PATCH 005/133] Improve tests slightly --- tests/overlay-local-store/inner.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/overlay-local-store/inner.sh b/tests/overlay-local-store/inner.sh index 524c801bd..ccf24b7b0 100755 --- a/tests/overlay-local-store/inner.sh +++ b/tests/overlay-local-store/inner.sh @@ -63,12 +63,15 @@ expect 1 nix-store --verify-path --store "$storeB" "$path" ### Do a redundant add +# upper layer should not have it +expect 1 stat $(toRealPath "$storeBTop/nix/store" "$path") + path=$(nix-store --store "$storeB" --add dummy) # lower store should have it from before stat $(toRealPath "$storeA/nix/store" "$path") -# upper store should not have redundant copy +# upper layer should still not have it (no redundant copy) # FIXME should fail stat $(toRealPath "$storeA/nix/store" "$path") From 31e98ed0a0b43596466f4980a1d3e49bc8e7a928 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 8 May 2023 16:48:55 -0400 Subject: [PATCH 006/133] Specialize `LocalOverlayStore::registerDrvOutput` --- src/libstore/local-overlay-store.cc | 11 +++++++++++ src/libstore/local-overlay-store.hh | 2 ++ 2 files changed, 13 insertions(+) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 4cb50aef0..2e3564df9 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -15,6 +15,17 @@ LocalOverlayStore::LocalOverlayStore(const Params & params) } +void LocalOverlayStore::registerDrvOutput(const Realisation & info) +{ + // First do queryRealisation on lower layer to populate DB + auto res = lowerStore->queryRealisation(info.id); + if (res) + LocalStore::registerDrvOutput(*res); + + LocalStore::registerDrvOutput(info); +} + + static RegisterStoreImplementation regLocalOverlayStore; } diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 3129bb7a0..e5329f5d9 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -69,6 +69,8 @@ public: private: // Overridden methods… + + void registerDrvOutput(const Realisation & info) override; }; } From 5406256d78244cf838eb917c35895035cbe1e402 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 8 May 2023 17:30:17 -0400 Subject: [PATCH 007/133] Specialize `LocalOverlayStore::queryPathInfoUncached` --- src/libstore/local-overlay-store.cc | 26 ++++++++++++++++++++++++++ src/libstore/local-overlay-store.hh | 3 +++ 2 files changed, 29 insertions(+) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 2e3564df9..545e8d28f 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -1,4 +1,5 @@ #include "local-overlay-store.hh" +#include "callback.hh" namespace nix { @@ -26,6 +27,31 @@ void LocalOverlayStore::registerDrvOutput(const Realisation & info) } +void LocalOverlayStore::queryPathInfoUncached(const StorePath & path, + Callback> callback) noexcept +{ + + auto callbackPtr = std::make_shared(std::move(callback)); + + // If we don't have it, check lower store + LocalStore::queryPathInfoUncached(path, + {[this, path, callbackPtr](std::future> fut) { + try { + (*callbackPtr)(fut.get()); + } catch (...) { + lowerStore->queryPathInfo(path, + {[path, callbackPtr](std::future> fut) { + try { + (*callbackPtr)(fut.get().get_ptr()); + } catch (...) { + callbackPtr->rethrow(); + } + }}); + } + }}); +} + + static RegisterStoreImplementation regLocalOverlayStore; } diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index e5329f5d9..12b705efb 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -71,6 +71,9 @@ private: // Overridden methods… void registerDrvOutput(const Realisation & info) override; + + void queryPathInfoUncached(const StorePath & path, + Callback> callback) noexcept override; }; } From 59a8099038179d2901a69b122d33d14544cd6e0d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 8 May 2023 17:37:40 -0400 Subject: [PATCH 008/133] Fix `LocalOverlayStore::queryPathInfoUncached`, FIXME in test --- src/libstore/local-overlay-store.cc | 23 +++++++++++++---------- tests/overlay-local-store/inner.sh | 3 +-- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 545e8d28f..bd48c12e4 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -33,21 +33,24 @@ void LocalOverlayStore::queryPathInfoUncached(const StorePath & path, auto callbackPtr = std::make_shared(std::move(callback)); - // If we don't have it, check lower store LocalStore::queryPathInfoUncached(path, {[this, path, callbackPtr](std::future> fut) { try { - (*callbackPtr)(fut.get()); + auto info = fut.get(); + if (info) + return (*callbackPtr)(std::move(info)); } catch (...) { - lowerStore->queryPathInfo(path, - {[path, callbackPtr](std::future> fut) { - try { - (*callbackPtr)(fut.get().get_ptr()); - } catch (...) { - callbackPtr->rethrow(); - } - }}); + return callbackPtr->rethrow(); } + // If we don't have it, check lower store + lowerStore->queryPathInfo(path, + {[path, callbackPtr](std::future> fut) { + try { + (*callbackPtr)(fut.get().get_ptr()); + } catch (...) { + return callbackPtr->rethrow(); + } + }}); }}); } diff --git a/tests/overlay-local-store/inner.sh b/tests/overlay-local-store/inner.sh index ccf24b7b0..69c6f6d0c 100755 --- a/tests/overlay-local-store/inner.sh +++ b/tests/overlay-local-store/inner.sh @@ -58,8 +58,7 @@ diff $(toRealPath "$storeA/nix/store" "$path") $(toRealPath "$TEST_ROOT/merged-s nix-store --verify-path --store "$storeA" "$path" # Verifying path in merged-store -# FIXME should succeed -expect 1 nix-store --verify-path --store "$storeB" "$path" +nix-store --verify-path --store "$storeB" "$path" ### Do a redundant add From b3d320c5946deb77bc76b86d4b4655ebfd502471 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 8 May 2023 18:50:16 -0400 Subject: [PATCH 009/133] Convert more methods Fixed one test, broke another --- src/libstore/local-overlay-store.cc | 80 ++++++++++++++++++++++++++++- src/libstore/local-overlay-store.hh | 29 +++++++++++ tests/overlay-local-store/inner.sh | 29 ++++++----- 3 files changed, 123 insertions(+), 15 deletions(-) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index bd48c12e4..a98d2e8fd 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -3,6 +3,12 @@ namespace nix { +Path LocalOverlayStoreConfig::toUpperPath(const StorePath & path) { + auto res = upperLayer + "/" + path.to_string(); + warn("upper path: %s", res); + return res; +} + LocalOverlayStore::LocalOverlayStore(const Params & params) : StoreConfig(params) , LocalFSStoreConfig(params) @@ -30,7 +36,6 @@ void LocalOverlayStore::registerDrvOutput(const Realisation & info) void LocalOverlayStore::queryPathInfoUncached(const StorePath & path, Callback> callback) noexcept { - auto callbackPtr = std::make_shared(std::move(callback)); LocalStore::queryPathInfoUncached(path, @@ -55,6 +60,79 @@ void LocalOverlayStore::queryPathInfoUncached(const StorePath & path, } +void LocalOverlayStore::queryRealisationUncached(const DrvOutput & drvOutput, + Callback> callback) noexcept +{ + auto callbackPtr = std::make_shared(std::move(callback)); + + LocalStore::queryRealisationUncached(drvOutput, + {[this, drvOutput, callbackPtr](std::future> fut) { + try { + auto info = fut.get(); + if (info) + return (*callbackPtr)(std::move(info)); + } catch (...) { + return callbackPtr->rethrow(); + } + // If we don't have it, check lower store + lowerStore->queryRealisation(drvOutput, + {[callbackPtr](std::future> fut) { + try { + (*callbackPtr)(fut.get()); + } catch (...) { + return callbackPtr->rethrow(); + } + }}); + }}); +} + + +bool LocalOverlayStore::isValidPathUncached(const StorePath & path) +{ + auto res = LocalStore::isValidPathUncached(path); + if (res) return res; + return lowerStore->isValidPath(path); +} + + +void LocalOverlayStore::addToStore(const ValidPathInfo & info, Source & source, + RepairFlag repair, CheckSigsFlag checkSigs) +{ + LocalStore::addToStore(info, source, repair, checkSigs); + if (lowerStore->isValidPath(info.path)) { + // dedup stores + deletePath(toUpperPath(info.path)); + } +} + + +StorePath LocalOverlayStore::addToStoreFromDump(Source & dump, std::string_view name, + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) +{ + auto path = LocalStore::addToStoreFromDump(dump, name, method, hashAlgo, repair, references); + if (lowerStore->isValidPath(path)) { + // dedup stores + deletePath(toUpperPath(path)); + } + return path; +} + + +StorePath LocalOverlayStore::addTextToStore( + std::string_view name, + std::string_view s, + const StorePathSet & references, + RepairFlag repair) +{ + auto path = LocalStore::addTextToStore(name, s, references, repair); + if (lowerStore->isValidPath(path)) { + // dedup stores + deletePath(toUpperPath(path)); + } + return path; +} + + static RegisterStoreImplementation regLocalOverlayStore; } diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 12b705efb..c087b3893 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -22,6 +22,12 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig for the lower store. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly). Must be a store with a store dir on the file system. + Must be used as OverlayFS lower layer for this store's store dir. + )"}; + + const Setting upperLayer{(StoreConfig*) this, "", "upper-layer", + R"( + Must be used as OverlayFS upper layer for this store's store dir. )"}; const std::string name() override { return "Experimental Local Overlay Store"; } @@ -34,6 +40,12 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig //#include "local-overlay-store.md" ; } + + /** + * Given a store path, get its location (if it is exists) in the + * upper layer of the overlayfs. + */ + Path toUpperPath(const StorePath & path); }; /** @@ -74,6 +86,23 @@ private: void queryPathInfoUncached(const StorePath & path, Callback> callback) noexcept override; + + bool isValidPathUncached(const StorePath & path) override; + + void addToStore(const ValidPathInfo & info, Source & source, + RepairFlag repair, CheckSigsFlag checkSigs) override; + + StorePath addToStoreFromDump(Source & dump, std::string_view name, + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) override; + + StorePath addTextToStore( + std::string_view name, + std::string_view s, + const StorePathSet & references, + RepairFlag repair) override; + + void queryRealisationUncached(const DrvOutput&, + Callback> callback) noexcept override; }; } diff --git a/tests/overlay-local-store/inner.sh b/tests/overlay-local-store/inner.sh index 69c6f6d0c..d9d3b76cf 100755 --- a/tests/overlay-local-store/inner.sh +++ b/tests/overlay-local-store/inner.sh @@ -12,7 +12,7 @@ export NIX_CONFIG='build-users-group = ' storeA="$TEST_ROOT/store-a" storeBTop="$TEST_ROOT/store-b" -storeB="local-overlay?root=$TEST_ROOT/merged-store&lower-store=$TEST_ROOT/store-a" +storeB="local-overlay?root=$TEST_ROOT/merged-store&lower-store=$storeA&upper-layer=$storeBTop" mkdir -p "$TEST_ROOT"/{store-a,store-b,merged-store/nix/store,workdir} @@ -71,18 +71,19 @@ path=$(nix-store --store "$storeB" --add dummy) stat $(toRealPath "$storeA/nix/store" "$path") # upper layer should still not have it (no redundant copy) -# FIXME should fail -stat $(toRealPath "$storeA/nix/store" "$path") +expect 1 stat $(toRealPath "$storeB/nix/store" "$path") -### Do a build in overlay store +## Ooops something went wrong -path=$(nix-build ./hermetic.nix --arg busybox $busybox --arg seed 2 --store "$storeB") - -# Checking for path in lower layer (should fail) -expect 1 stat $(toRealPath "$storeA/nix/store" "$path") - -# Checking for path in upper layer -stat $(toRealPath "$storeBTop" "$path") - -# Verifying path in overlay store -nix-store --verify-path --store "$storeB" "$path" +## ### Do a build in overlay store +## +## path=$(nix-build ./hermetic.nix --arg busybox $busybox --arg seed 2 --store "$storeB") +## +## # Checking for path in lower layer (should fail) +## expect 1 stat $(toRealPath "$storeA/nix/store" "$path") +## +## # Checking for path in upper layer +## stat $(toRealPath "$storeBTop" "$path") +## +## # Verifying path in overlay store +## nix-store --verify-path --store "$storeB" "$path" From ddaf2750b5af28d56b99f79d3ac20ae11d3464d3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 9 May 2023 10:22:38 -0400 Subject: [PATCH 010/133] Specialize more methods, fix tests --- src/libstore/local-overlay-store.cc | 30 +++++++++++++++++++++++++---- src/libstore/local-overlay-store.hh | 2 ++ src/libstore/local-store.hh | 2 +- tests/overlay-local-store/inner.sh | 26 ++++++++++++------------- 4 files changed, 41 insertions(+), 19 deletions(-) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index a98d2e8fd..ad1947698 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -4,9 +4,7 @@ namespace nix { Path LocalOverlayStoreConfig::toUpperPath(const StorePath & path) { - auto res = upperLayer + "/" + path.to_string(); - warn("upper path: %s", res); - return res; + return upperLayer + "/" + path.to_string(); } LocalOverlayStore::LocalOverlayStore(const Params & params) @@ -91,7 +89,31 @@ bool LocalOverlayStore::isValidPathUncached(const StorePath & path) { auto res = LocalStore::isValidPathUncached(path); if (res) return res; - return lowerStore->isValidPath(path); + res = lowerStore->isValidPath(path); + if (res) { + // Get path info from lower store so upper DB genuinely has it. + LocalStore::registerValidPath(*lowerStore->queryPathInfo(path)); + } + return res; +} + + +void LocalOverlayStore::registerValidPaths(const ValidPathInfos & infos) +{ + // First, get any from lower store so we merge + { + StorePathSet notInUpper; + for (auto & [p, _] : infos) + if (!LocalStore::isValidPathUncached(p)) // avoid divergence + notInUpper.insert(p); + auto pathsInLower = lowerStore->queryValidPaths(notInUpper); + ValidPathInfos inLower; + for (auto & p : pathsInLower) + inLower.insert_or_assign(p, *lowerStore->queryPathInfo(p)); + LocalStore::registerValidPaths(inLower); + } + // Then do original request + LocalStore::registerValidPaths(infos); } diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index c087b3893..daebe8547 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -89,6 +89,8 @@ private: bool isValidPathUncached(const StorePath & path) override; + void registerValidPaths(const ValidPathInfos & infos) override; + void addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) override; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 55add18dd..b61ec3590 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -232,7 +232,7 @@ public: */ void registerValidPath(const ValidPathInfo & info); - void registerValidPaths(const ValidPathInfos & infos); + virtual void registerValidPaths(const ValidPathInfos & infos); unsigned int getProtocol() override; diff --git a/tests/overlay-local-store/inner.sh b/tests/overlay-local-store/inner.sh index d9d3b76cf..dc8ec8c2c 100755 --- a/tests/overlay-local-store/inner.sh +++ b/tests/overlay-local-store/inner.sh @@ -22,7 +22,7 @@ mkdir -p "$TEST_ROOT"/{store-a,store-b,merged-store/nix/store,workdir} nix-store --store "$storeA" --add dummy # Build something in lower store -path=$(nix-build ./hermetic.nix --arg busybox "$busybox" --arg seed 1 --store "$storeA") +path=$(nix-build ./hermetic.nix --arg busybox "$busybox" --arg seed 1 --store "$storeA" --no-out-link) mount -t overlay overlay \ -o lowerdir="$storeA/nix/store" \ @@ -73,17 +73,15 @@ stat $(toRealPath "$storeA/nix/store" "$path") # upper layer should still not have it (no redundant copy) expect 1 stat $(toRealPath "$storeB/nix/store" "$path") -## Ooops something went wrong +### Do a build in overlay store -## ### Do a build in overlay store -## -## path=$(nix-build ./hermetic.nix --arg busybox $busybox --arg seed 2 --store "$storeB") -## -## # Checking for path in lower layer (should fail) -## expect 1 stat $(toRealPath "$storeA/nix/store" "$path") -## -## # Checking for path in upper layer -## stat $(toRealPath "$storeBTop" "$path") -## -## # Verifying path in overlay store -## nix-store --verify-path --store "$storeB" "$path" +path=$(nix-build ./hermetic.nix --arg busybox $busybox --arg seed 2 --store "$storeB" --no-out-link) + +# Checking for path in lower layer (should fail) +expect 1 stat $(toRealPath "$storeA/nix/store" "$path") + +# Checking for path in upper layer +stat $(toRealPath "$storeBTop" "$path") + +# Verifying path in overlay store +nix-store --verify-path --store "$storeB" "$path" From e7c3399ed2e62ecf8a0fb5294f19ffab2061cc04 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 9 May 2023 10:40:10 -0400 Subject: [PATCH 011/133] Specialize `LocalOverlayStore::queryPathFromHashPart` With test --- src/libstore/local-overlay-store.cc | 10 ++++++++++ src/libstore/local-overlay-store.hh | 2 ++ tests/overlay-local-store/inner.sh | 8 ++++++++ 3 files changed, 20 insertions(+) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index ad1947698..cb409c2bd 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -98,6 +98,16 @@ bool LocalOverlayStore::isValidPathUncached(const StorePath & path) } +std::optional LocalOverlayStore::queryPathFromHashPart(const std::string & hashPart) +{ + auto res = LocalStore::queryPathFromHashPart(hashPart); + if (res) + return res; + else + return lowerStore->queryPathFromHashPart(hashPart); +} + + void LocalOverlayStore::registerValidPaths(const ValidPathInfos & infos) { // First, get any from lower store so we merge diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index daebe8547..88da4ba4a 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -89,6 +89,8 @@ private: bool isValidPathUncached(const StorePath & path) override; + std::optional queryPathFromHashPart(const std::string & hashPart) override; + void registerValidPaths(const ValidPathInfos & infos) override; void addToStore(const ValidPathInfo & info, Source & source, diff --git a/tests/overlay-local-store/inner.sh b/tests/overlay-local-store/inner.sh index dc8ec8c2c..279f005a9 100755 --- a/tests/overlay-local-store/inner.sh +++ b/tests/overlay-local-store/inner.sh @@ -60,6 +60,14 @@ nix-store --verify-path --store "$storeA" "$path" # Verifying path in merged-store nix-store --verify-path --store "$storeB" "$path" +hashPart=$(echo $path | sed "s^$NIX_STORE_DIR/^^" | sed 's/-.*//') + +# Lower store can find from hash part +[[ $(nix store --store $storeA path-from-hash-part $hashPart) == $path ]] + +# merged store can find from hash part +[[ $(nix store --store $storeB path-from-hash-part $hashPart) == $path ]] + ### Do a redundant add # upper layer should not have it From 5059be53b1553f99b95f31227998de69ecf74d90 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 9 May 2023 16:42:28 -0400 Subject: [PATCH 012/133] Fix recursive ingestion from lower store --- src/libstore/local-overlay-store.cc | 7 ++++++- tests/overlay-local-store/inner.sh | 15 +++++++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index cb409c2bd..431b82c90 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -92,7 +92,12 @@ bool LocalOverlayStore::isValidPathUncached(const StorePath & path) res = lowerStore->isValidPath(path); if (res) { // Get path info from lower store so upper DB genuinely has it. - LocalStore::registerValidPath(*lowerStore->queryPathInfo(path)); + auto p = lowerStore->queryPathInfo(path); + // recur on references, syncing entire closure. + for (auto & r : p->references) + if (r != path) + isValidPath(r); + LocalStore::registerValidPath(*p); } return res; } diff --git a/tests/overlay-local-store/inner.sh b/tests/overlay-local-store/inner.sh index 279f005a9..1cb29ceb7 100755 --- a/tests/overlay-local-store/inner.sh +++ b/tests/overlay-local-store/inner.sh @@ -22,7 +22,8 @@ mkdir -p "$TEST_ROOT"/{store-a,store-b,merged-store/nix/store,workdir} nix-store --store "$storeA" --add dummy # Build something in lower store -path=$(nix-build ./hermetic.nix --arg busybox "$busybox" --arg seed 1 --store "$storeA" --no-out-link) +drvPath=$(nix-instantiate --store $storeA ./hermetic.nix --arg busybox "$busybox" --arg seed 1) +path=$(nix-store --store "$storeA" --realise $drvPath) mount -t overlay overlay \ -o lowerdir="$storeA/nix/store" \ @@ -38,9 +39,9 @@ cleanupOverlay () { trap cleanupOverlay EXIT toRealPath () { - storeDir=$1; shift - storePath=$1; shift - echo $storeDir$(echo $storePath | sed "s^$NIX_STORE_DIR^^") + storeDir=$1; shift + storePath=$1; shift + echo $storeDir$(echo $storePath | sed "s^$NIX_STORE_DIR^^") } ### Check status @@ -60,6 +61,12 @@ nix-store --verify-path --store "$storeA" "$path" # Verifying path in merged-store nix-store --verify-path --store "$storeB" "$path" +[[ \ + $(nix-store --store $storeA --query --outputs $drvPath) \ + == \ + $(nix-store --store $storeB --query --outputs $drvPath) \ + ]] + hashPart=$(echo $path | sed "s^$NIX_STORE_DIR/^^" | sed 's/-.*//') # Lower store can find from hash part From 8339c170d781825acd55f30af4d76c5c14d50511 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 9 May 2023 16:49:44 -0400 Subject: [PATCH 013/133] More tests --- tests/overlay-local-store/inner.sh | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/tests/overlay-local-store/inner.sh b/tests/overlay-local-store/inner.sh index 1cb29ceb7..d13d8b9ab 100755 --- a/tests/overlay-local-store/inner.sh +++ b/tests/overlay-local-store/inner.sh @@ -55,18 +55,26 @@ expect 1 stat $(toRealPath "$storeBTop" "$path") # Checking for path in overlay store matching lower layer diff $(toRealPath "$storeA/nix/store" "$path") $(toRealPath "$TEST_ROOT/merged-store/nix/store" "$path") -# Verifying path in lower layer -nix-store --verify-path --store "$storeA" "$path" - -# Verifying path in merged-store -nix-store --verify-path --store "$storeB" "$path" +# Checking derivers query agreement +[[ \ + $(nix-store --store $storeA --query --deriver $path) \ + == \ + $(nix-store --store $storeB --query --deriver $path) \ + ]] +# Checking outputs query agreement [[ \ $(nix-store --store $storeA --query --outputs $drvPath) \ == \ $(nix-store --store $storeB --query --outputs $drvPath) \ ]] +# Verifying path in lower layer +nix-store --verify-path --store "$storeA" "$path" + +# Verifying path in merged-store +nix-store --verify-path --store "$storeB" "$path" + hashPart=$(echo $path | sed "s^$NIX_STORE_DIR/^^" | sed 's/-.*//') # Lower store can find from hash part From 4173743a3c1f0418c298d12cc71ffa26f6bc848d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 9 May 2023 17:20:58 -0400 Subject: [PATCH 014/133] Implement more queries --- src/libstore/local-overlay-store.cc | 16 ++++++++++++++++ src/libstore/local-overlay-store.hh | 4 ++++ tests/overlay-local-store/inner.sh | 15 +++++++++++++++ 3 files changed, 35 insertions(+) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 431b82c90..c6b721b48 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -103,6 +103,22 @@ bool LocalOverlayStore::isValidPathUncached(const StorePath & path) } +void LocalOverlayStore::queryReferrers(const StorePath & path, StorePathSet & referrers) +{ + LocalStore::queryReferrers(path, referrers); + lowerStore->queryReferrers(path, referrers); +} + + +StorePathSet LocalOverlayStore::queryValidDerivers(const StorePath & path) +{ + auto res = LocalStore::queryValidDerivers(path); + for (auto p : lowerStore->queryValidDerivers(path)) + res.insert(p); + return res; +} + + std::optional LocalOverlayStore::queryPathFromHashPart(const std::string & hashPart) { auto res = LocalStore::queryPathFromHashPart(hashPart); diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 88da4ba4a..9bc513b3e 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -89,6 +89,10 @@ private: bool isValidPathUncached(const StorePath & path) override; + void queryReferrers(const StorePath & path, StorePathSet & referrers) override; + + StorePathSet queryValidDerivers(const StorePath & path) override; + std::optional queryPathFromHashPart(const std::string & hashPart) override; void registerValidPaths(const ValidPathInfos & infos) override; diff --git a/tests/overlay-local-store/inner.sh b/tests/overlay-local-store/inner.sh index d13d8b9ab..7ab587b87 100755 --- a/tests/overlay-local-store/inner.sh +++ b/tests/overlay-local-store/inner.sh @@ -55,6 +55,21 @@ expect 1 stat $(toRealPath "$storeBTop" "$path") # Checking for path in overlay store matching lower layer diff $(toRealPath "$storeA/nix/store" "$path") $(toRealPath "$TEST_ROOT/merged-store/nix/store" "$path") +# Checking requisites query agreement +[[ \ + $(nix-store --store $storeA --query --requisites $drvPath) \ + == \ + $(nix-store --store $storeB --query --requisites $drvPath) \ + ]] + +# Checking referrers query agreement +busyboxStore=$(nix store --store $storeA add-path $busybox) +[[ \ + $(nix-store --store $storeA --query --referrers $busyboxStore) \ + == \ + $(nix-store --store $storeB --query --referrers $busyboxStore) \ + ]] + # Checking derivers query agreement [[ \ $(nix-store --store $storeA --query --deriver $path) \ From b5591ece4c4afae6a0ea71396a72eea529b39b79 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Mon, 15 May 2023 10:20:16 +0100 Subject: [PATCH 015/133] Check that overlay store directory is mounted correctly. Nix does not manage the overlayfs mount point itself, but the correct functioning of the overlay store does depend on this mount point being set up correctly. Rather than just assume this is the case, check that the lowerdir and upperdir options are what we expect them to be. This check is on by default, but can be disabled if needed. --- src/libstore/local-overlay-store.cc | 27 +++++++++++++++++++++++++++ src/libstore/local-overlay-store.hh | 3 +++ 2 files changed, 30 insertions(+) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index c6b721b48..a40f55fdb 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -1,5 +1,6 @@ #include "local-overlay-store.hh" #include "callback.hh" +#include namespace nix { @@ -17,6 +18,32 @@ LocalOverlayStore::LocalOverlayStore(const Params & params) , LocalStore(params) , lowerStore(openStore(lowerStoreUri).dynamic_pointer_cast()) { + if (checkMount.get()) { + std::smatch match; + std::string mountInfo; + auto mounts = readFile("/proc/self/mounts"); + auto regex = std::regex(R"((^|\n)overlay )" + realStoreDir.get() + R"( .*(\n|$))"); + + // Mount points can be stacked, so there might be multiple matching entries. + // Loop until the last match, which will be the current state of the mount point. + while (std::regex_search(mounts, match, regex)) { + mountInfo = match.str(); + mounts = match.suffix(); + } + + auto checkOption = [&](std::string option, std::string value) { + return std::regex_search(mountInfo, std::regex("\\b" + option + "=" + value + "( |,)")); + }; + + auto expectedLowerDir = lowerStore->realStoreDir.get(); + if (!checkOption("lowerdir", expectedLowerDir) || !checkOption("upperdir", upperLayer)) { + debug("expected lowerdir: %s", expectedLowerDir); + debug("expected upperdir: %s", upperLayer); + debug("actual mount: %s", mountInfo); + throw Error("overlay filesystem '%s' mounted incorrectly", + realStoreDir.get()); + } + } } diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 9bc513b3e..8280ed273 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -30,6 +30,9 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig Must be used as OverlayFS upper layer for this store's store dir. )"}; + Setting checkMount{(StoreConfig*) this, true, "check-mount", + "Check that the overlay filesystem is correctly mounted."}; + const std::string name() override { return "Experimental Local Overlay Store"; } std::string doc() override From b0989cb10b6e1a59070e8c999520c0fbfd25657d Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Mon, 15 May 2023 10:28:43 +0100 Subject: [PATCH 016/133] Support percent encoded URIs for lower store. --- src/libstore/local-overlay-store.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index a40f55fdb..ab7af6aea 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -1,5 +1,6 @@ #include "local-overlay-store.hh" #include "callback.hh" +#include "url.hh" #include namespace nix { @@ -16,7 +17,7 @@ LocalOverlayStore::LocalOverlayStore(const Params & params) , Store(params) , LocalFSStore(params) , LocalStore(params) - , lowerStore(openStore(lowerStoreUri).dynamic_pointer_cast()) + , lowerStore(openStore(percentDecode(lowerStoreUri.get())).dynamic_pointer_cast()) { if (checkMount.get()) { std::smatch match; From 0df37edb1c1ae5af43496f009221f2d4fe23a629 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Mon, 15 May 2023 10:29:06 +0100 Subject: [PATCH 017/133] Make upper-layer a PathSetting instead of a Setting. --- src/libstore/local-overlay-store.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 8280ed273..525a54745 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -25,7 +25,7 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig Must be used as OverlayFS lower layer for this store's store dir. )"}; - const Setting upperLayer{(StoreConfig*) this, "", "upper-layer", + const PathSetting upperLayer{(StoreConfig*) this, false, "", "upper-layer", R"( Must be used as OverlayFS upper layer for this store's store dir. )"}; From b7e5aaf90d50d06e057cceefaa831ea5e4a671bd Mon Sep 17 00:00:00 2001 From: cidkidnix Date: Mon, 15 May 2023 13:40:01 -0500 Subject: [PATCH 018/133] Add test for checking that we reject bad local overlay store uris --- tests/overlay-local-store/inner.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/overlay-local-store/inner.sh b/tests/overlay-local-store/inner.sh index 7ab587b87..c8d49eb19 100755 --- a/tests/overlay-local-store/inner.sh +++ b/tests/overlay-local-store/inner.sh @@ -14,6 +14,16 @@ storeA="$TEST_ROOT/store-a" storeBTop="$TEST_ROOT/store-b" storeB="local-overlay?root=$TEST_ROOT/merged-store&lower-store=$storeA&upper-layer=$storeBTop" +mkdir -p $TEST_ROOT/bad_test +badTestRoot=$TEST_ROOT/bad_test +storeBadRoot="local-overlay?root=$badTestRoot&lower-store=$storeA&upper-layer=$storeBTop" +storeBadLower="local-overlay?root=$TEST_ROOT/merged-store&lower-store=$badTestRoot&upper-layer=$storeBTop" +storeBadUpper="local-overlay?root=$TEST_ROOT/merged-store&lower-store=$storeA&upper-layer=$badTestRoot" + +declare -a storesBad=( + "$storeBadRoot" "$storeBadLower" "$storeBadUpper" +) + mkdir -p "$TEST_ROOT"/{store-a,store-b,merged-store/nix/store,workdir} # Mounting Overlay Store @@ -105,6 +115,11 @@ expect 1 stat $(toRealPath "$storeBTop/nix/store" "$path") path=$(nix-store --store "$storeB" --add dummy) +for i in "${storesBad[@]}"; do + echo $i + expectStderr 1 nix-store --store "$i" --add dummy | grepQuiet "overlay filesystem .* mounted incorrectly" +done + # lower store should have it from before stat $(toRealPath "$storeA/nix/store" "$path") From 0979a374c555a34f1ee622c3f205b1702a59ccc6 Mon Sep 17 00:00:00 2001 From: cidkidnix Date: Mon, 15 May 2023 14:34:57 -0500 Subject: [PATCH 019/133] Begin to split up overlay-local-store tests The bad-uris tests are now in their own file. "Outer" is a bad name, but it will be split up next. --- tests/common.sh | 2 +- tests/local.mk | 3 +- tests/overlay-local-store.sh | 10 ----- tests/overlay-local-store/bad-uris.sh | 25 ++++++++++++ tests/overlay-local-store/common.sh | 52 ++++++++++++++++++++++++ tests/overlay-local-store/inner.sh | 57 +++------------------------ tests/overlay-local-store/outer.sh | 5 +++ 7 files changed, 90 insertions(+), 64 deletions(-) delete mode 100644 tests/overlay-local-store.sh create mode 100644 tests/overlay-local-store/bad-uris.sh create mode 100644 tests/overlay-local-store/common.sh create mode 100755 tests/overlay-local-store/outer.sh diff --git a/tests/common.sh b/tests/common.sh index 8941671d6..7b0922c9f 100644 --- a/tests/common.sh +++ b/tests/common.sh @@ -4,7 +4,7 @@ if [[ -z "${COMMON_SH_SOURCED-}" ]]; then COMMON_SH_SOURCED=1 -source "$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")/common/vars-and-functions.sh" +source "$(readlink -f "$(dirname "${BASH_SOURCE[0]-$0}")")/common/vars-and-functions.sh" if [[ -n "${NIX_DAEMON_PACKAGE:-}" ]]; then startDaemon fi diff --git a/tests/local.mk b/tests/local.mk index 61fb1a85b..ea5550c75 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -133,7 +133,8 @@ nix_tests = \ impure-derivations.sh \ path-from-hash-part.sh \ toString-path.sh \ - overlay-local-store.sh + overlay-local-store/outer.sh \ + overlay-local-store/bad-uris.sh ifeq ($(HAVE_LIBCPUID), 1) nix_tests += compute-levels.sh diff --git a/tests/overlay-local-store.sh b/tests/overlay-local-store.sh deleted file mode 100644 index 242c16e20..000000000 --- a/tests/overlay-local-store.sh +++ /dev/null @@ -1,10 +0,0 @@ -source common.sh - -requireSandboxSupport -[[ $busybox =~ busybox ]] || skipTest "no busybox" -if [[ $(uname) != Linux ]]; then skipTest "Need Linux for overlayfs"; fi -needLocalStore "The test uses --store always so we would just be bypassing the daemon" - -echo "drop-supplementary-groups = false" >> "$NIX_CONF_DIR"/nix.conf - -exec unshare --mount --map-root-user overlay-local-store/inner.sh diff --git a/tests/overlay-local-store/bad-uris.sh b/tests/overlay-local-store/bad-uris.sh new file mode 100644 index 000000000..d4261bd97 --- /dev/null +++ b/tests/overlay-local-store/bad-uris.sh @@ -0,0 +1,25 @@ +source common.sh + +requireEnvironment +setupConfig +storeDirs + +mkdir -p $TEST_ROOT/bad_test +badTestRoot=$TEST_ROOT/bad_test +storeBadRoot="local-overlay?root=$badTestRoot&lower-store=$storeA&upper-layer=$storeBTop" +storeBadLower="local-overlay?root=$TEST_ROOT/merged-store&lower-store=$badTestRoot&upper-layer=$storeBTop" +storeBadUpper="local-overlay?root=$TEST_ROOT/merged-store&lower-store=$storeA&upper-layer=$badTestRoot" + +declare -a storesBad=( + "$storeBadRoot" "$storeBadLower" "$storeBadUpper" +) + +for i in "${storesBad[@]}"; do + echo $i + unshare --mount --map-root-user bash <> "$NIX_CONF_DIR"/nix.conf + echo "build-users-group = " >> "$NIX_CONF_DIR"/nix.conf +} + +storeDirs () { + storeA="$TEST_ROOT/store-a" + storeBTop="$TEST_ROOT/store-b" + storeB="local-overlay?root=$TEST_ROOT/merged-store&lower-store=$storeA&upper-layer=$storeBTop" + # Creating testing directories + mkdir -p "$TEST_ROOT"/{store-a/nix/store,store-b,merged-store/nix/store,workdir} +} + +# Mounting Overlay Store +mountOverlayfs () { + mount -t overlay overlay \ + -o lowerdir="$storeA/nix/store" \ + -o upperdir="$storeBTop" \ + -o workdir="$TEST_ROOT/workdir" \ + "$TEST_ROOT/merged-store/nix/store" \ + || skipTest "overlayfs is not supported" + + cleanupOverlay () { + umount "$TEST_ROOT/merged-store/nix/store" + rm -r $TEST_ROOT/workdir + } + trap cleanupOverlay EXIT +} + +toRealPath () { + storeDir=$1; shift + storePath=$1; shift + echo $storeDir$(echo $storePath | sed "s^$NIX_STORE_DIR^^") +} + +initLowerStore () { + # Init lower store with some stuff + nix-store --store "$storeA" --add ../dummy + + # Build something in lower store + drvPath=$(nix-instantiate --store $storeA ../hermetic.nix --arg busybox "$busybox" --arg seed 1) + path=$(nix-store --store "$storeA" --realise $drvPath) +} diff --git a/tests/overlay-local-store/inner.sh b/tests/overlay-local-store/inner.sh index c8d49eb19..adbe29557 100755 --- a/tests/overlay-local-store/inner.sh +++ b/tests/overlay-local-store/inner.sh @@ -6,53 +6,11 @@ set -x source common.sh -export NIX_CONFIG='build-users-group = ' +storeDirs -# Creating testing directories +initLowerStore -storeA="$TEST_ROOT/store-a" -storeBTop="$TEST_ROOT/store-b" -storeB="local-overlay?root=$TEST_ROOT/merged-store&lower-store=$storeA&upper-layer=$storeBTop" - -mkdir -p $TEST_ROOT/bad_test -badTestRoot=$TEST_ROOT/bad_test -storeBadRoot="local-overlay?root=$badTestRoot&lower-store=$storeA&upper-layer=$storeBTop" -storeBadLower="local-overlay?root=$TEST_ROOT/merged-store&lower-store=$badTestRoot&upper-layer=$storeBTop" -storeBadUpper="local-overlay?root=$TEST_ROOT/merged-store&lower-store=$storeA&upper-layer=$badTestRoot" - -declare -a storesBad=( - "$storeBadRoot" "$storeBadLower" "$storeBadUpper" -) - -mkdir -p "$TEST_ROOT"/{store-a,store-b,merged-store/nix/store,workdir} - -# Mounting Overlay Store - -# Init lower store with some stuff -nix-store --store "$storeA" --add dummy - -# Build something in lower store -drvPath=$(nix-instantiate --store $storeA ./hermetic.nix --arg busybox "$busybox" --arg seed 1) -path=$(nix-store --store "$storeA" --realise $drvPath) - -mount -t overlay overlay \ - -o lowerdir="$storeA/nix/store" \ - -o upperdir="$storeBTop" \ - -o workdir="$TEST_ROOT/workdir" \ - "$TEST_ROOT/merged-store/nix/store" \ - || skipTest "overlayfs is not supported" - -cleanupOverlay () { - umount "$TEST_ROOT/merged-store/nix/store" - rm -r $TEST_ROOT/workdir -} -trap cleanupOverlay EXIT - -toRealPath () { - storeDir=$1; shift - storePath=$1; shift - echo $storeDir$(echo $storePath | sed "s^$NIX_STORE_DIR^^") -} +mountOverlayfs ### Check status @@ -113,12 +71,7 @@ hashPart=$(echo $path | sed "s^$NIX_STORE_DIR/^^" | sed 's/-.*//') # upper layer should not have it expect 1 stat $(toRealPath "$storeBTop/nix/store" "$path") -path=$(nix-store --store "$storeB" --add dummy) - -for i in "${storesBad[@]}"; do - echo $i - expectStderr 1 nix-store --store "$i" --add dummy | grepQuiet "overlay filesystem .* mounted incorrectly" -done +path=$(nix-store --store "$storeB" --add ../dummy) # lower store should have it from before stat $(toRealPath "$storeA/nix/store" "$path") @@ -128,7 +81,7 @@ expect 1 stat $(toRealPath "$storeB/nix/store" "$path") ### Do a build in overlay store -path=$(nix-build ./hermetic.nix --arg busybox $busybox --arg seed 2 --store "$storeB" --no-out-link) +path=$(nix-build ../hermetic.nix --arg busybox $busybox --arg seed 2 --store "$storeB" --no-out-link) # Checking for path in lower layer (should fail) expect 1 stat $(toRealPath "$storeA/nix/store" "$path") diff --git a/tests/overlay-local-store/outer.sh b/tests/overlay-local-store/outer.sh new file mode 100755 index 000000000..ba55f1d06 --- /dev/null +++ b/tests/overlay-local-store/outer.sh @@ -0,0 +1,5 @@ +source common.sh + +requireEnvironment +setupConfig +exec unshare --mount --map-root-user ./inner.sh From b1fba1c2a13ca8a9e295ee0a572df207d3591db1 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 15 May 2023 16:44:36 -0400 Subject: [PATCH 020/133] Fix PS4 for heredocs --- tests/common/vars-and-functions.sh.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/common/vars-and-functions.sh.in b/tests/common/vars-and-functions.sh.in index a9e6c802f..dc7ce13cc 100644 --- a/tests/common/vars-and-functions.sh.in +++ b/tests/common/vars-and-functions.sh.in @@ -4,7 +4,7 @@ if [[ -z "${COMMON_VARS_AND_FUNCTIONS_SH_SOURCED-}" ]]; then COMMON_VARS_AND_FUNCTIONS_SH_SOURCED=1 -export PS4='+(${BASH_SOURCE[0]}:$LINENO) ' +export PS4='+(${BASH_SOURCE[0]-$0}:$LINENO) ' export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/${TEST_NAME:-default} export NIX_STORE_DIR From 97deb00cbc5697d86a04ec7491b5b0da26ee5357 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 15 May 2023 18:13:11 -0400 Subject: [PATCH 021/133] Create notion of "test group", use for local overlay store --- mk/lib.mk | 9 +++++++++ mk/tests.mk | 5 +---- tests/local.mk | 10 +++++++--- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/mk/lib.mk b/mk/lib.mk index 34fa624d8..a6c2b32e4 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -10,6 +10,7 @@ bin-scripts := noinst-scripts := man-pages := install-tests := +install-tests-groups := ifdef HOST_OS HOST_KERNEL = $(firstword $(subst -, ,$(HOST_OS))) @@ -122,6 +123,14 @@ $(foreach script, $(bin-scripts), $(eval programs-list += $(script))) $(foreach script, $(noinst-scripts), $(eval programs-list += $(script))) $(foreach template, $(template-files), $(eval $(call instantiate-template,$(template)))) $(foreach test, $(install-tests), $(eval $(call run-install-test,$(test)))) +$(foreach test, $(install-tests), $(eval installcheck: $(test).test)) +$(foreach test-group, $(install-tests-groups), \ + $(eval installcheck: $(test-group).test-group) \ + $(eval .PHONY: $(test-group).test-group) \ + $(foreach test, $($(test-group)-tests), \ + $(eval $(call run-install-test,$(test))) \ + $(eval $(test-group).test-group: $(test).test))) + $(foreach file, $(man-pages), $(eval $(call install-data-in, $(file), $(mandir)/man$(patsubst .%,%,$(suffix $(file)))))) diff --git a/mk/tests.mk b/mk/tests.mk index 3ebbd86e3..dca3c5da2 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -2,10 +2,7 @@ test-deps = -define run-install-test - - installcheck: $1.test - +define run-install-test $1 .PHONY: $1.test $1.test: $1 $(test-deps) @env BASH=$(bash) $(bash) mk/run-test.sh $1 < /dev/null diff --git a/tests/local.mk b/tests/local.mk index 778d087b1..2dea58e7b 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -135,9 +135,13 @@ nix_tests = \ flakes/show.sh \ impure-derivations.sh \ path-from-hash-part.sh \ - toString-path.sh \ - overlay-local-store/outer.sh \ - overlay-local-store/bad-uris.sh + toString-path.sh + +overlay-local-store-tests := \ + $(d)/overlay-local-store/outer.sh \ + $(d)/overlay-local-store/bad-uris.sh + +install-tests-groups += overlay-local-store ifeq ($(HAVE_LIBCPUID), 1) nix_tests += compute-levels.sh From 5d18120ba837039718b3d08d627c7c231f379a40 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 15 May 2023 23:00:18 -0400 Subject: [PATCH 022/133] Split tests some more Good for parallelism and easier reading. --- tests/local.mk | 4 ++- tests/overlay-local-store/build-inner.sh | 26 +++++++++++++++++++ .../{outer.sh => build.sh} | 2 +- .../{inner.sh => check-post-init-inner.sh} | 26 ------------------- tests/overlay-local-store/check-post-init.sh | 5 ++++ tests/overlay-local-store/common.sh | 26 +++++++++++-------- .../redundant-add-inner.sh | 26 +++++++++++++++++++ tests/overlay-local-store/redundant-add.sh | 5 ++++ 8 files changed, 81 insertions(+), 39 deletions(-) create mode 100755 tests/overlay-local-store/build-inner.sh rename tests/overlay-local-store/{outer.sh => build.sh} (50%) rename tests/overlay-local-store/{inner.sh => check-post-init-inner.sh} (68%) create mode 100755 tests/overlay-local-store/check-post-init.sh create mode 100755 tests/overlay-local-store/redundant-add-inner.sh create mode 100755 tests/overlay-local-store/redundant-add.sh diff --git a/tests/local.mk b/tests/local.mk index 2dea58e7b..dd168ce3f 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -138,7 +138,9 @@ nix_tests = \ toString-path.sh overlay-local-store-tests := \ - $(d)/overlay-local-store/outer.sh \ + $(d)/overlay-local-store/check-post-init.sh \ + $(d)/overlay-local-store/redundant-add.sh \ + $(d)/overlay-local-store/build.sh \ $(d)/overlay-local-store/bad-uris.sh install-tests-groups += overlay-local-store diff --git a/tests/overlay-local-store/build-inner.sh b/tests/overlay-local-store/build-inner.sh new file mode 100755 index 000000000..969de282d --- /dev/null +++ b/tests/overlay-local-store/build-inner.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +set -x + +source common.sh + +storeDirs + +initLowerStore + +mountOverlayfs + +### Do a build in overlay store + +path=$(nix-build ../hermetic.nix --arg busybox $busybox --arg seed 2 --store "$storeB" --no-out-link) + +# Checking for path in lower layer (should fail) +expect 1 stat $(toRealPath "$storeA/nix/store" "$path") + +# Checking for path in upper layer +stat $(toRealPath "$storeBTop" "$path") + +# Verifying path in overlay store +nix-store --verify-path --store "$storeB" "$path" diff --git a/tests/overlay-local-store/outer.sh b/tests/overlay-local-store/build.sh similarity index 50% rename from tests/overlay-local-store/outer.sh rename to tests/overlay-local-store/build.sh index ba55f1d06..758585400 100755 --- a/tests/overlay-local-store/outer.sh +++ b/tests/overlay-local-store/build.sh @@ -2,4 +2,4 @@ source common.sh requireEnvironment setupConfig -exec unshare --mount --map-root-user ./inner.sh +execUnshare ./build-inner.sh diff --git a/tests/overlay-local-store/inner.sh b/tests/overlay-local-store/check-post-init-inner.sh similarity index 68% rename from tests/overlay-local-store/inner.sh rename to tests/overlay-local-store/check-post-init-inner.sh index adbe29557..421b64934 100755 --- a/tests/overlay-local-store/inner.sh +++ b/tests/overlay-local-store/check-post-init-inner.sh @@ -65,29 +65,3 @@ hashPart=$(echo $path | sed "s^$NIX_STORE_DIR/^^" | sed 's/-.*//') # merged store can find from hash part [[ $(nix store --store $storeB path-from-hash-part $hashPart) == $path ]] - -### Do a redundant add - -# upper layer should not have it -expect 1 stat $(toRealPath "$storeBTop/nix/store" "$path") - -path=$(nix-store --store "$storeB" --add ../dummy) - -# lower store should have it from before -stat $(toRealPath "$storeA/nix/store" "$path") - -# upper layer should still not have it (no redundant copy) -expect 1 stat $(toRealPath "$storeB/nix/store" "$path") - -### Do a build in overlay store - -path=$(nix-build ../hermetic.nix --arg busybox $busybox --arg seed 2 --store "$storeB" --no-out-link) - -# Checking for path in lower layer (should fail) -expect 1 stat $(toRealPath "$storeA/nix/store" "$path") - -# Checking for path in upper layer -stat $(toRealPath "$storeBTop" "$path") - -# Verifying path in overlay store -nix-store --verify-path --store "$storeB" "$path" diff --git a/tests/overlay-local-store/check-post-init.sh b/tests/overlay-local-store/check-post-init.sh new file mode 100755 index 000000000..985bf978e --- /dev/null +++ b/tests/overlay-local-store/check-post-init.sh @@ -0,0 +1,5 @@ +source common.sh + +requireEnvironment +setupConfig +execUnshare ./check-post-init-inner.sh diff --git a/tests/overlay-local-store/common.sh b/tests/overlay-local-store/common.sh index 6d9b69b4c..149102000 100644 --- a/tests/overlay-local-store/common.sh +++ b/tests/overlay-local-store/common.sh @@ -1,23 +1,23 @@ source ../common.sh requireEnvironment () { - requireSandboxSupport - [[ $busybox =~ busybox ]] || skipTest "no busybox" - if [[ $(uname) != Linux ]]; then skipTest "Need Linux for overlayfs"; fi - needLocalStore "The test uses --store always so we would just be bypassing the daemon" + requireSandboxSupport + [[ $busybox =~ busybox ]] || skipTest "no busybox" + if [[ $(uname) != Linux ]]; then skipTest "Need Linux for overlayfs"; fi + needLocalStore "The test uses --store always so we would just be bypassing the daemon" } setupConfig () { - echo "drop-supplementary-groups = false" >> "$NIX_CONF_DIR"/nix.conf - echo "build-users-group = " >> "$NIX_CONF_DIR"/nix.conf + echo "drop-supplementary-groups = false" >> "$NIX_CONF_DIR"/nix.conf + echo "build-users-group = " >> "$NIX_CONF_DIR"/nix.conf } storeDirs () { - storeA="$TEST_ROOT/store-a" - storeBTop="$TEST_ROOT/store-b" - storeB="local-overlay?root=$TEST_ROOT/merged-store&lower-store=$storeA&upper-layer=$storeBTop" - # Creating testing directories - mkdir -p "$TEST_ROOT"/{store-a/nix/store,store-b,merged-store/nix/store,workdir} + storeA="$TEST_ROOT/store-a" + storeBTop="$TEST_ROOT/store-b" + storeB="local-overlay?root=$TEST_ROOT/merged-store&lower-store=$storeA&upper-layer=$storeBTop" + # Creating testing directories + mkdir -p "$TEST_ROOT"/{store-a/nix/store,store-b,merged-store/nix/store,workdir} } # Mounting Overlay Store @@ -50,3 +50,7 @@ initLowerStore () { drvPath=$(nix-instantiate --store $storeA ../hermetic.nix --arg busybox "$busybox" --arg seed 1) path=$(nix-store --store "$storeA" --realise $drvPath) } + +execUnshare () { + exec unshare --mount --map-root-user "$@" +} diff --git a/tests/overlay-local-store/redundant-add-inner.sh b/tests/overlay-local-store/redundant-add-inner.sh new file mode 100755 index 000000000..921069c25 --- /dev/null +++ b/tests/overlay-local-store/redundant-add-inner.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +set -x + +source common.sh + +storeDirs + +initLowerStore + +mountOverlayfs + +### Do a redundant add + +# upper layer should not have it +expect 1 stat $(toRealPath "$storeBTop/nix/store" "$path") + +path=$(nix-store --store "$storeB" --add ../dummy) + +# lower store should have it from before +stat $(toRealPath "$storeA/nix/store" "$path") + +# upper layer should still not have it (no redundant copy) +expect 1 stat $(toRealPath "$storeB/nix/store" "$path") diff --git a/tests/overlay-local-store/redundant-add.sh b/tests/overlay-local-store/redundant-add.sh new file mode 100755 index 000000000..fbd4799e7 --- /dev/null +++ b/tests/overlay-local-store/redundant-add.sh @@ -0,0 +1,5 @@ +source common.sh + +requireEnvironment +setupConfig +execUnshare ./redundant-add-inner.sh From 0ec7f2fb3f935f39585a6f1daa21a7520e1a4b40 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 15 May 2023 23:10:53 -0400 Subject: [PATCH 023/133] Create `local.mk` for local-overlay-store tests --- Makefile | 1 + tests/local.mk | 8 -------- tests/overlay-local-store/local.mk | 7 +++++++ 3 files changed, 8 insertions(+), 8 deletions(-) create mode 100644 tests/overlay-local-store/local.mk diff --git a/Makefile b/Makefile index d6b49473a..3ce5cc5e3 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,7 @@ makefiles += \ src/libstore/tests/local.mk \ src/libexpr/tests/local.mk \ tests/local.mk \ + tests/overlay-local-store/local.mk \ tests/plugins/local.mk else makefiles += \ diff --git a/tests/local.mk b/tests/local.mk index dd168ce3f..9cb81e1f0 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -137,14 +137,6 @@ nix_tests = \ path-from-hash-part.sh \ toString-path.sh -overlay-local-store-tests := \ - $(d)/overlay-local-store/check-post-init.sh \ - $(d)/overlay-local-store/redundant-add.sh \ - $(d)/overlay-local-store/build.sh \ - $(d)/overlay-local-store/bad-uris.sh - -install-tests-groups += overlay-local-store - ifeq ($(HAVE_LIBCPUID), 1) nix_tests += compute-levels.sh endif diff --git a/tests/overlay-local-store/local.mk b/tests/overlay-local-store/local.mk new file mode 100644 index 000000000..b94238a67 --- /dev/null +++ b/tests/overlay-local-store/local.mk @@ -0,0 +1,7 @@ +overlay-local-store-tests := \ + $(d)/check-post-init.sh \ + $(d)/redundant-add.sh \ + $(d)/build.sh \ + $(d)/bad-uris.sh + +install-tests-groups += overlay-local-store From 4d69bd034ae3435b20d1b5d38a819247a3031c25 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 16 May 2023 13:10:25 +0100 Subject: [PATCH 024/133] More detailed explanation of check-mount setting. --- src/libstore/local-overlay-store.hh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 525a54745..6f798c460 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -31,7 +31,15 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig )"}; Setting checkMount{(StoreConfig*) this, true, "check-mount", - "Check that the overlay filesystem is correctly mounted."}; + R"( + Check that the overlay filesystem is correctly mounted. + + Nix does not manage the overlayfs mount point itself, but the correct + functioning of the overlay store does depend on this mount point being set up + correctly. Rather than just assume this is the case, check that the lowerdir + and upperdir options are what we expect them to be. This check is on by + default, but can be disabled if needed. + )"}; const std::string name() override { return "Experimental Local Overlay Store"; } From de359da09a0a3a0b68cbcb20c25adc3908046296 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 16 May 2023 13:14:58 +0100 Subject: [PATCH 025/133] Add read-only setting to LocalStoreConfig. --- src/libstore/local-store.hh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 55add18dd..1557c6242 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -46,6 +46,11 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig "require-sigs", "Whether store paths copied into this store should have a trusted signature."}; + Setting readOnly{(StoreConfig*) this, + false, + "read-only", + "TODO"}; + const std::string name() override { return "Local Store"; } std::string doc() override; From 79583c2d38b75d227886679bca35a5b38a6e2119 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 16 May 2023 13:29:16 +0100 Subject: [PATCH 026/133] Do not attempt to chmod per-user dir when read-only. --- src/libstore/local-store.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 7fb312c37..d219f3e95 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -204,8 +204,10 @@ LocalStore::LocalStore(const Params & params) for (auto & perUserDir : {profilesDir + "/per-user", gcRootsDir + "/per-user"}) { createDirs(perUserDir); - if (chmod(perUserDir.c_str(), 0755) == -1) - throw SysError("could not set permissions on '%s' to 755", perUserDir); + if (!readOnly) { + if (chmod(perUserDir.c_str(), 0755) == -1) + throw SysError("could not set permissions on '%s' to 755", perUserDir); + } } /* Optionally, create directories and set permissions for a From 50bbdc65c8adccaf2d34fd90ee9f3897c347c667 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 16 May 2023 13:53:31 +0100 Subject: [PATCH 027/133] Do not attempt to acquire big-lock when read-only. --- src/libstore/local-store.cc | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index d219f3e95..94176c62f 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -271,10 +271,12 @@ LocalStore::LocalStore(const Params & params) /* Acquire the big fat lock in shared mode to make sure that no schema upgrade is in progress. */ - Path globalLockPath = dbDir + "/big-lock"; - globalLock = openLockFile(globalLockPath.c_str(), true); + if (!readOnly) { + Path globalLockPath = dbDir + "/big-lock"; + globalLock = openLockFile(globalLockPath.c_str(), true); + } - if (!lockFile(globalLock.get(), ltRead, false)) { + if (!readOnly && !lockFile(globalLock.get(), ltRead, false)) { printInfo("waiting for the big Nix store lock..."); lockFile(globalLock.get(), ltRead, true); } @@ -305,7 +307,7 @@ LocalStore::LocalStore(const Params & params) "which is no longer supported. To convert to the new format,\n" "please upgrade Nix to version 1.11 first."); - if (!lockFile(globalLock.get(), ltWrite, false)) { + if (!readOnly && !lockFile(globalLock.get(), ltWrite, false)) { printInfo("waiting for exclusive access to the Nix store..."); lockFile(globalLock.get(), ltNone, false); // We have acquired a shared lock; release it to prevent deadlocks lockFile(globalLock.get(), ltWrite, true); @@ -340,7 +342,8 @@ LocalStore::LocalStore(const Params & params) writeFile(schemaPath, fmt("%1%", nixSchemaVersion), 0666, true); - lockFile(globalLock.get(), ltRead, true); + if (!readOnly) + lockFile(globalLock.get(), ltRead, true); } else openDB(*state, false); From c22936ca6ad68630f2982a62245b91b11c4fb699 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 16 May 2023 13:56:29 +0100 Subject: [PATCH 028/133] Do not attempt to migrate to CA schema when read-only. --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 94176c62f..5d9e96441 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -348,7 +348,7 @@ LocalStore::LocalStore(const Params & params) else openDB(*state, false); - if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { + if (!readOnly && experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { migrateCASchema(state->db, dbDir + "/ca-schema", globalLock); } From 7f443e04283b027dff8589eb54473313be877577 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 16 May 2023 13:57:13 +0100 Subject: [PATCH 029/133] Do not check for write access to database when read-only. --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 5d9e96441..b0f6709aa 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -480,7 +480,7 @@ int LocalStore::getSchema() void LocalStore::openDB(State & state, bool create) { - if (access(dbDir.c_str(), R_OK | W_OK)) + if (access(dbDir.c_str(), R_OK | (readOnly ? 0 : W_OK))) throw SysError("Nix database directory '%1%' is not writable", dbDir); /* Open the Nix database. */ From afed9ccfadc24a6a58da781cfe7eb5d1a4e4f7c3 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 16 May 2023 13:57:56 +0100 Subject: [PATCH 030/133] Add enum for intended sqlite database open modes. --- src/libstore/sqlite.hh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index 6e14852cb..8f4bc49a8 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -11,6 +11,24 @@ struct sqlite3_stmt; namespace nix { +enum class SQLiteOpenMode { + /** + * Open the database in read-write mode. + * If the database does not exist, it will be created. + */ + Normal, + /** + * Open the database in read-write mode. + * Fails with an error if the database does not exist. + */ + NoCreate, + /** + * Open the database in read-only mode. + * Fails with an error if the database does not exist. + */ + ReadOnly +}; + /** * RAII wrapper to close a SQLite database automatically. */ From 78fdd6f24e79f50f7d27a2576685398e67928f61 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 16 May 2023 14:13:03 +0100 Subject: [PATCH 031/133] Open sqlite database according to new modes. --- src/libstore/local-store.cc | 9 ++++++++- src/libstore/sqlite.cc | 7 ++++--- src/libstore/sqlite.hh | 2 +- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index b0f6709aa..1c5d8fc9e 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -480,13 +480,20 @@ int LocalStore::getSchema() void LocalStore::openDB(State & state, bool create) { + if (create && readOnly) { + throw Error("unable to create database while in read-only mode"); + } + if (access(dbDir.c_str(), R_OK | (readOnly ? 0 : W_OK))) throw SysError("Nix database directory '%1%' is not writable", dbDir); /* Open the Nix database. */ std::string dbPath = dbDir + "/db.sqlite"; auto & db(state.db); - state.db = SQLite(dbPath, create); + auto openMode = readOnly ? SQLiteOpenMode::ReadOnly + : create ? SQLiteOpenMode::Normal + : SQLiteOpenMode::NoCreate; + state.db = SQLite(dbPath, openMode); #ifdef __CYGWIN__ /* The cygwin version of sqlite3 has a patch which calls diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index df334c23c..8159744b7 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -50,14 +50,15 @@ static void traceSQL(void * x, const char * sql) notice("SQL<[%1%]>", sql); }; -SQLite::SQLite(const Path & path, bool create) +SQLite::SQLite(const Path & path, SQLiteOpenMode mode) { // useSQLiteWAL also indicates what virtual file system we need. Using // `unix-dotfile` is needed on NFS file systems and on Windows' Subsystem // for Linux (WSL) where useSQLiteWAL should be false by default. const char *vfs = settings.useSQLiteWAL ? 0 : "unix-dotfile"; - int flags = SQLITE_OPEN_READWRITE; - if (create) flags |= SQLITE_OPEN_CREATE; + int flags = mode == SQLiteOpenMode::ReadOnly + ? SQLITE_OPEN_READONLY : SQLITE_OPEN_READWRITE; + if (mode == SQLiteOpenMode::Normal) flags |= SQLITE_OPEN_CREATE; int ret = sqlite3_open_v2(path.c_str(), &db, flags, vfs); if (ret != SQLITE_OK) { const char * err = sqlite3_errstr(ret); diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index 8f4bc49a8..33f0eeed8 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -36,7 +36,7 @@ struct SQLite { sqlite3 * db = 0; SQLite() { } - SQLite(const Path & path, bool create = true); + SQLite(const Path & path, SQLiteOpenMode mode = SQLiteOpenMode::Normal); SQLite(const SQLite & from) = delete; SQLite& operator = (const SQLite & from) = delete; SQLite& operator = (SQLite && from) { db = from.db; from.db = 0; return *this; } From aa376f4ab17142516b19de6aa0b990b6d097b2f2 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 16 May 2023 15:29:29 +0100 Subject: [PATCH 032/133] Need to open database using immutable parameter. This requires switching on SQLITE_OPEN_URI because there is no open flag to make the database immutable. Without immutable, sqlite will still attempt to create journal and wal files, even when the database is opened read-only. https://www.sqlite.org/c3ref/open.html The immutable parameter is a boolean query parameter that indicates that the database file is stored on read-only media. When immutable is set, SQLite assumes that the database file cannot be changed, even by a process with higher privilege, and so the database is opened read-only and all locking and change detection is disabled. --- src/libstore/sqlite.cc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 8159744b7..4166cc2cd 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -1,6 +1,7 @@ #include "sqlite.hh" #include "globals.hh" #include "util.hh" +#include "url.hh" #include @@ -52,14 +53,16 @@ static void traceSQL(void * x, const char * sql) SQLite::SQLite(const Path & path, SQLiteOpenMode mode) { + bool readOnly = mode == SQLiteOpenMode::ReadOnly; + // useSQLiteWAL also indicates what virtual file system we need. Using // `unix-dotfile` is needed on NFS file systems and on Windows' Subsystem // for Linux (WSL) where useSQLiteWAL should be false by default. const char *vfs = settings.useSQLiteWAL ? 0 : "unix-dotfile"; - int flags = mode == SQLiteOpenMode::ReadOnly - ? SQLITE_OPEN_READONLY : SQLITE_OPEN_READWRITE; + int flags = readOnly ? SQLITE_OPEN_READONLY : SQLITE_OPEN_READWRITE; if (mode == SQLiteOpenMode::Normal) flags |= SQLITE_OPEN_CREATE; - int ret = sqlite3_open_v2(path.c_str(), &db, flags, vfs); + auto uri = "file:" + percentEncode(path) + "?immutable=" + (readOnly ? "1" : "0"); + int ret = sqlite3_open_v2(uri.c_str(), &db, SQLITE_OPEN_URI | flags, vfs); if (ret != SQLITE_OK) { const char * err = sqlite3_errstr(ret); throw Error("cannot open SQLite database '%s': %s", path, err); From b1a7b26eef46106deec0a954d400e5a1aac47945 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 16 May 2023 15:48:40 +0100 Subject: [PATCH 033/133] Rename ReadOnly to Immutable and clarify its purpose. --- src/libstore/local-store.cc | 2 +- src/libstore/sqlite.cc | 7 +++---- src/libstore/sqlite.hh | 7 +++++-- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 1c5d8fc9e..59685400f 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -490,7 +490,7 @@ void LocalStore::openDB(State & state, bool create) /* Open the Nix database. */ std::string dbPath = dbDir + "/db.sqlite"; auto & db(state.db); - auto openMode = readOnly ? SQLiteOpenMode::ReadOnly + auto openMode = readOnly ? SQLiteOpenMode::Immutable : create ? SQLiteOpenMode::Normal : SQLiteOpenMode::NoCreate; state.db = SQLite(dbPath, openMode); diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 4166cc2cd..7c8decb74 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -53,15 +53,14 @@ static void traceSQL(void * x, const char * sql) SQLite::SQLite(const Path & path, SQLiteOpenMode mode) { - bool readOnly = mode == SQLiteOpenMode::ReadOnly; - // useSQLiteWAL also indicates what virtual file system we need. Using // `unix-dotfile` is needed on NFS file systems and on Windows' Subsystem // for Linux (WSL) where useSQLiteWAL should be false by default. const char *vfs = settings.useSQLiteWAL ? 0 : "unix-dotfile"; - int flags = readOnly ? SQLITE_OPEN_READONLY : SQLITE_OPEN_READWRITE; + bool immutable = mode == SQLiteOpenMode::Immutable; + int flags = immutable ? SQLITE_OPEN_READONLY : SQLITE_OPEN_READWRITE; if (mode == SQLiteOpenMode::Normal) flags |= SQLITE_OPEN_CREATE; - auto uri = "file:" + percentEncode(path) + "?immutable=" + (readOnly ? "1" : "0"); + auto uri = "file:" + percentEncode(path) + "?immutable=" + (immutable ? "1" : "0"); int ret = sqlite3_open_v2(uri.c_str(), &db, SQLITE_OPEN_URI | flags, vfs); if (ret != SQLITE_OK) { const char * err = sqlite3_errstr(ret); diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index 33f0eeed8..b0e84f8ed 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -23,10 +23,13 @@ enum class SQLiteOpenMode { */ NoCreate, /** - * Open the database in read-only mode. + * Open the database in immutable mode. + * In addition to the database being read-only, + * no wal or journal files will be created by sqlite. + * Use this mode if the database is on a read-only filesystem. * Fails with an error if the database does not exist. */ - ReadOnly + Immutable }; /** From 5966b76c974a8f2eddac8c610b754a2dbcf8bb9b Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Wed, 17 May 2023 08:49:51 +0100 Subject: [PATCH 034/133] Document the new read-only local store setting. --- src/libstore/local-store.hh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 1557c6242..7fcf645e7 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -49,7 +49,18 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig Setting readOnly{(StoreConfig*) this, false, "read-only", - "TODO"}; + R"( + Allow this store to be opened when its database is on a read-only filesystem. + + Normally Nix will attempt to open the store database in read-write mode, even + for querying (when write access is not needed). This causes it to fail if the + database is on a read-only filesystem. + + Enable read-only mode to disable locking and open the SQLite database with the + **imutable** parameter set. Do not use this unless the filesystem is read-only. + Using it when the filesystem is writable can cause incorrect query results or + corruption errors if the database is changed by another process. + )"}; const std::string name() override { return "Local Store"; } From 85a24530524e5a7410fd00714a4a8f4633b4f758 Mon Sep 17 00:00:00 2001 From: cidkidnix Date: Wed, 17 May 2023 13:48:59 -0500 Subject: [PATCH 035/133] Add tests for read-only local store Make sure we don't go down the path of making temproots when doing operations on a read-only store --- src/libstore/gc.cc | 4 ++++ tests/local.mk | 3 ++- tests/read-only-store.sh | 29 +++++++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 tests/read-only-store.sh diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 0038ec802..09f6ddb9e 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -110,6 +110,10 @@ void LocalStore::createTempRootsFile() void LocalStore::addTempRoot(const StorePath & path) { + if (readOnly) { + debug("Read-only store doesn't support creating lock files for temp roots, but nothing can be deleted anyways."); + return; + } createTempRootsFile(); /* Open/create the global GC lock file. */ diff --git a/tests/local.mk b/tests/local.mk index 9e340e2e2..576529567 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -134,7 +134,8 @@ nix_tests = \ flakes/show.sh \ impure-derivations.sh \ path-from-hash-part.sh \ - toString-path.sh + toString-path.sh \ + read-only-store.sh ifeq ($(HAVE_LIBCPUID), 1) nix_tests += compute-levels.sh diff --git a/tests/read-only-store.sh b/tests/read-only-store.sh new file mode 100644 index 000000000..20ded9634 --- /dev/null +++ b/tests/read-only-store.sh @@ -0,0 +1,29 @@ +source common.sh + +clearStore + +## Testing read-only mode without forcing the underlying store to actually be read-only + +# Make sure the command fails when the store doesn't already have a database +expectStderr 1 nix-store --store local?read-only=true --add dummy | grepQuiet "unable to create database while in read-only mode" + +# Make sure the store actually has a current-database +nix-store --add dummy + +# Try again and make sure we fail when adding a item not already in the store +expectStderr 1 nix-store --store local?read-only=true --add eval.nix | grepQuiet "attempt to write a readonly database" + +# Make sure we can get an already-present store-path in the database +nix-store --store local?read-only=true --add dummy + +## Ensure store is actually read-only +chmod -R -w $TEST_ROOT/store +chmod -R -w $TEST_ROOT/var + +# Make sure we fail on add operations on the read-only store +# This is only for adding files that are not *already* in the store +expectStderr 1 nix-store --add eval.nix | grepQuiet "error: opening lock file '$(readlink -e $TEST_ROOT)/var/nix/db/big-lock'" +expectStderr 1 nix-store --store local?read-only=true --add eval.nix | grepQuiet "Permission denied" + +# Should succeed +nix-store --store local?read-only=true --add dummy From 9290af763a7eb93e1967d540c7505c4514b2c546 Mon Sep 17 00:00:00 2001 From: Dylan Green <67574902+cidkidnix@users.noreply.github.com> Date: Wed, 17 May 2023 14:52:13 -0500 Subject: [PATCH 036/133] Update tests/read-only-store.sh Co-authored-by: John Ericson --- tests/read-only-store.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/read-only-store.sh b/tests/read-only-store.sh index 20ded9634..b440fe356 100644 --- a/tests/read-only-store.sh +++ b/tests/read-only-store.sh @@ -16,7 +16,9 @@ expectStderr 1 nix-store --store local?read-only=true --add eval.nix | grepQuiet # Make sure we can get an already-present store-path in the database nix-store --store local?read-only=true --add dummy -## Ensure store is actually read-only +## Testing read-only mode with an underlying store that is actually read-only + +# Ensure store is actually read-only chmod -R -w $TEST_ROOT/store chmod -R -w $TEST_ROOT/var From 60c014972171edf271b173ab03e13866c2839035 Mon Sep 17 00:00:00 2001 From: Dylan Green <67574902+cidkidnix@users.noreply.github.com> Date: Wed, 17 May 2023 14:56:48 -0500 Subject: [PATCH 037/133] Apply suggestions from code review Co-authored-by: John Ericson --- src/libstore/gc.cc | 1 + tests/read-only-store.sh | 19 +++++++++++++------ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 09f6ddb9e..3c9544017 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -114,6 +114,7 @@ void LocalStore::addTempRoot(const StorePath & path) debug("Read-only store doesn't support creating lock files for temp roots, but nothing can be deleted anyways."); return; } + createTempRootsFile(); /* Open/create the global GC lock file. */ diff --git a/tests/read-only-store.sh b/tests/read-only-store.sh index b440fe356..72d855d42 100644 --- a/tests/read-only-store.sh +++ b/tests/read-only-store.sh @@ -2,19 +2,26 @@ source common.sh clearStore +happy () { + # We can do a read-only query just fine with a read-only store + nix --store local?read-only=true path-info $dummyPath + + # We can "write" an already-present store-path a read-only store, because no IO is actually required + nix-store --store local?read-only=true --add dummy +} ## Testing read-only mode without forcing the underlying store to actually be read-only # Make sure the command fails when the store doesn't already have a database expectStderr 1 nix-store --store local?read-only=true --add dummy | grepQuiet "unable to create database while in read-only mode" -# Make sure the store actually has a current-database -nix-store --add dummy +# Make sure the store actually has a current-database, with at least one store object +dummyPath=$(nix-store --add dummy) # Try again and make sure we fail when adding a item not already in the store expectStderr 1 nix-store --store local?read-only=true --add eval.nix | grepQuiet "attempt to write a readonly database" -# Make sure we can get an already-present store-path in the database -nix-store --store local?read-only=true --add dummy +# Test a few operations that should work with the read-only store in its current state +happy ## Testing read-only mode with an underlying store that is actually read-only @@ -27,5 +34,5 @@ chmod -R -w $TEST_ROOT/var expectStderr 1 nix-store --add eval.nix | grepQuiet "error: opening lock file '$(readlink -e $TEST_ROOT)/var/nix/db/big-lock'" expectStderr 1 nix-store --store local?read-only=true --add eval.nix | grepQuiet "Permission denied" -# Should succeed -nix-store --store local?read-only=true --add dummy +# Test the same operations from before should again succeed +happy From fe174d72a2b8ef7a4a6415bf284b1e282e2052cf Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Thu, 18 May 2023 13:44:39 +0100 Subject: [PATCH 038/133] Fix spelling of 'immutable' in documentation. --- src/libstore/local-store.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 7fcf645e7..37e31210f 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -57,7 +57,7 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig database is on a read-only filesystem. Enable read-only mode to disable locking and open the SQLite database with the - **imutable** parameter set. Do not use this unless the filesystem is read-only. + **immutable** parameter set. Do not use this unless the filesystem is read-only. Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process. )"}; From d55e38b98a281b6915cd4b57b597059d4be9b0ac Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Thu, 18 May 2023 13:45:31 +0100 Subject: [PATCH 039/133] Check earlier whether schema migration is required. --- src/libstore/local-store.cc | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 59685400f..99fb9f434 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -284,6 +284,14 @@ LocalStore::LocalStore(const Params & params) /* Check the current database schema and if necessary do an upgrade. */ int curSchema = getSchema(); + if (readOnly && curSchema < nixSchemaVersion) { + debug("current schema version: %d", curSchema); + debug("supported schema version: %d", nixSchemaVersion); + throw Error(curSchema == 0 ? + "database does not exist, and cannot be created in read-only mode" : + "database schema needs migrating, but this cannot be done in read-only mode"); + } + if (curSchema > nixSchemaVersion) throw Error("current Nix store schema is version %1%, but I only support %2%", curSchema, nixSchemaVersion); @@ -307,7 +315,7 @@ LocalStore::LocalStore(const Params & params) "which is no longer supported. To convert to the new format,\n" "please upgrade Nix to version 1.11 first."); - if (!readOnly && !lockFile(globalLock.get(), ltWrite, false)) { + if (!lockFile(globalLock.get(), ltWrite, false)) { printInfo("waiting for exclusive access to the Nix store..."); lockFile(globalLock.get(), ltNone, false); // We have acquired a shared lock; release it to prevent deadlocks lockFile(globalLock.get(), ltWrite, true); @@ -342,8 +350,7 @@ LocalStore::LocalStore(const Params & params) writeFile(schemaPath, fmt("%1%", nixSchemaVersion), 0666, true); - if (!readOnly) - lockFile(globalLock.get(), ltRead, true); + lockFile(globalLock.get(), ltRead, true); } else openDB(*state, false); From 8ffeb1c4e58e8ed0032b097d3d1d9625726f5fec Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Thu, 18 May 2023 13:47:31 +0100 Subject: [PATCH 040/133] Throw error instead of silently skipping CA migration. --- src/libstore/local-store.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 99fb9f434..c25aa0e72 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -355,8 +355,12 @@ LocalStore::LocalStore(const Params & params) else openDB(*state, false); - if (!readOnly && experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { - migrateCASchema(state->db, dbDir + "/ca-schema", globalLock); + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { + if (!readOnly) { + migrateCASchema(state->db, dbDir + "/ca-schema", globalLock); + } else { + throw Error("need to migrate to CA schema, but this cannot be done in read-only mode"); + } } /* Prepare SQL statements. */ From 0c36fe6c8cb4af5558c3a6a75240ef3b01510084 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Thu, 18 May 2023 14:36:24 +0100 Subject: [PATCH 041/133] Update test to match new error message. --- tests/read-only-store.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/read-only-store.sh b/tests/read-only-store.sh index 72d855d42..408ffa63f 100644 --- a/tests/read-only-store.sh +++ b/tests/read-only-store.sh @@ -12,7 +12,7 @@ happy () { ## Testing read-only mode without forcing the underlying store to actually be read-only # Make sure the command fails when the store doesn't already have a database -expectStderr 1 nix-store --store local?read-only=true --add dummy | grepQuiet "unable to create database while in read-only mode" +expectStderr 1 nix-store --store local?read-only=true --add dummy | grepQuiet "database does not exist, and cannot be created in read-only mode" # Make sure the store actually has a current-database, with at least one store object dummyPath=$(nix-store --add dummy) From 72518000860b56f5cd08a3b3a4fdfc61c021e2e2 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Mon, 22 May 2023 11:38:37 +0100 Subject: [PATCH 042/133] Put read-only setting behind an experimental flag. --- src/libstore/local-store.cc | 4 ++++ src/libutil/experimental-features.cc | 18 +++++++++++++++++- src/libutil/experimental-features.hh | 1 + 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c25aa0e72..4188d413f 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -202,6 +202,10 @@ LocalStore::LocalStore(const Params & params) createSymlink(profilesDir, gcRootsDir + "/profiles"); } + if (readOnly) { + experimentalFeatureSettings.require(Xp::ReadOnlyLocalStore); + } + for (auto & perUserDir : {profilesDir + "/per-user", gcRootsDir + "/per-user"}) { createDirs(perUserDir); if (!readOnly) { diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index ad0ec0427..29f5b84db 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -12,7 +12,7 @@ struct ExperimentalFeatureDetails std::string_view description; }; -constexpr std::array xpFeatureDetails = {{ +constexpr std::array xpFeatureDetails = {{ { .tag = Xp::CaDerivations, .name = "ca-derivations", @@ -209,6 +209,22 @@ constexpr std::array xpFeatureDetails = {{ files. )", }, + { + .tag = Xp::ReadOnlyLocalStore, + .name = "read-only-local-store", + .description = R"( + Allow the use of the `read-only` parameter in local store URIs. + + Set this parameter to `true` to allow stores with databases on read-only + filesystems to be opened for querying; ordinarily Nix will refuse to do this. + + Enabling this setting disables the locking required for safe concurrent + access, so you should be certain that the database will not be changed. + While the filesystem the database resides on might be read-only to this + process, consider whether another user, process, or system, might have + write access to it. + )", + }, }}; static_assert( diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index 409100592..19d425b6d 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -30,6 +30,7 @@ enum struct ExperimentalFeature DiscardReferences, DaemonTrustOverride, DynamicDerivations, + ReadOnlyLocalStore, }; /** From d6ea3b6a19b7d48a7fb4cc03b0eaf080217fa84f Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Mon, 22 May 2023 12:14:07 +0100 Subject: [PATCH 043/133] Need to enable read-only-local-store flag for test. --- tests/read-only-store.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/read-only-store.sh b/tests/read-only-store.sh index 408ffa63f..2f89b849f 100644 --- a/tests/read-only-store.sh +++ b/tests/read-only-store.sh @@ -1,5 +1,7 @@ source common.sh +enableFeatures "read-only-local-store" + clearStore happy () { From c47f744e05d8c80bd65083b0fc0fbfef2ff0c08f Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Thu, 1 Jun 2023 14:14:13 +0100 Subject: [PATCH 044/133] Also skip makeStoreWritable when read-only=true. --- src/libstore/local-store.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 4188d413f..17385cdfb 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -190,7 +190,11 @@ LocalStore::LocalStore(const Params & params) /* Create missing state directories if they don't already exist. */ createDirs(realStoreDir); - makeStoreWritable(); + if (readOnly) { + experimentalFeatureSettings.require(Xp::ReadOnlyLocalStore); + } else { + makeStoreWritable(); + } createDirs(linksDir); Path profilesDir = stateDir + "/profiles"; createDirs(profilesDir); @@ -202,10 +206,6 @@ LocalStore::LocalStore(const Params & params) createSymlink(profilesDir, gcRootsDir + "/profiles"); } - if (readOnly) { - experimentalFeatureSettings.require(Xp::ReadOnlyLocalStore); - } - for (auto & perUserDir : {profilesDir + "/per-user", gcRootsDir + "/per-user"}) { createDirs(perUserDir); if (!readOnly) { From 98edbb968617b2478f52ab527bc4b1a75933b8c6 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Wed, 24 May 2023 11:24:07 +0100 Subject: [PATCH 045/133] Factor out GC path deletion so it can be overridden. --- src/libstore/gc-store.hh | 7 +++++++ src/libstore/gc.cc | 11 +++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/libstore/gc-store.hh b/src/libstore/gc-store.hh index 2c26c65c4..da1551056 100644 --- a/src/libstore/gc-store.hh +++ b/src/libstore/gc-store.hh @@ -97,6 +97,13 @@ struct GcStore : public virtual Store * Perform a garbage collection. */ virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0; + + /** + * Called by `collectGarbage` to recursively delete a path. + * The default implementation simply calls `deletePath`, but it can be + * overridden by stores that wish to provide their own deletion behaviour. + */ + virtual void deleteGCPath(const Path & path, uint64_t & bytesFreed); }; } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 3c9544017..959f4d3b1 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -654,7 +654,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) results.paths.insert(path); uint64_t bytesFreed; - deletePath(realPath, bytesFreed); + deleteGCPath(realPath, bytesFreed); + results.bytesFreed += bytesFreed; if (results.bytesFreed > options.maxFreed) { @@ -872,7 +873,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) if (unlink(path.c_str()) == -1) throw SysError("deleting '%1%'", path); - /* Do not accound for deleted file here. Rely on deletePath() + /* Do not account for deleted file here. Rely on deletePath() accounting. */ } @@ -890,6 +891,12 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) } +void GcStore::deleteGCPath(const Path & path, uint64_t & bytesFreed) +{ + deletePath(path, bytesFreed); +} + + void LocalStore::autoGC(bool sync) { static auto fakeFreeSpaceFile = getEnv("_NIX_TEST_FREE_SPACE_FILE"); From a48acfd68424960c9a7a5d4eeb1af2a7ea91835d Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Wed, 24 May 2023 11:26:33 +0100 Subject: [PATCH 046/133] Skip deletion of lower paths for overlay store GC. --- src/libstore/local-overlay-store.cc | 14 +++++++++++++- src/libstore/local-overlay-store.hh | 2 ++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index ab7af6aea..f42b37324 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -182,7 +182,7 @@ void LocalOverlayStore::addToStore(const ValidPathInfo & info, Source & source, LocalStore::addToStore(info, source, repair, checkSigs); if (lowerStore->isValidPath(info.path)) { // dedup stores - deletePath(toUpperPath(info.path)); + deletePath(toUpperPath(info.path)); // TODO: Investigate whether this can trigger 'stale file handle' errors. } } @@ -214,6 +214,18 @@ StorePath LocalOverlayStore::addTextToStore( } +void LocalOverlayStore::deleteGCPath(const Path & path, uint64_t & bytesFreed) +{ + auto mergedDir = realStoreDir.get() + "/"; + if (path.substr(0, mergedDir.length()) != mergedDir) { + warn("local-overlay: unexpected gc path '%s' ", path); + return; + } + if (pathExists(toUpperPath({path.substr(mergedDir.length())}))) { + GcStore::deleteGCPath(path, bytesFreed); + } +} + static RegisterStoreImplementation regLocalOverlayStore; } diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 6f798c460..701024bfb 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -122,6 +122,8 @@ private: void queryRealisationUncached(const DrvOutput&, Callback> callback) noexcept override; + + void deleteGCPath(const Path & path, uint64_t & bytesFreed) override; }; } From 8a9baa0a3091bd2ebd8d6027e0130c8b6dec76c5 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 6 Jun 2023 12:06:32 +0100 Subject: [PATCH 047/133] More sensible to have deleteGCPath in LocalStore. --- src/libstore/gc-store.hh | 7 ------- src/libstore/gc.cc | 6 ------ src/libstore/local-overlay-store.cc | 2 +- src/libstore/local-store.cc | 6 ++++++ src/libstore/local-store.hh | 7 +++++++ 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/libstore/gc-store.hh b/src/libstore/gc-store.hh index da1551056..2c26c65c4 100644 --- a/src/libstore/gc-store.hh +++ b/src/libstore/gc-store.hh @@ -97,13 +97,6 @@ struct GcStore : public virtual Store * Perform a garbage collection. */ virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0; - - /** - * Called by `collectGarbage` to recursively delete a path. - * The default implementation simply calls `deletePath`, but it can be - * overridden by stores that wish to provide their own deletion behaviour. - */ - virtual void deleteGCPath(const Path & path, uint64_t & bytesFreed); }; } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 959f4d3b1..80875efed 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -891,12 +891,6 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) } -void GcStore::deleteGCPath(const Path & path, uint64_t & bytesFreed) -{ - deletePath(path, bytesFreed); -} - - void LocalStore::autoGC(bool sync) { static auto fakeFreeSpaceFile = getEnv("_NIX_TEST_FREE_SPACE_FILE"); diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index f42b37324..061478c52 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -222,7 +222,7 @@ void LocalOverlayStore::deleteGCPath(const Path & path, uint64_t & bytesFreed) return; } if (pathExists(toUpperPath({path.substr(mergedDir.length())}))) { - GcStore::deleteGCPath(path, bytesFreed); + LocalStore::deleteGCPath(path, bytesFreed); } } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 17385cdfb..172fa1ef0 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -457,6 +457,12 @@ AutoCloseFD LocalStore::openGCLock() } +void LocalStore::deleteGCPath(const Path & path, uint64_t & bytesFreed) +{ + deletePath(path, bytesFreed); +} + + LocalStore::~LocalStore() { std::shared_future future; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 46a660c68..fcddffa82 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -222,6 +222,13 @@ public: void collectGarbage(const GCOptions & options, GCResults & results) override; + /** + * Called by `collectGarbage` to recursively delete a path. + * The default implementation simply calls `deletePath`, but it can be + * overridden by stores that wish to provide their own deletion behaviour. + */ + virtual void deleteGCPath(const Path & path, uint64_t & bytesFreed); + /** * Optimise the disk space usage of the Nix store by hard-linking * files with the same contents. From ee1241da8649fb310f8b5ac28d27f3c08117cdf9 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 6 Jun 2023 12:12:25 +0100 Subject: [PATCH 048/133] Remove unnecessary overrides of add methods. --- src/libstore/local-overlay-store.cc | 38 ----------------------------- src/libstore/local-overlay-store.hh | 12 --------- 2 files changed, 50 deletions(-) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 061478c52..ddccb8c5a 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -176,44 +176,6 @@ void LocalOverlayStore::registerValidPaths(const ValidPathInfos & infos) } -void LocalOverlayStore::addToStore(const ValidPathInfo & info, Source & source, - RepairFlag repair, CheckSigsFlag checkSigs) -{ - LocalStore::addToStore(info, source, repair, checkSigs); - if (lowerStore->isValidPath(info.path)) { - // dedup stores - deletePath(toUpperPath(info.path)); // TODO: Investigate whether this can trigger 'stale file handle' errors. - } -} - - -StorePath LocalOverlayStore::addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) -{ - auto path = LocalStore::addToStoreFromDump(dump, name, method, hashAlgo, repair, references); - if (lowerStore->isValidPath(path)) { - // dedup stores - deletePath(toUpperPath(path)); - } - return path; -} - - -StorePath LocalOverlayStore::addTextToStore( - std::string_view name, - std::string_view s, - const StorePathSet & references, - RepairFlag repair) -{ - auto path = LocalStore::addTextToStore(name, s, references, repair); - if (lowerStore->isValidPath(path)) { - // dedup stores - deletePath(toUpperPath(path)); - } - return path; -} - - void LocalOverlayStore::deleteGCPath(const Path & path, uint64_t & bytesFreed) { auto mergedDir = realStoreDir.get() + "/"; diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 701024bfb..e6b877b04 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -108,18 +108,6 @@ private: void registerValidPaths(const ValidPathInfos & infos) override; - void addToStore(const ValidPathInfo & info, Source & source, - RepairFlag repair, CheckSigsFlag checkSigs) override; - - StorePath addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) override; - - StorePath addTextToStore( - std::string_view name, - std::string_view s, - const StorePathSet & references, - RepairFlag repair) override; - void queryRealisationUncached(const DrvOutput&, Callback> callback) noexcept override; From 7ed0ab2dabececd84578ddf70ecad0e528d67a28 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Thu, 15 Jun 2023 12:48:06 +0100 Subject: [PATCH 049/133] Check _NIX_TEST_NO_SANDBOX when setting _canUseSandbox. --- tests/common/vars-and-functions.sh.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/common/vars-and-functions.sh.in b/tests/common/vars-and-functions.sh.in index dc7ce13cc..ad0623871 100644 --- a/tests/common/vars-and-functions.sh.in +++ b/tests/common/vars-and-functions.sh.in @@ -141,7 +141,7 @@ restartDaemon() { startDaemon } -if [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then +if [[ -z "${_NIX_TEST_NO_SANDBOX:-}" ]] && [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then _canUseSandbox=1 fi From fad0dd4afb66577f9c17b5a315d120ac0f5acd94 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Thu, 15 Jun 2023 12:58:59 +0100 Subject: [PATCH 050/133] Skip build-remote-trustless unless sandbox is supported. --- tests/build-remote-trustless-should-fail-0.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/build-remote-trustless-should-fail-0.sh b/tests/build-remote-trustless-should-fail-0.sh index fad1def59..b14101f83 100644 --- a/tests/build-remote-trustless-should-fail-0.sh +++ b/tests/build-remote-trustless-should-fail-0.sh @@ -2,6 +2,7 @@ source common.sh enableFeatures "daemon-trust-override" +requireSandboxSupport restartDaemon [[ $busybox =~ busybox ]] || skipTest "no busybox" From 264b644c534418f3a27b7e6013afa4aee7bdf89c Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Thu, 15 Jun 2023 13:22:17 +0100 Subject: [PATCH 051/133] More detail on why read-only mode disables locking. --- src/libutil/experimental-features.cc | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index b758a0262..1fbc99b34 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -223,11 +223,14 @@ constexpr std::array xpFeatureDetails = {{ Set this parameter to `true` to allow stores with databases on read-only filesystems to be opened for querying; ordinarily Nix will refuse to do this. - Enabling this setting disables the locking required for safe concurrent - access, so you should be certain that the database will not be changed. - While the filesystem the database resides on might be read-only to this - process, consider whether another user, process, or system, might have - write access to it. + This is because SQLite requires write access to the database file to perform + the file locking operations necessary for safe concurrent access. When `read-only` + is set to `true`, the database will be opened in immutable mode. + + Under this mode, SQLite will not do any locking at all, so you should be certain + that the database will not be changed. While the filesystem the database resides + on might be read-only to this process, consider whether another user, process, + or system, might have write access to it. )", }, }}; From 7cdaa0b8a626004b19b87b1eec74d1adb5f20263 Mon Sep 17 00:00:00 2001 From: Ben Radford <104896700+benradf@users.noreply.github.com> Date: Thu, 15 Jun 2023 13:25:15 +0100 Subject: [PATCH 052/133] Update tests/read-only-store.sh Co-authored-by: Valentin Gagarin --- tests/read-only-store.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/read-only-store.sh b/tests/read-only-store.sh index 2f89b849f..d63920c19 100644 --- a/tests/read-only-store.sh +++ b/tests/read-only-store.sh @@ -2,6 +2,8 @@ source common.sh enableFeatures "read-only-local-store" +needLocalStore "cannot open store read-only when daemon has already opened it writeable" + clearStore happy () { From 78e2f931d048f9c41e5432cd8e49bf6bc0c3baae Mon Sep 17 00:00:00 2001 From: Ben Radford <104896700+benradf@users.noreply.github.com> Date: Thu, 15 Jun 2023 13:32:16 +0100 Subject: [PATCH 053/133] Update src/libstore/local-store.cc Co-authored-by: Valentin Gagarin --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 17385cdfb..364be5dd5 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -363,7 +363,7 @@ LocalStore::LocalStore(const Params & params) if (!readOnly) { migrateCASchema(state->db, dbDir + "/ca-schema", globalLock); } else { - throw Error("need to migrate to CA schema, but this cannot be done in read-only mode"); + throw Error("need to migrate to content-addressed schema, but this cannot be done in read-only mode"); } } From 984b01924a58dc80b0e5da7722ccfa108cbca036 Mon Sep 17 00:00:00 2001 From: Ben Radford <104896700+benradf@users.noreply.github.com> Date: Thu, 15 Jun 2023 13:32:35 +0100 Subject: [PATCH 054/133] Update src/libstore/local-store.cc Co-authored-by: Valentin Gagarin --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 364be5dd5..e69460e6c 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -496,7 +496,7 @@ int LocalStore::getSchema() void LocalStore::openDB(State & state, bool create) { if (create && readOnly) { - throw Error("unable to create database while in read-only mode"); + throw Error("cannot create database while in read-only mode"); } if (access(dbDir.c_str(), R_OK | (readOnly ? 0 : W_OK))) From a7b1b92d817348669fc2cd34c085dae57ceed5b8 Mon Sep 17 00:00:00 2001 From: Ben Radford <104896700+benradf@users.noreply.github.com> Date: Thu, 15 Jun 2023 13:32:56 +0100 Subject: [PATCH 055/133] Update src/libstore/local-store.hh Co-authored-by: Valentin Gagarin --- src/libstore/local-store.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 31a6f4fd9..35c4febef 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -50,7 +50,7 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig false, "read-only", R"( - Allow this store to be opened when its database is on a read-only filesystem. + Allow this store to be opened when its [database](@docroot@/glossary.md#gloss-nix-database) is on a read-only filesystem. Normally Nix will attempt to open the store database in read-write mode, even for querying (when write access is not needed). This causes it to fail if the From 4642b60afe2a79b00ae51ca1ec1fc35eb5a224c5 Mon Sep 17 00:00:00 2001 From: Ben Radford <104896700+benradf@users.noreply.github.com> Date: Thu, 15 Jun 2023 13:33:26 +0100 Subject: [PATCH 056/133] Update src/libstore/local-store.hh Co-authored-by: Valentin Gagarin --- src/libstore/local-store.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 35c4febef..a1ab3bb4d 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -57,7 +57,7 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig database is on a read-only filesystem. Enable read-only mode to disable locking and open the SQLite database with the - **immutable** parameter set. Do not use this unless the filesystem is read-only. + [`immutable` parameter](https://www.sqlite.org/c3ref/open.html) set. Do not use this unless the filesystem is read-only. Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process. )"}; From f2fe9822c14bc86e5da8f8cb97cb16a961ecc46f Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Thu, 15 Jun 2023 13:31:37 +0100 Subject: [PATCH 057/133] Comment explaining what schema version 0 means. --- src/libstore/local-store.hh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index a1ab3bb4d..dd563348b 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -285,6 +285,10 @@ public: private: + /** + * Retrieve the current version of the database schema. + * If the database does not exist yet, the version returned will be 0. + */ int getSchema(); void openDB(State & state, bool create); From f5d83a80293426b2f869af206b93b5cdfeb34ec9 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Thu, 15 Jun 2023 13:36:28 +0100 Subject: [PATCH 058/133] One line per sentence in markdown docs. --- src/libstore/local-store.hh | 14 +++++++------- src/libutil/experimental-features.cc | 14 +++++--------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index dd563348b..0e7b1334f 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -52,14 +52,14 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig R"( Allow this store to be opened when its [database](@docroot@/glossary.md#gloss-nix-database) is on a read-only filesystem. - Normally Nix will attempt to open the store database in read-write mode, even - for querying (when write access is not needed). This causes it to fail if the - database is on a read-only filesystem. + Normally Nix will attempt to open the store database in read-write mode, even for querying (when write access is not needed). + This causes it to fail if the database is on a read-only filesystem. - Enable read-only mode to disable locking and open the SQLite database with the - [`immutable` parameter](https://www.sqlite.org/c3ref/open.html) set. Do not use this unless the filesystem is read-only. - Using it when the filesystem is writable can cause incorrect query results or - corruption errors if the database is changed by another process. + Enable read-only mode to disable locking and open the SQLite database with the [`immutable` parameter](https://www.sqlite.org/c3ref/open.html) set. + + **Warning** + Do not use this unless the filesystem is read-only. + Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process. )"}; const std::string name() override { return "Local Store"; } diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 1fbc99b34..9705c5b38 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -220,17 +220,13 @@ constexpr std::array xpFeatureDetails = {{ .description = R"( Allow the use of the `read-only` parameter in local store URIs. - Set this parameter to `true` to allow stores with databases on read-only - filesystems to be opened for querying; ordinarily Nix will refuse to do this. + Set this parameter to `true` to allow stores with databases on read-only filesystems to be opened for querying; ordinarily Nix will refuse to do this. - This is because SQLite requires write access to the database file to perform - the file locking operations necessary for safe concurrent access. When `read-only` - is set to `true`, the database will be opened in immutable mode. + This is because SQLite requires write access to the database file to perform the file locking operations necessary for safe concurrent access. + When `read-only` is set to `true`, the database will be opened in immutable mode. - Under this mode, SQLite will not do any locking at all, so you should be certain - that the database will not be changed. While the filesystem the database resides - on might be read-only to this process, consider whether another user, process, - or system, might have write access to it. + Under this mode, SQLite will not do any locking at all, so you should be certain that the database will not be changed. + While the filesystem the database resides on might be read-only to this process, consider whether another user, process, or system, might have write access to it. )", }, }}; From feb8d552aeca36adde5a9c25b92fd9eb205a761e Mon Sep 17 00:00:00 2001 From: Ben Radford <104896700+benradf@users.noreply.github.com> Date: Mon, 19 Jun 2023 13:22:41 +0100 Subject: [PATCH 059/133] Update src/libstore/local-store.hh Co-authored-by: Valentin Gagarin --- src/libstore/local-store.hh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 0e7b1334f..bd4794eb8 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -52,8 +52,7 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig R"( Allow this store to be opened when its [database](@docroot@/glossary.md#gloss-nix-database) is on a read-only filesystem. - Normally Nix will attempt to open the store database in read-write mode, even for querying (when write access is not needed). - This causes it to fail if the database is on a read-only filesystem. + Normally Nix will attempt to open the store database in read-write mode, even for querying (when write access is not needed), causing it to fail if the database is on a read-only filesystem. Enable read-only mode to disable locking and open the SQLite database with the [`immutable` parameter](https://www.sqlite.org/c3ref/open.html) set. From ef40448b1cb33165af5522f72d5d352d66035671 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Mon, 19 Jun 2023 13:47:21 +0100 Subject: [PATCH 060/133] Remove redundant description on experimental flag. --- src/libstore/local-store.hh | 1 + src/libutil/experimental-features.cc | 8 -------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index bd4794eb8..1af64c77b 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -59,6 +59,7 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig **Warning** Do not use this unless the filesystem is read-only. Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process. + While the filesystem the database resides on might appear to be read-only, consider whether another user or system might have write access to it. )"}; const std::string name() override { return "Local Store"; } diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 7c2b29a07..3a04083ed 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -226,14 +226,6 @@ constexpr std::array xpFeatureDetails = {{ .name = "read-only-local-store", .description = R"( Allow the use of the `read-only` parameter in local store URIs. - - Set this parameter to `true` to allow stores with databases on read-only filesystems to be opened for querying; ordinarily Nix will refuse to do this. - - This is because SQLite requires write access to the database file to perform the file locking operations necessary for safe concurrent access. - When `read-only` is set to `true`, the database will be opened in immutable mode. - - Under this mode, SQLite will not do any locking at all, so you should be certain that the database will not be changed. - While the filesystem the database resides on might be read-only to this process, consider whether another user, process, or system, might have write access to it. )", }, }}; From b09baa3bc31226cac5ccdbd7ba9d72a2ed389207 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Mon, 19 Jun 2023 13:57:10 +0100 Subject: [PATCH 061/133] Link to LocalStore section of nix3-help-stores section. --- src/libutil/experimental-features.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 3a04083ed..7c4112d32 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -225,7 +225,7 @@ constexpr std::array xpFeatureDetails = {{ .tag = Xp::ReadOnlyLocalStore, .name = "read-only-local-store", .description = R"( - Allow the use of the `read-only` parameter in local store URIs. + Allow the use of the `read-only` parameter in [local store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-store) URIs. )", }, }}; From ba492a98baa0fa37b0914e3f7d00f2859cef5bcd Mon Sep 17 00:00:00 2001 From: Ben Radford <104896700+benradf@users.noreply.github.com> Date: Mon, 19 Jun 2023 14:07:31 +0100 Subject: [PATCH 062/133] Update src/libstore/local-store.hh Co-authored-by: Valentin Gagarin --- src/libstore/local-store.hh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 1af64c77b..8a3b0b43f 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -56,10 +56,11 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig Enable read-only mode to disable locking and open the SQLite database with the [`immutable` parameter](https://www.sqlite.org/c3ref/open.html) set. - **Warning** - Do not use this unless the filesystem is read-only. - Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process. - While the filesystem the database resides on might appear to be read-only, consider whether another user or system might have write access to it. + > **Warning** + > Do not use this unless the filesystem is read-only. + > + > Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process. + > While the filesystem the database resides on might appear to be read-only, consider whether another user or system might have write access to it. )"}; const std::string name() override { return "Local Store"; } From 4e72b8483ede9242ddde99eb8011b18098172444 Mon Sep 17 00:00:00 2001 From: Ben Radford <104896700+benradf@users.noreply.github.com> Date: Mon, 19 Jun 2023 16:01:43 +0100 Subject: [PATCH 063/133] Update src/libstore/sqlite.hh Co-authored-by: John Ericson --- src/libstore/sqlite.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index b0e84f8ed..0c08267f7 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -29,7 +29,7 @@ enum class SQLiteOpenMode { * Use this mode if the database is on a read-only filesystem. * Fails with an error if the database does not exist. */ - Immutable + Immutable, }; /** From 2add2309397bd576712c1b90d364fb90525f4797 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 9 Jul 2023 21:53:06 -0400 Subject: [PATCH 064/133] Fix build --- src/libstore/local-overlay-store.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index e6b877b04..6f995ba39 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -25,7 +25,7 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig Must be used as OverlayFS lower layer for this store's store dir. )"}; - const PathSetting upperLayer{(StoreConfig*) this, false, "", "upper-layer", + const PathSetting upperLayer{(StoreConfig*) this, "", "upper-layer", R"( Must be used as OverlayFS upper layer for this store's store dir. )"}; From 735a672e1f3bbad8e32b211ce33fa4a308ae355a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 9 Jul 2023 22:24:51 -0400 Subject: [PATCH 065/133] Introduce notion of a test group, use for CA tests Grouping our tests should make it easier to understand the intent than one long poorly-arranged list. It also is convenient for running just the tests for a specific component when working on that component. We need at least one test group so this isn't dead code; I decided to collect the tests for the `ca-derivations` and `dynamic-derivations` experimental features in groups. Do ```bash make ca.test-group -jN ``` and ```bash make dyn-drv.test-group -jN ``` to try running just them. I originally did this as part of #8397 for being able to just the local overlay store alone. I am PRing it separately now so we can separate general infra from new features. --- Makefile | 2 ++ doc/manual/src/contributing/testing.md | 28 +++++++++++++++++++++++++ mk/lib.mk | 12 ++++++++++- mk/tests.mk | 8 +++++-- tests/ca/local.mk | 27 ++++++++++++++++++++++++ tests/dyn-drv/local.mk | 11 ++++++++++ tests/local.mk | 29 +++----------------------- 7 files changed, 88 insertions(+), 29 deletions(-) create mode 100644 tests/ca/local.mk create mode 100644 tests/dyn-drv/local.mk diff --git a/Makefile b/Makefile index c6220482a..31b54b93d 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,8 @@ makefiles += \ src/libstore/tests/local.mk \ src/libexpr/tests/local.mk \ tests/local.mk \ + tests/ca/local.mk \ + tests/dyn-drv/local.mk \ tests/test-libstoreconsumer/local.mk \ tests/plugins/local.mk else diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md index e5f80a928..da46a638d 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/contributing/testing.md @@ -14,6 +14,8 @@ You can run the whole testsuite with `make check`, or the tests for a specific c The functional tests reside under the `tests` directory and are listed in `tests/local.mk`. Each test is a bash script. +### Running the whole test suite + The whole test suite can be run with: ```shell-session @@ -23,6 +25,32 @@ ran test tests/bar.sh... [PASS] ... ``` +### Grouping tests + +Sometimes it is useful to group related tests so they can be easily run together without running the entire test suite. +For example, `tests/ca/local.mk` defines a "ca" test group for tests relating to content-addressed derivation outputs. + +That test group can be run like this: + +```shell-session +$ make ca.test-group -j50 +ran test tests/ca/nix-run.sh... [PASS] +ran test tests/ca/import-derivation.sh... [PASS] +... +``` + +The testgroup is defined in Make like this: +```makefile +$(test-group-name)-tests := \ + $(d)/test0.sh \ + $(d)/test1.sh \ + ... + +install-tests-groups += $(test-group-name) +``` + +### Running individual tests + Individual tests can be run with `make`: ```shell-session diff --git a/mk/lib.mk b/mk/lib.mk index 34fa624d8..cddaf9eb1 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -10,6 +10,7 @@ bin-scripts := noinst-scripts := man-pages := install-tests := +install-tests-groups := ifdef HOST_OS HOST_KERNEL = $(firstword $(subst -, ,$(HOST_OS))) @@ -121,7 +122,16 @@ $(foreach script, $(bin-scripts), $(eval $(call install-program-in,$(script),$(b $(foreach script, $(bin-scripts), $(eval programs-list += $(script))) $(foreach script, $(noinst-scripts), $(eval programs-list += $(script))) $(foreach template, $(template-files), $(eval $(call instantiate-template,$(template)))) -$(foreach test, $(install-tests), $(eval $(call run-install-test,$(test)))) +$(foreach test, $(install-tests), \ + $(eval $(call run-install-test,$(test))) \ + $(eval installcheck: $(test).test)) +$(foreach test-group, $(install-tests-groups), \ + $(eval $(call run-install-test-group,$(test-group))) \ + $(eval installcheck: $(test-group).test-group) \ + $(foreach test, $($(test-group)-tests), \ + $(eval $(call run-install-test,$(test))) \ + $(eval $(test-group).test-group: $(test).test))) + $(foreach file, $(man-pages), $(eval $(call install-data-in, $(file), $(mandir)/man$(patsubst .%,%,$(suffix $(file)))))) diff --git a/mk/tests.mk b/mk/tests.mk index 3ebbd86e3..ec8128bdf 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -4,8 +4,6 @@ test-deps = define run-install-test - installcheck: $1.test - .PHONY: $1.test $1.test: $1 $(test-deps) @env BASH=$(bash) $(bash) mk/run-test.sh $1 < /dev/null @@ -16,6 +14,12 @@ define run-install-test endef +define run-install-test-group + + .PHONY: $1.test-group + +endef + .PHONY: check installcheck print-top-help += \ diff --git a/tests/ca/local.mk b/tests/ca/local.mk new file mode 100644 index 000000000..d15312708 --- /dev/null +++ b/tests/ca/local.mk @@ -0,0 +1,27 @@ +ca-tests := \ + $(d)/build-with-garbage-path.sh \ + $(d)/build.sh \ + $(d)/concurrent-builds.sh \ + $(d)/derivation-json.sh \ + $(d)/duplicate-realisation-in-closure.sh \ + $(d)/gc.sh \ + $(d)/import-derivation.sh \ + $(d)/new-build-cmd.sh \ + $(d)/nix-copy.sh \ + $(d)/nix-run.sh \ + $(d)/nix-shell.sh \ + $(d)/post-hook.sh \ + $(d)/recursive.sh \ + $(d)/repl.sh \ + $(d)/selfref-gc.sh \ + $(d)/signatures.sh \ + $(d)/substitute.sh \ + $(d)/why-depends.sh + +install-tests-groups += ca + +clean-files += \ + $(d)/config.nix + +test-deps += \ + tests/ca/config.nix diff --git a/tests/dyn-drv/local.mk b/tests/dyn-drv/local.mk new file mode 100644 index 000000000..f065a5627 --- /dev/null +++ b/tests/dyn-drv/local.mk @@ -0,0 +1,11 @@ +dyn-drv-tests := \ + $(d)/text-hashed-output.sh \ + $(d)/recursive-mod-json.sh + +install-tests-groups += dyn-drv + +clean-files += \ + $(d)/config.nix + +test-deps += \ + tests/dyn-drv/config.nix diff --git a/tests/local.mk b/tests/local.mk index 88848926b..8cf79fcb3 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -14,7 +14,6 @@ nix_tests = \ flakes/absolute-paths.sh \ flakes/build-paths.sh \ flakes/flake-in-submodule.sh \ - ca/gc.sh \ gc.sh \ nix-collect-garbage-d.sh \ remote-store.sh \ @@ -27,8 +26,6 @@ nix_tests = \ user-envs-migration.sh \ binary-cache.sh \ multiple-outputs.sh \ - ca/build.sh \ - ca/new-build-cmd.sh \ nix-build.sh \ gc-concurrent.sh \ repair.sh \ @@ -46,24 +43,17 @@ nix_tests = \ referrers.sh \ optimise-store.sh \ substitute-with-invalid-ca.sh \ - ca/concurrent-builds.sh \ signing.sh \ - ca/build-with-garbage-path.sh \ hash.sh \ gc-non-blocking.sh \ check.sh \ - ca/substitute.sh \ nix-shell.sh \ - ca/signatures.sh \ - ca/nix-shell.sh \ - ca/nix-copy.sh \ check-refs.sh \ build-remote-input-addressed.sh \ secure-drv-outputs.sh \ restricted.sh \ fetchGitSubmodules.sh \ flakes/search-root.sh \ - ca/duplicate-realisation-in-closure.sh \ readfile-context.sh \ nix-channel.sh \ recursive.sh \ @@ -79,10 +69,7 @@ nix_tests = \ nar-access.sh \ pure-eval.sh \ eval.sh \ - ca/post-hook.sh \ repl.sh \ - ca/repl.sh \ - ca/recursive.sh \ binary-cache-build-remote.sh \ search.sh \ logging.sh \ @@ -107,13 +94,8 @@ nix_tests = \ fmt.sh \ eval-store.sh \ why-depends.sh \ - ca/why-depends.sh \ derivation-json.sh \ - ca/derivation-json.sh \ import-derivation.sh \ - ca/import-derivation.sh \ - dyn-drv/text-hashed-output.sh \ - dyn-drv/recursive-mod-json.sh \ nix_path.sh \ case-hack.sh \ placeholders.sh \ @@ -122,8 +104,7 @@ nix_tests = \ build.sh \ build-delete.sh \ output-normalization.sh \ - ca/nix-run.sh \ - selfref-gc.sh ca/selfref-gc.sh \ + selfref-gc.sh \ db-migration.sh \ bash-profile.sh \ pass-as-file.sh \ @@ -147,16 +128,12 @@ install-tests += $(foreach x, $(nix_tests), $(d)/$(x)) clean-files += \ $(d)/common/vars-and-functions.sh \ - $(d)/config.nix \ - $(d)/ca/config.nix \ - $(d)/dyn-drv/config.nix + $(d)/config.nix test-deps += \ tests/common/vars-and-functions.sh \ tests/config.nix \ - tests/ca/config.nix \ - tests/test-libstoreconsumer/test-libstoreconsumer \ - tests/dyn-drv/config.nix + tests/test-libstoreconsumer/test-libstoreconsumer ifeq ($(BUILD_SHARED_LIBS), 1) test-deps += tests/plugins/libplugintest.$(SO_EXT) From 83cfa82e52e26a287a5bdb60863751096ab02c63 Mon Sep 17 00:00:00 2001 From: cidkidnix Date: Thu, 13 Jul 2023 14:39:46 -0500 Subject: [PATCH 066/133] Add unset to NIX_STORE_DIR for local-overlay tests --- tests/overlay-local-store/build-inner.sh | 4 ++++ tests/overlay-local-store/check-post-init-inner.sh | 4 ++++ tests/overlay-local-store/redundant-add-inner.sh | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/tests/overlay-local-store/build-inner.sh b/tests/overlay-local-store/build-inner.sh index 969de282d..beb40b2fc 100755 --- a/tests/overlay-local-store/build-inner.sh +++ b/tests/overlay-local-store/build-inner.sh @@ -6,6 +6,10 @@ set -x source common.sh +# Avoid store dir being inside sandbox build-dir +unset NIX_STORE_DIR +unset NIX_STATE_DIR + storeDirs initLowerStore diff --git a/tests/overlay-local-store/check-post-init-inner.sh b/tests/overlay-local-store/check-post-init-inner.sh index 421b64934..37ed8c113 100755 --- a/tests/overlay-local-store/check-post-init-inner.sh +++ b/tests/overlay-local-store/check-post-init-inner.sh @@ -6,6 +6,10 @@ set -x source common.sh +# Avoid store dir being inside sandbox build-dir +unset NIX_STORE_DIR +unset NIX_STATE_DIR + storeDirs initLowerStore diff --git a/tests/overlay-local-store/redundant-add-inner.sh b/tests/overlay-local-store/redundant-add-inner.sh index 921069c25..97969b40e 100755 --- a/tests/overlay-local-store/redundant-add-inner.sh +++ b/tests/overlay-local-store/redundant-add-inner.sh @@ -6,6 +6,10 @@ set -x source common.sh +# Avoid store dir being inside sandbox build-dir +unset NIX_STORE_DIR +unset NIX_STATE_DIR + storeDirs initLowerStore From 37598a13e877980d4654c632d10921c41c9ad976 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 18 Jul 2023 09:26:48 +0100 Subject: [PATCH 067/133] Revert "Check _NIX_TEST_NO_SANDBOX when setting _canUseSandbox." This reverts commit 7ed0ab2dabececd84578ddf70ecad0e528d67a28. --- tests/common/vars-and-functions.sh.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/common/vars-and-functions.sh.in b/tests/common/vars-and-functions.sh.in index ad0623871..dc7ce13cc 100644 --- a/tests/common/vars-and-functions.sh.in +++ b/tests/common/vars-and-functions.sh.in @@ -141,7 +141,7 @@ restartDaemon() { startDaemon } -if [[ -z "${_NIX_TEST_NO_SANDBOX:-}" ]] && [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then +if [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then _canUseSandbox=1 fi From f66b65a30ad1f14dd757874c12255eef42368b04 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 18 Jul 2023 09:29:27 +0100 Subject: [PATCH 068/133] Revert "Skip build-remote-trustless unless sandbox is supported." This reverts commit fad0dd4afb66577f9c17b5a315d120ac0f5acd94. --- tests/build-remote-trustless-should-fail-0.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/build-remote-trustless-should-fail-0.sh b/tests/build-remote-trustless-should-fail-0.sh index b14101f83..fad1def59 100644 --- a/tests/build-remote-trustless-should-fail-0.sh +++ b/tests/build-remote-trustless-should-fail-0.sh @@ -2,7 +2,6 @@ source common.sh enableFeatures "daemon-trust-override" -requireSandboxSupport restartDaemon [[ $busybox =~ busybox ]] || skipTest "no busybox" From a33ee5c84305b85f5bcd0ea1be43b44cf601693f Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 18 Jul 2023 10:49:44 +0100 Subject: [PATCH 069/133] Paths added to lower store are accessible via overlay. --- tests/overlay-local-store/add-lower-inner.sh | 31 +++++++++++++++++++ tests/overlay-local-store/add-lower.sh | 5 +++ tests/overlay-local-store/common.sh | 11 ++++++- tests/overlay-local-store/local.mk | 3 +- .../redundant-add-inner.sh | 5 +-- 5 files changed, 51 insertions(+), 4 deletions(-) create mode 100755 tests/overlay-local-store/add-lower-inner.sh create mode 100755 tests/overlay-local-store/add-lower.sh diff --git a/tests/overlay-local-store/add-lower-inner.sh b/tests/overlay-local-store/add-lower-inner.sh new file mode 100755 index 000000000..40a1f397c --- /dev/null +++ b/tests/overlay-local-store/add-lower-inner.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +set -x + +source common.sh + +# Avoid store dir being inside sandbox build-dir +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +storeDirs + +initLowerStore + +mountOverlayfs + +# Add something to the overlay store +overlayPath=$(addTextToStore "$storeB" "overlay-file" "Add to overlay store") +stat "$TEST_ROOT/merged-store/$overlayPath" + +# Now add something to the lower store +lowerPath=$(addTextToStore "$storeA" "lower-file" "Add to lower store") +stat "$TEST_ROOT/store-a/$lowerPath" + +# Remount overlayfs to ensure synchronization +mount -o remount "$TEST_ROOT/merged-store/nix/store" + +# Path should be accessible via overlay store +stat "$TEST_ROOT/merged-store/$lowerPath" diff --git a/tests/overlay-local-store/add-lower.sh b/tests/overlay-local-store/add-lower.sh new file mode 100755 index 000000000..f0ac46a91 --- /dev/null +++ b/tests/overlay-local-store/add-lower.sh @@ -0,0 +1,5 @@ +source common.sh + +requireEnvironment +setupConfig +execUnshare ./add-lower-inner.sh diff --git a/tests/overlay-local-store/common.sh b/tests/overlay-local-store/common.sh index 149102000..7850c068b 100644 --- a/tests/overlay-local-store/common.sh +++ b/tests/overlay-local-store/common.sh @@ -8,7 +8,7 @@ requireEnvironment () { } setupConfig () { - echo "drop-supplementary-groups = false" >> "$NIX_CONF_DIR"/nix.conf + echo "require-drop-supplementary-groups = false" >> "$NIX_CONF_DIR"/nix.conf echo "build-users-group = " >> "$NIX_CONF_DIR"/nix.conf } @@ -54,3 +54,12 @@ initLowerStore () { execUnshare () { exec unshare --mount --map-root-user "$@" } + +addTextToStore() { + storeDir=$1; shift + filename=$1; shift + content=$1; shift + filePath="$TEST_HOME/$filename" + echo "$content" > "$filePath" + nix-store --store "$storeDir" --add "$filePath" +} diff --git a/tests/overlay-local-store/local.mk b/tests/overlay-local-store/local.mk index b94238a67..5d14e2561 100644 --- a/tests/overlay-local-store/local.mk +++ b/tests/overlay-local-store/local.mk @@ -2,6 +2,7 @@ overlay-local-store-tests := \ $(d)/check-post-init.sh \ $(d)/redundant-add.sh \ $(d)/build.sh \ - $(d)/bad-uris.sh + $(d)/bad-uris.sh \ + $(d)/add-lower.sh install-tests-groups += overlay-local-store diff --git a/tests/overlay-local-store/redundant-add-inner.sh b/tests/overlay-local-store/redundant-add-inner.sh index 97969b40e..cfdae68b4 100755 --- a/tests/overlay-local-store/redundant-add-inner.sh +++ b/tests/overlay-local-store/redundant-add-inner.sh @@ -7,7 +7,7 @@ set -x source common.sh # Avoid store dir being inside sandbox build-dir -unset NIX_STORE_DIR +unset NIX_STORE_DIR # TODO: This causes toRealPath to fail (it expects this var to be set) unset NIX_STATE_DIR storeDirs @@ -27,4 +27,5 @@ path=$(nix-store --store "$storeB" --add ../dummy) stat $(toRealPath "$storeA/nix/store" "$path") # upper layer should still not have it (no redundant copy) -expect 1 stat $(toRealPath "$storeB/nix/store" "$path") +expect 1 stat $(toRealPath "$storeB/nix/store" "$path") # TODO: Check this is failing for the right reason. + # $storeB is a store URI not a directory path From 0ccf6382af4c34dafacd2a417ba658e62d81adbf Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 18 Jul 2023 12:30:33 +0100 Subject: [PATCH 070/133] Add test for verifying overlay store. --- tests/overlay-local-store/local.mk | 3 ++- tests/overlay-local-store/verify-inner.sh | 32 +++++++++++++++++++++++ tests/overlay-local-store/verify.sh | 5 ++++ 3 files changed, 39 insertions(+), 1 deletion(-) create mode 100755 tests/overlay-local-store/verify-inner.sh create mode 100755 tests/overlay-local-store/verify.sh diff --git a/tests/overlay-local-store/local.mk b/tests/overlay-local-store/local.mk index 5d14e2561..5e8c76b4e 100644 --- a/tests/overlay-local-store/local.mk +++ b/tests/overlay-local-store/local.mk @@ -3,6 +3,7 @@ overlay-local-store-tests := \ $(d)/redundant-add.sh \ $(d)/build.sh \ $(d)/bad-uris.sh \ - $(d)/add-lower.sh + $(d)/add-lower.sh \ + $(d)/verify.sh install-tests-groups += overlay-local-store diff --git a/tests/overlay-local-store/verify-inner.sh b/tests/overlay-local-store/verify-inner.sh new file mode 100755 index 000000000..b68800e5c --- /dev/null +++ b/tests/overlay-local-store/verify-inner.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +set -x + +source common.sh + +# Avoid store dir being inside sandbox build-dir +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +storeDirs + +initLowerStore + +mountOverlayfs + +#path=$(nix-store --store "$storeB" --add ../dummy) + +path=$(nix-build --store $storeB ../hermetic.nix --arg busybox "$busybox" --arg seed 1) + +inputDrvPath=$(find "$storeA" -name "*-hermetic-input-1.drv") +rm -v "$inputDrvPath" + +#tree "$TEST_ROOT" + +#rm -v "$TEST_ROOT/store-a/$path" + +nix-store --store "$storeB" --verify + +echo "SUCCESS" diff --git a/tests/overlay-local-store/verify.sh b/tests/overlay-local-store/verify.sh new file mode 100755 index 000000000..8b44603ff --- /dev/null +++ b/tests/overlay-local-store/verify.sh @@ -0,0 +1,5 @@ +source common.sh + +requireEnvironment +setupConfig +execUnshare ./verify-inner.sh From 58085e4eff1e06524f9ad8f6e9f271c19dcbba9e Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 18 Jul 2023 13:10:34 +0100 Subject: [PATCH 071/133] Have verify test exercise check-contents too. --- tests/overlay-local-store/verify-inner.sh | 25 +++++++++++++++-------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/tests/overlay-local-store/verify-inner.sh b/tests/overlay-local-store/verify-inner.sh index b68800e5c..5f8cbcf0e 100755 --- a/tests/overlay-local-store/verify-inner.sh +++ b/tests/overlay-local-store/verify-inner.sh @@ -16,17 +16,24 @@ initLowerStore mountOverlayfs -#path=$(nix-store --store "$storeB" --add ../dummy) +# Realise a derivation from the lower store to propagate paths to overlay DB +nix-store --store "$storeB" --realise $drvPath -path=$(nix-build --store $storeB ../hermetic.nix --arg busybox "$busybox" --arg seed 1) +# Also ensure dummy file exists in overlay DB +dummyPath=$(nix-store --store "$storeB" --add ../dummy) -inputDrvPath=$(find "$storeA" -name "*-hermetic-input-1.drv") -rm -v "$inputDrvPath" +# Verify should be successful at this point +nix-store --store "$storeB" --verify --check-contents -#tree "$TEST_ROOT" +# Now delete one of the derivation inputs in the lower store +inputDrvFullPath=$(find "$storeA" -name "*-hermetic-input-1.drv") +inputDrvPath=${inputDrvFullPath/*\/nix\/store\///nix/store/} +rm -v "$inputDrvFullPath" -#rm -v "$TEST_ROOT/store-a/$path" +# And truncate the contents of dummy file in lower store +find "$storeA" -name "*-dummy" -exec truncate -s 0 {} \; -nix-store --store "$storeB" --verify - -echo "SUCCESS" +# Verify should fail with the messages about missing input and modified dummy file +verifyOutput=$(expectStderr 1 nix-store --store "$storeB" --verify --check-contents) +<<<"$verifyOutput" grepQuiet "path '$inputDrvPath' disappeared, but it still has valid referrers!" +<<<"$verifyOutput" grepQuiet "path '$dummyPath' was modified! expected hash" From d5cd74a4012d583d11d491bec9f1fa9a07b11973 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 18 Jul 2023 13:49:13 +0100 Subject: [PATCH 072/133] Override verifyStore to always pass NoRepair for LocalOverlayStore. --- src/libstore/local-overlay-store.cc | 7 +++++++ src/libstore/local-overlay-store.hh | 2 ++ tests/overlay-local-store/verify-inner.sh | 3 ++- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index ddccb8c5a..182955fbe 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -188,6 +188,13 @@ void LocalOverlayStore::deleteGCPath(const Path & path, uint64_t & bytesFreed) } } +bool LocalOverlayStore::verifyStore(bool checkContents, RepairFlag repair) +{ + if (repair) + warn("local-overlay: store does not support --verify --repair"); + return LocalStore::verifyStore(checkContents, NoRepair); +} + static RegisterStoreImplementation regLocalOverlayStore; } diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 6f995ba39..8fa99ee14 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -112,6 +112,8 @@ private: Callback> callback) noexcept override; void deleteGCPath(const Path & path, uint64_t & bytesFreed) override; + + bool verifyStore(bool checkContents, RepairFlag repair) override; }; } diff --git a/tests/overlay-local-store/verify-inner.sh b/tests/overlay-local-store/verify-inner.sh index 5f8cbcf0e..8f8839302 100755 --- a/tests/overlay-local-store/verify-inner.sh +++ b/tests/overlay-local-store/verify-inner.sh @@ -34,6 +34,7 @@ rm -v "$inputDrvFullPath" find "$storeA" -name "*-dummy" -exec truncate -s 0 {} \; # Verify should fail with the messages about missing input and modified dummy file -verifyOutput=$(expectStderr 1 nix-store --store "$storeB" --verify --check-contents) +verifyOutput=$(expectStderr 1 nix-store --store "$storeB" --verify --check-contents --repair) <<<"$verifyOutput" grepQuiet "path '$inputDrvPath' disappeared, but it still has valid referrers!" <<<"$verifyOutput" grepQuiet "path '$dummyPath' was modified! expected hash" +<<<"$verifyOutput" grepQuiet "store does not support --verify --repair" From 614efc1240dbfefbdabced808decd8defd33df8e Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 18 Jul 2023 13:59:22 +0100 Subject: [PATCH 073/133] Add test for store optimise path deduplication. --- src/libstore/local-overlay-store.cc | 5 +++++ src/libstore/local-overlay-store.hh | 2 ++ tests/overlay-local-store/local.mk | 3 ++- tests/overlay-local-store/optimise-inner.sh | 19 +++++++++++++++++++ tests/overlay-local-store/optimise.sh | 5 +++++ 5 files changed, 33 insertions(+), 1 deletion(-) create mode 100755 tests/overlay-local-store/optimise-inner.sh create mode 100755 tests/overlay-local-store/optimise.sh diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 182955fbe..da0c8e3d5 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -188,6 +188,11 @@ void LocalOverlayStore::deleteGCPath(const Path & path, uint64_t & bytesFreed) } } +void LocalOverlayStore::optimiseStore() +{ + warn("not implemented"); +} + bool LocalOverlayStore::verifyStore(bool checkContents, RepairFlag repair) { if (repair) diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 8fa99ee14..ef377b7a6 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -113,6 +113,8 @@ private: void deleteGCPath(const Path & path, uint64_t & bytesFreed) override; + void optimiseStore() override; + bool verifyStore(bool checkContents, RepairFlag repair) override; }; diff --git a/tests/overlay-local-store/local.mk b/tests/overlay-local-store/local.mk index 5e8c76b4e..3a6d00bc1 100644 --- a/tests/overlay-local-store/local.mk +++ b/tests/overlay-local-store/local.mk @@ -4,6 +4,7 @@ overlay-local-store-tests := \ $(d)/build.sh \ $(d)/bad-uris.sh \ $(d)/add-lower.sh \ - $(d)/verify.sh + $(d)/verify.sh \ + $(d)/optimise.sh install-tests-groups += overlay-local-store diff --git a/tests/overlay-local-store/optimise-inner.sh b/tests/overlay-local-store/optimise-inner.sh new file mode 100755 index 000000000..76c5a0cb6 --- /dev/null +++ b/tests/overlay-local-store/optimise-inner.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +set -x + +source common.sh + +# Avoid store dir being inside sandbox build-dir +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +storeDirs + +initLowerStore + +mountOverlayfs + +nix-store --store "$storeB" --optimise diff --git a/tests/overlay-local-store/optimise.sh b/tests/overlay-local-store/optimise.sh new file mode 100755 index 000000000..569afa248 --- /dev/null +++ b/tests/overlay-local-store/optimise.sh @@ -0,0 +1,5 @@ +source common.sh + +requireEnvironment +setupConfig +execUnshare ./optimise-inner.sh From a9510f950228957cde98001b67e123aa2d4d62c9 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Wed, 19 Jul 2023 11:23:54 +0100 Subject: [PATCH 074/133] Implement test for store path deduplication. --- tests/overlay-local-store/common.sh | 16 ++++------ tests/overlay-local-store/optimise-inner.sh | 33 +++++++++++++++++++++ 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/tests/overlay-local-store/common.sh b/tests/overlay-local-store/common.sh index 7850c068b..5a678c947 100644 --- a/tests/overlay-local-store/common.sh +++ b/tests/overlay-local-store/common.sh @@ -22,11 +22,12 @@ storeDirs () { # Mounting Overlay Store mountOverlayfs () { + mergedStorePath="$TEST_ROOT/merged-store/nix/store" mount -t overlay overlay \ -o lowerdir="$storeA/nix/store" \ -o upperdir="$storeBTop" \ -o workdir="$TEST_ROOT/workdir" \ - "$TEST_ROOT/merged-store/nix/store" \ + "$mergedStorePath" \ || skipTest "overlayfs is not supported" cleanupOverlay () { @@ -36,6 +37,10 @@ mountOverlayfs () { trap cleanupOverlay EXIT } +remountOverlayfs () { + mount -o remount "$mergedStorePath" +} + toRealPath () { storeDir=$1; shift storePath=$1; shift @@ -54,12 +59,3 @@ initLowerStore () { execUnshare () { exec unshare --mount --map-root-user "$@" } - -addTextToStore() { - storeDir=$1; shift - filename=$1; shift - content=$1; shift - filePath="$TEST_HOME/$filename" - echo "$content" > "$filePath" - nix-store --store "$storeDir" --add "$filePath" -} diff --git a/tests/overlay-local-store/optimise-inner.sh b/tests/overlay-local-store/optimise-inner.sh index 76c5a0cb6..c60c742a2 100755 --- a/tests/overlay-local-store/optimise-inner.sh +++ b/tests/overlay-local-store/optimise-inner.sh @@ -16,4 +16,37 @@ initLowerStore mountOverlayfs +# Create a file to add to store +dupFilePath="$TEST_ROOT/dup-file" +echo Duplicate > "$dupFilePath" + +# Add it to the overlay store (it will be written to the upper layer) +dupFileStorePath=$(nix-store --store "$storeB" --add "$dupFilePath") + +# Now add it to the lower store so the store path is duplicated +nix-store --store "$storeA" --add "$dupFilePath" + +# Ensure overlayfs and layers and synchronised +remountOverlayfs + +dupFilename="${dupFileStorePath#/nix/store}" +lowerPath="$storeA/$dupFileStorePath" +upperPath="$storeBTop/$dupFilename" +overlayPath="$mergedStorePath/$dupFilename" + +# Check store path exists in both layers and overlay +lowerInode=$(stat -c %i "$lowerPath") +upperInode=$(stat -c %i "$upperPath") +overlayInode=$(stat -c %i "$overlayPath") +[[ $upperInode == $overlayInode ]] +[[ $upperInode != $lowerInode ]] + +# Run optimise to deduplicate store paths nix-store --store "$storeB" --optimise +remountOverlayfs + +stat "$lowerPath" +stat "$overlayPath" +expect 1 stat "$upperPath" + +#expect 1 stat $(toRealPath "$storeA/nix/store" "$path") From 8ddbcb736a17d08ce899de345548064eebf673c9 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Wed, 19 Jul 2023 12:32:32 +0100 Subject: [PATCH 075/133] Implement overlay store deduplication. --- src/libstore/local-overlay-store.cc | 18 +++++++++++++++++- tests/overlay-local-store/optimise-inner.sh | 3 +-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index da0c8e3d5..9e530ed9b 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -190,7 +190,23 @@ void LocalOverlayStore::deleteGCPath(const Path & path, uint64_t & bytesFreed) void LocalOverlayStore::optimiseStore() { - warn("not implemented"); + Activity act(*logger, actOptimiseStore); + + // Note for LocalOverlayStore, queryAllValidPaths only returns paths in upper layer + auto paths = queryAllValidPaths(); + + act.progress(0, paths.size()); + + uint64_t done = 0; + + for (auto & path : paths) { + if (lowerStore->isValidPath(path)) { + // Deduplicate store path + deletePath(toUpperPath(path)); + } + done++; + act.progress(done, paths.size()); + } } bool LocalOverlayStore::verifyStore(bool checkContents, RepairFlag repair) diff --git a/tests/overlay-local-store/optimise-inner.sh b/tests/overlay-local-store/optimise-inner.sh index c60c742a2..2b778b311 100755 --- a/tests/overlay-local-store/optimise-inner.sh +++ b/tests/overlay-local-store/optimise-inner.sh @@ -45,8 +45,7 @@ overlayInode=$(stat -c %i "$overlayPath") nix-store --store "$storeB" --optimise remountOverlayfs +# Check path only exists in lower store stat "$lowerPath" stat "$overlayPath" expect 1 stat "$upperPath" - -#expect 1 stat $(toRealPath "$storeA/nix/store" "$path") From d1c77b201a1b0e6c14470b69c134652a8858fc18 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Wed, 19 Jul 2023 13:25:37 +0100 Subject: [PATCH 076/133] Explicitly exec shell to fix ENOENT errors. --- tests/overlay-local-store/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/overlay-local-store/common.sh b/tests/overlay-local-store/common.sh index 5a678c947..0e98e931f 100644 --- a/tests/overlay-local-store/common.sh +++ b/tests/overlay-local-store/common.sh @@ -57,5 +57,5 @@ initLowerStore () { } execUnshare () { - exec unshare --mount --map-root-user "$@" + exec unshare --mount --map-root-user "$SHELL" "$@" } From 44f855d14ee40dfeaf52c4f01e2de124fe0f18fc Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Wed, 19 Jul 2023 15:09:58 +0100 Subject: [PATCH 077/133] Missing addTextToStore function. --- tests/overlay-local-store/common.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/overlay-local-store/common.sh b/tests/overlay-local-store/common.sh index 0e98e931f..2fba2e7f6 100644 --- a/tests/overlay-local-store/common.sh +++ b/tests/overlay-local-store/common.sh @@ -59,3 +59,12 @@ initLowerStore () { execUnshare () { exec unshare --mount --map-root-user "$SHELL" "$@" } + +addTextToStore() { + storeDir=$1; shift + filename=$1; shift + content=$1; shift + filePath="$TEST_HOME/$filename" + echo "$content" > "$filePath" + nix-store --store "$storeDir" --add "$filePath" +} From 7fda19e2f139a7e41cb53cdbdd692dbc98755d91 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Wed, 19 Jul 2023 16:46:51 +0100 Subject: [PATCH 078/133] Mount tmpfs first to ensure overlayfs works consistently. --- tests/overlay-local-store/common.sh | 19 +++++++++++-------- tests/overlay-local-store/optimise-inner.sh | 2 +- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/tests/overlay-local-store/common.sh b/tests/overlay-local-store/common.sh index 2fba2e7f6..6aec96ba1 100644 --- a/tests/overlay-local-store/common.sh +++ b/tests/overlay-local-store/common.sh @@ -13,26 +13,29 @@ setupConfig () { } storeDirs () { - storeA="$TEST_ROOT/store-a" - storeBTop="$TEST_ROOT/store-b" - storeB="local-overlay?root=$TEST_ROOT/merged-store&lower-store=$storeA&upper-layer=$storeBTop" + storesRoot="$TEST_ROOT/stores" + mkdir -p "$storesRoot" + mount -t tmpfs tmpfs "$storesRoot" + storeA="$storesRoot/store-a" + storeBTop="$storesRoot/store-b" + storeB="local-overlay?root=$storesRoot/merged-store&lower-store=$storeA&upper-layer=$storeBTop" # Creating testing directories - mkdir -p "$TEST_ROOT"/{store-a/nix/store,store-b,merged-store/nix/store,workdir} + mkdir -p "$storesRoot"/{store-a/nix/store,store-b,merged-store/nix/store,workdir} } # Mounting Overlay Store mountOverlayfs () { - mergedStorePath="$TEST_ROOT/merged-store/nix/store" + mergedStorePath="$storesRoot/merged-store/nix/store" mount -t overlay overlay \ -o lowerdir="$storeA/nix/store" \ -o upperdir="$storeBTop" \ - -o workdir="$TEST_ROOT/workdir" \ + -o workdir="$storesRoot/workdir" \ "$mergedStorePath" \ || skipTest "overlayfs is not supported" cleanupOverlay () { - umount "$TEST_ROOT/merged-store/nix/store" - rm -r $TEST_ROOT/workdir + umount "$storesRoot/merged-store/nix/store" + rm -r $storesRoot/workdir } trap cleanupOverlay EXIT } diff --git a/tests/overlay-local-store/optimise-inner.sh b/tests/overlay-local-store/optimise-inner.sh index 2b778b311..079e22326 100755 --- a/tests/overlay-local-store/optimise-inner.sh +++ b/tests/overlay-local-store/optimise-inner.sh @@ -17,7 +17,7 @@ initLowerStore mountOverlayfs # Create a file to add to store -dupFilePath="$TEST_ROOT/dup-file" +dupFilePath="$storesRoot/dup-file" echo Duplicate > "$dupFilePath" # Add it to the overlay store (it will be written to the upper layer) From 9769a0ae7d840f1df7db3de4524180016830e57e Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Thu, 20 Jul 2023 10:27:11 +0100 Subject: [PATCH 079/133] Ensure all overlay tests use new tmpfs store paths. --- tests/overlay-local-store/add-lower-inner.sh | 8 +++--- tests/overlay-local-store/bad-uris.sh | 4 +-- .../check-post-init-inner.sh | 2 +- tests/overlay-local-store/common.sh | 28 +++++++++++-------- tests/overlay-local-store/optimise-inner.sh | 2 +- 5 files changed, 25 insertions(+), 19 deletions(-) diff --git a/tests/overlay-local-store/add-lower-inner.sh b/tests/overlay-local-store/add-lower-inner.sh index 40a1f397c..8c71f0780 100755 --- a/tests/overlay-local-store/add-lower-inner.sh +++ b/tests/overlay-local-store/add-lower-inner.sh @@ -18,14 +18,14 @@ mountOverlayfs # Add something to the overlay store overlayPath=$(addTextToStore "$storeB" "overlay-file" "Add to overlay store") -stat "$TEST_ROOT/merged-store/$overlayPath" +stat "$storeVolume/merged-store/$overlayPath" # Now add something to the lower store lowerPath=$(addTextToStore "$storeA" "lower-file" "Add to lower store") -stat "$TEST_ROOT/store-a/$lowerPath" +stat "$storeVolume/store-a/$lowerPath" # Remount overlayfs to ensure synchronization -mount -o remount "$TEST_ROOT/merged-store/nix/store" +mount -o remount "$storeVolume/merged-store/nix/store" # Path should be accessible via overlay store -stat "$TEST_ROOT/merged-store/$lowerPath" +stat "$storeVolume/merged-store/$lowerPath" diff --git a/tests/overlay-local-store/bad-uris.sh b/tests/overlay-local-store/bad-uris.sh index d4261bd97..c59264735 100644 --- a/tests/overlay-local-store/bad-uris.sh +++ b/tests/overlay-local-store/bad-uris.sh @@ -7,8 +7,8 @@ storeDirs mkdir -p $TEST_ROOT/bad_test badTestRoot=$TEST_ROOT/bad_test storeBadRoot="local-overlay?root=$badTestRoot&lower-store=$storeA&upper-layer=$storeBTop" -storeBadLower="local-overlay?root=$TEST_ROOT/merged-store&lower-store=$badTestRoot&upper-layer=$storeBTop" -storeBadUpper="local-overlay?root=$TEST_ROOT/merged-store&lower-store=$storeA&upper-layer=$badTestRoot" +storeBadLower="local-overlay?root=$storeVolume/merged-store&lower-store=$badTestRoot&upper-layer=$storeBTop" +storeBadUpper="local-overlay?root=$storeVolume/merged-store&lower-store=$storeA&upper-layer=$badTestRoot" declare -a storesBad=( "$storeBadRoot" "$storeBadLower" "$storeBadUpper" diff --git a/tests/overlay-local-store/check-post-init-inner.sh b/tests/overlay-local-store/check-post-init-inner.sh index 37ed8c113..c7d1c70d4 100755 --- a/tests/overlay-local-store/check-post-init-inner.sh +++ b/tests/overlay-local-store/check-post-init-inner.sh @@ -25,7 +25,7 @@ stat $(toRealPath "$storeA/nix/store" "$path") expect 1 stat $(toRealPath "$storeBTop" "$path") # Checking for path in overlay store matching lower layer -diff $(toRealPath "$storeA/nix/store" "$path") $(toRealPath "$TEST_ROOT/merged-store/nix/store" "$path") +diff $(toRealPath "$storeA/nix/store" "$path") $(toRealPath "$storeVolume/merged-store/nix/store" "$path") # Checking requisites query agreement [[ \ diff --git a/tests/overlay-local-store/common.sh b/tests/overlay-local-store/common.sh index 6aec96ba1..686523e9c 100644 --- a/tests/overlay-local-store/common.sh +++ b/tests/overlay-local-store/common.sh @@ -12,30 +12,36 @@ setupConfig () { echo "build-users-group = " >> "$NIX_CONF_DIR"/nix.conf } + + storeDirs () { - storesRoot="$TEST_ROOT/stores" - mkdir -p "$storesRoot" - mount -t tmpfs tmpfs "$storesRoot" - storeA="$storesRoot/store-a" - storeBTop="$storesRoot/store-b" - storeB="local-overlay?root=$storesRoot/merged-store&lower-store=$storeA&upper-layer=$storeBTop" + # Attempt to create store dirs on tmpfs volume. + # This ensures lowerdir, upperdir and workdir will be on + # a consistent filesystem that fully supports OverlayFS. + storeVolume="$TEST_ROOT/stores" + mkdir -p "$storeVolume" + mount -t tmpfs tmpfs "$storeVolume" || true # But continue anyway if that fails. + + storeA="$storeVolume/store-a" + storeBTop="$storeVolume/store-b" + storeB="local-overlay?root=$storeVolume/merged-store&lower-store=$storeA&upper-layer=$storeBTop" # Creating testing directories - mkdir -p "$storesRoot"/{store-a/nix/store,store-b,merged-store/nix/store,workdir} + mkdir -p "$storeVolume"/{store-a/nix/store,store-b,merged-store/nix/store,workdir} } # Mounting Overlay Store mountOverlayfs () { - mergedStorePath="$storesRoot/merged-store/nix/store" + mergedStorePath="$storeVolume/merged-store/nix/store" mount -t overlay overlay \ -o lowerdir="$storeA/nix/store" \ -o upperdir="$storeBTop" \ - -o workdir="$storesRoot/workdir" \ + -o workdir="$storeVolume/workdir" \ "$mergedStorePath" \ || skipTest "overlayfs is not supported" cleanupOverlay () { - umount "$storesRoot/merged-store/nix/store" - rm -r $storesRoot/workdir + umount "$storeVolume/merged-store/nix/store" + rm -r $storeVolume/workdir } trap cleanupOverlay EXIT } diff --git a/tests/overlay-local-store/optimise-inner.sh b/tests/overlay-local-store/optimise-inner.sh index 079e22326..2b778b311 100755 --- a/tests/overlay-local-store/optimise-inner.sh +++ b/tests/overlay-local-store/optimise-inner.sh @@ -17,7 +17,7 @@ initLowerStore mountOverlayfs # Create a file to add to store -dupFilePath="$storesRoot/dup-file" +dupFilePath="$TEST_ROOT/dup-file" echo Duplicate > "$dupFilePath" # Add it to the overlay store (it will be written to the upper layer) From 878c84d5ee88f2e6c56abb42211a729c4d342e14 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Thu, 20 Jul 2023 10:27:35 +0100 Subject: [PATCH 080/133] Fix errors about NIX_STORE_DIR being unset. --- tests/overlay-local-store/check-post-init-inner.sh | 2 +- tests/overlay-local-store/common.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/overlay-local-store/check-post-init-inner.sh b/tests/overlay-local-store/check-post-init-inner.sh index c7d1c70d4..2e7db2adc 100755 --- a/tests/overlay-local-store/check-post-init-inner.sh +++ b/tests/overlay-local-store/check-post-init-inner.sh @@ -62,7 +62,7 @@ nix-store --verify-path --store "$storeA" "$path" # Verifying path in merged-store nix-store --verify-path --store "$storeB" "$path" -hashPart=$(echo $path | sed "s^$NIX_STORE_DIR/^^" | sed 's/-.*//') +hashPart=$(echo $path | sed "s^${NIX_STORE_DIR:-/nix/store}/^^" | sed 's/-.*//') # Lower store can find from hash part [[ $(nix store --store $storeA path-from-hash-part $hashPart) == $path ]] diff --git a/tests/overlay-local-store/common.sh b/tests/overlay-local-store/common.sh index 686523e9c..ac4a59cd8 100644 --- a/tests/overlay-local-store/common.sh +++ b/tests/overlay-local-store/common.sh @@ -53,7 +53,7 @@ remountOverlayfs () { toRealPath () { storeDir=$1; shift storePath=$1; shift - echo $storeDir$(echo $storePath | sed "s^$NIX_STORE_DIR^^") + echo $storeDir$(echo $storePath | sed "s^${NIX_STORE_DIR:-/nix/store}^^") } initLowerStore () { From 2c66a093e05bc466d00c9365bc43567fd9d5c334 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Thu, 20 Jul 2023 11:03:14 +0100 Subject: [PATCH 081/133] Define storeBRoot variable distinct from storeB URI. --- tests/overlay-local-store/add-lower-inner.sh | 6 +++--- tests/overlay-local-store/bad-uris.sh | 4 ++-- tests/overlay-local-store/check-post-init-inner.sh | 2 +- tests/overlay-local-store/common.sh | 10 +++++----- tests/overlay-local-store/redundant-add-inner.sh | 5 ++--- 5 files changed, 13 insertions(+), 14 deletions(-) diff --git a/tests/overlay-local-store/add-lower-inner.sh b/tests/overlay-local-store/add-lower-inner.sh index 8c71f0780..ca7db7ab6 100755 --- a/tests/overlay-local-store/add-lower-inner.sh +++ b/tests/overlay-local-store/add-lower-inner.sh @@ -18,14 +18,14 @@ mountOverlayfs # Add something to the overlay store overlayPath=$(addTextToStore "$storeB" "overlay-file" "Add to overlay store") -stat "$storeVolume/merged-store/$overlayPath" +stat "$storeBRoot/$overlayPath" # Now add something to the lower store lowerPath=$(addTextToStore "$storeA" "lower-file" "Add to lower store") stat "$storeVolume/store-a/$lowerPath" # Remount overlayfs to ensure synchronization -mount -o remount "$storeVolume/merged-store/nix/store" +remountOverlayfs # Path should be accessible via overlay store -stat "$storeVolume/merged-store/$lowerPath" +stat "$storeBRoot/$lowerPath" diff --git a/tests/overlay-local-store/bad-uris.sh b/tests/overlay-local-store/bad-uris.sh index c59264735..462bf27eb 100644 --- a/tests/overlay-local-store/bad-uris.sh +++ b/tests/overlay-local-store/bad-uris.sh @@ -7,8 +7,8 @@ storeDirs mkdir -p $TEST_ROOT/bad_test badTestRoot=$TEST_ROOT/bad_test storeBadRoot="local-overlay?root=$badTestRoot&lower-store=$storeA&upper-layer=$storeBTop" -storeBadLower="local-overlay?root=$storeVolume/merged-store&lower-store=$badTestRoot&upper-layer=$storeBTop" -storeBadUpper="local-overlay?root=$storeVolume/merged-store&lower-store=$storeA&upper-layer=$badTestRoot" +storeBadLower="local-overlay?root=$storeBRoot&lower-store=$badTestRoot&upper-layer=$storeBTop" +storeBadUpper="local-overlay?root=$storeBRoot&lower-store=$storeA&upper-layer=$badTestRoot" declare -a storesBad=( "$storeBadRoot" "$storeBadLower" "$storeBadUpper" diff --git a/tests/overlay-local-store/check-post-init-inner.sh b/tests/overlay-local-store/check-post-init-inner.sh index 2e7db2adc..0f4654cc2 100755 --- a/tests/overlay-local-store/check-post-init-inner.sh +++ b/tests/overlay-local-store/check-post-init-inner.sh @@ -25,7 +25,7 @@ stat $(toRealPath "$storeA/nix/store" "$path") expect 1 stat $(toRealPath "$storeBTop" "$path") # Checking for path in overlay store matching lower layer -diff $(toRealPath "$storeA/nix/store" "$path") $(toRealPath "$storeVolume/merged-store/nix/store" "$path") +diff $(toRealPath "$storeA/nix/store" "$path") $(toRealPath "$storeBRoot/nix/store" "$path") # Checking requisites query agreement [[ \ diff --git a/tests/overlay-local-store/common.sh b/tests/overlay-local-store/common.sh index ac4a59cd8..2b23352ab 100644 --- a/tests/overlay-local-store/common.sh +++ b/tests/overlay-local-store/common.sh @@ -24,30 +24,30 @@ storeDirs () { storeA="$storeVolume/store-a" storeBTop="$storeVolume/store-b" - storeB="local-overlay?root=$storeVolume/merged-store&lower-store=$storeA&upper-layer=$storeBTop" + storeBRoot="$storeVolume/merged-store" + storeB="local-overlay?root=$storeBRoot&lower-store=$storeA&upper-layer=$storeBTop" # Creating testing directories mkdir -p "$storeVolume"/{store-a/nix/store,store-b,merged-store/nix/store,workdir} } # Mounting Overlay Store mountOverlayfs () { - mergedStorePath="$storeVolume/merged-store/nix/store" mount -t overlay overlay \ -o lowerdir="$storeA/nix/store" \ -o upperdir="$storeBTop" \ -o workdir="$storeVolume/workdir" \ - "$mergedStorePath" \ + "$storeBRoot/nix/store" \ || skipTest "overlayfs is not supported" cleanupOverlay () { - umount "$storeVolume/merged-store/nix/store" + umount "$storeBRoot/nix/store" rm -r $storeVolume/workdir } trap cleanupOverlay EXIT } remountOverlayfs () { - mount -o remount "$mergedStorePath" + mount -o remount "$storeBRoot/nix/store" } toRealPath () { diff --git a/tests/overlay-local-store/redundant-add-inner.sh b/tests/overlay-local-store/redundant-add-inner.sh index cfdae68b4..34b841e38 100755 --- a/tests/overlay-local-store/redundant-add-inner.sh +++ b/tests/overlay-local-store/redundant-add-inner.sh @@ -7,7 +7,7 @@ set -x source common.sh # Avoid store dir being inside sandbox build-dir -unset NIX_STORE_DIR # TODO: This causes toRealPath to fail (it expects this var to be set) +unset NIX_STORE_DIR unset NIX_STATE_DIR storeDirs @@ -27,5 +27,4 @@ path=$(nix-store --store "$storeB" --add ../dummy) stat $(toRealPath "$storeA/nix/store" "$path") # upper layer should still not have it (no redundant copy) -expect 1 stat $(toRealPath "$storeB/nix/store" "$path") # TODO: Check this is failing for the right reason. - # $storeB is a store URI not a directory path +expect 1 stat $(toRealPath "$storeBTop" "$path") From 2fc00ec19f8eb7ec2285135a180763d632102762 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Thu, 20 Jul 2023 11:27:28 +0100 Subject: [PATCH 082/133] Fix unbound variable error in optimise test. --- tests/overlay-local-store/optimise-inner.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/overlay-local-store/optimise-inner.sh b/tests/overlay-local-store/optimise-inner.sh index 2b778b311..b7994054c 100755 --- a/tests/overlay-local-store/optimise-inner.sh +++ b/tests/overlay-local-store/optimise-inner.sh @@ -32,7 +32,7 @@ remountOverlayfs dupFilename="${dupFileStorePath#/nix/store}" lowerPath="$storeA/$dupFileStorePath" upperPath="$storeBTop/$dupFilename" -overlayPath="$mergedStorePath/$dupFilename" +overlayPath="$storeBRoot/nix/store/$dupFilename" # Check store path exists in both layers and overlay lowerInode=$(stat -c %i "$lowerPath") From 0e595a52a32003ed448b1da9319a639059f993a6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 24 Jul 2023 15:39:59 -0400 Subject: [PATCH 083/133] Remove trailing whitespace --- src/libstore/local-overlay-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 9e530ed9b..aced582ca 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -87,7 +87,7 @@ void LocalOverlayStore::queryPathInfoUncached(const StorePath & path, void LocalOverlayStore::queryRealisationUncached(const DrvOutput & drvOutput, - Callback> callback) noexcept + Callback> callback) noexcept { auto callbackPtr = std::make_shared(std::move(callback)); From 3731208dc120b5d12f1c5c63ec58bcdb34c42195 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 24 Jul 2023 16:54:14 -0400 Subject: [PATCH 084/133] Adopt GC test for local-overlay store Doesn't yet pass. Fixes are needed. --- tests/overlay-local-store/gc-inner.sh | 57 +++++++++++++++++++++++++++ tests/overlay-local-store/gc.sh | 5 +++ tests/overlay-local-store/local.mk | 1 + 3 files changed, 63 insertions(+) create mode 100644 tests/overlay-local-store/gc-inner.sh create mode 100755 tests/overlay-local-store/gc.sh diff --git a/tests/overlay-local-store/gc-inner.sh b/tests/overlay-local-store/gc-inner.sh new file mode 100644 index 000000000..01ec48b84 --- /dev/null +++ b/tests/overlay-local-store/gc-inner.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +source common.sh + +# Avoid store dir being inside sandbox build-dir +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +storeDirs + +initLowerStore + +mountOverlayfs + +export NIX_REMOTE="$storeB" +stateB="$storeBRoot/nix/var/nix" +outPath=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg seed 2) + +# Set a GC root. +mkdir -p "$stateB" +rm -f "$stateB"/gcroots/foo +ln -sf $outPath "$stateB"/gcroots/foo + +[ "$(nix-store -q --roots $outPath)" = "$stateB/gcroots/foo -> $outPath" ] + +nix-store --gc --print-roots | grep $outPath +nix-store --gc --print-live | grep $outPath +if nix-store --gc --print-dead | grep -E $outPath$; then false; fi + +nix-store --gc --print-dead + +expect 1 nix-store --delete $outPath +test -e "$storeBRoot/$outPath" + +shopt -s nullglob +for i in $storeBRoot/*; do + if [[ $i =~ /trash ]]; then continue; fi # compat with old daemon + touch $i.lock + touch $i.chroot +done + +nix-collect-garbage + +# Check that the root and its dependencies haven't been deleted. +cat "$storeBRoot/$outPath" + +rm "$stateB"/gcroots/foo + +nix-collect-garbage + +# Check that the output has been GC'd. +test ! -e $outPath + +# Check that the store is empty. +[ "$(ls -1 "$storeBTop" | wc -l)" = "0" ] diff --git a/tests/overlay-local-store/gc.sh b/tests/overlay-local-store/gc.sh new file mode 100755 index 000000000..1e1fb203e --- /dev/null +++ b/tests/overlay-local-store/gc.sh @@ -0,0 +1,5 @@ +source common.sh + +requireEnvironment +setupConfig +execUnshare ./gc-inner.sh diff --git a/tests/overlay-local-store/local.mk b/tests/overlay-local-store/local.mk index 3a6d00bc1..1f8de8f27 100644 --- a/tests/overlay-local-store/local.mk +++ b/tests/overlay-local-store/local.mk @@ -4,6 +4,7 @@ overlay-local-store-tests := \ $(d)/build.sh \ $(d)/bad-uris.sh \ $(d)/add-lower.sh \ + $(d)/gc.sh \ $(d)/verify.sh \ $(d)/optimise.sh From 497464f4945200522f05a318720ad0a158095e8e Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 25 Jul 2023 13:30:21 +0100 Subject: [PATCH 085/133] Extend verify test to check that repair is supported. --- src/libstore/local-overlay-store.cc | 7 ----- src/libstore/local-overlay-store.hh | 2 -- tests/overlay-local-store/common.sh | 8 +++-- tests/overlay-local-store/verify-inner.sh | 36 +++++++++++++++++++---- 4 files changed, 37 insertions(+), 16 deletions(-) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 9e530ed9b..c8464f64d 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -209,13 +209,6 @@ void LocalOverlayStore::optimiseStore() } } -bool LocalOverlayStore::verifyStore(bool checkContents, RepairFlag repair) -{ - if (repair) - warn("local-overlay: store does not support --verify --repair"); - return LocalStore::verifyStore(checkContents, NoRepair); -} - static RegisterStoreImplementation regLocalOverlayStore; } diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index ef377b7a6..349d9e6ed 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -114,8 +114,6 @@ private: void deleteGCPath(const Path & path, uint64_t & bytesFreed) override; void optimiseStore() override; - - bool verifyStore(bool checkContents, RepairFlag repair) override; }; } diff --git a/tests/overlay-local-store/common.sh b/tests/overlay-local-store/common.sh index 2b23352ab..6bb5bc391 100644 --- a/tests/overlay-local-store/common.sh +++ b/tests/overlay-local-store/common.sh @@ -7,9 +7,13 @@ requireEnvironment () { needLocalStore "The test uses --store always so we would just be bypassing the daemon" } +addConfig () { + echo "$1" >> "$NIX_CONF_DIR/nix.conf" +} + setupConfig () { - echo "require-drop-supplementary-groups = false" >> "$NIX_CONF_DIR"/nix.conf - echo "build-users-group = " >> "$NIX_CONF_DIR"/nix.conf + addConfig "require-drop-supplementary-groups = false" + addConfig "build-users-group = " } diff --git a/tests/overlay-local-store/verify-inner.sh b/tests/overlay-local-store/verify-inner.sh index 8f8839302..de40c3d05 100755 --- a/tests/overlay-local-store/verify-inner.sh +++ b/tests/overlay-local-store/verify-inner.sh @@ -16,25 +16,51 @@ initLowerStore mountOverlayfs + +## Initialise stores for test + # Realise a derivation from the lower store to propagate paths to overlay DB nix-store --store "$storeB" --realise $drvPath # Also ensure dummy file exists in overlay DB dummyPath=$(nix-store --store "$storeB" --add ../dummy) +# Add something to the lower store that will not be propagated to overlay DB +lowerOnlyPath=$(addTextToStore "$storeA" lower-only "Only in lower store") + # Verify should be successful at this point nix-store --store "$storeB" --verify --check-contents -# Now delete one of the derivation inputs in the lower store +# Make a backup so we can repair later +backupStore="$storeVolume/backup" +mkdir "$backupStore" +tar -cC "$storeBRoot" nix | tar -xC "$backupStore" + + +## Deliberately corrupt store paths + +# Delete one of the derivation inputs in the lower store inputDrvFullPath=$(find "$storeA" -name "*-hermetic-input-1.drv") inputDrvPath=${inputDrvFullPath/*\/nix\/store\///nix/store/} rm -v "$inputDrvFullPath" -# And truncate the contents of dummy file in lower store +# Truncate the contents of dummy file in lower store find "$storeA" -name "*-dummy" -exec truncate -s 0 {} \; -# Verify should fail with the messages about missing input and modified dummy file -verifyOutput=$(expectStderr 1 nix-store --store "$storeB" --verify --check-contents --repair) +# Also truncate the file that only exists in lower store +truncate -s 0 "$storeA/$lowerOnlyPath" + + +## Now test that verify and repair work as expected + +# Verify overlay store without attempting to repair it +verifyOutput=$(expectStderr 1 nix-store --store "$storeB" --verify --check-contents) <<<"$verifyOutput" grepQuiet "path '$inputDrvPath' disappeared, but it still has valid referrers!" <<<"$verifyOutput" grepQuiet "path '$dummyPath' was modified! expected hash" -<<<"$verifyOutput" grepQuiet "store does not support --verify --repair" +<<<"$verifyOutput" expectStderr 1 grepQuiet "$lowerOnlyPath" # Expect no error for corrupted lower-only path + +# Attempt to repair using backup +addConfig "substituters = $backupStore" +repairOutput=$(nix-store --store "$storeB" --verify --check-contents --repair 2>&1) +<<<"$repairOutput" grepQuiet "copying path '$inputDrvPath'" +<<<"$repairOutput" grepQuiet "copying path '$dummyPath'" From 9ef0a9e8aa2e27991434c104ad73b1b95b241f08 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 25 Jul 2023 10:28:11 -0400 Subject: [PATCH 086/133] Fix hard linking issue causing overlay fs copy-ups --- src/libstore/build/local-derivation-goal.cc | 7 ++++--- src/libstore/local-fs-store.hh | 10 ++++++++++ src/libstore/local-overlay-store.cc | 7 +++++++ src/libstore/local-overlay-store.hh | 2 ++ 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index b7a27490c..3bc88ee86 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -386,8 +386,9 @@ void LocalDerivationGoal::cleanupPostOutputsRegisteredModeNonCheck() #if __linux__ -static void linkOrCopy(const Path & from, const Path & to) +static void linkOrCopy(LocalFSStore & store, const StorePath & from_, const Path & to) { + auto from = store.toRealPathForHardLink(from_); if (link(from.c_str(), to.c_str()) == -1) { /* Hard-linking fails if we exceed the maximum link count on a file (e.g. 32000 of ext3), which is quite possible after a @@ -712,7 +713,7 @@ void LocalDerivationGoal::startBuilder() if (S_ISDIR(lstat(r).st_mode)) dirsInChroot.insert_or_assign(p, r); else - linkOrCopy(r, chrootRootDir + p); + linkOrCopy(getLocalStore(), i, chrootRootDir + p); } /* If we're repairing, checking or rebuilding part of a @@ -1574,7 +1575,7 @@ void LocalDerivationGoal::addDependency(const StorePath & path) throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path)); } else - linkOrCopy(source, target); + linkOrCopy(getLocalStore(), path, target); #else throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox", diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 488109501..19858f5c8 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -73,6 +73,16 @@ public: return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1); } + /** + * If the real path is hardlinked with something else, we might + * prefer to refer to the other path instead. This is the case with + * overlayfs, for example. + */ + virtual Path toRealPathForHardLink(const StorePath & storePath) + { + return Store::toRealPath(storePath); + } + std::optional getBuildLogExact(const StorePath & path) override; }; diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 700a5227b..47d09dc75 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -209,6 +209,13 @@ void LocalOverlayStore::optimiseStore() } } +Path LocalOverlayStore::toRealPathForHardLink(const StorePath & path) +{ + return lowerStore->isValidPath(path) + ? lowerStore->Store::toRealPath(path) + : Store::toRealPath(path); +} + static RegisterStoreImplementation regLocalOverlayStore; } diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 349d9e6ed..64e2ef488 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -114,6 +114,8 @@ private: void deleteGCPath(const Path & path, uint64_t & bytesFreed) override; void optimiseStore() override; + + Path toRealPathForHardLink(const StorePath & storePath) override; }; } From 19c43c5d787c113053ef651c3d22fdfde61075e2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 25 Jul 2023 11:44:39 -0400 Subject: [PATCH 087/133] Write test for deleting objects referenced from below Currently fails, as expected. --- tests/hermetic.nix | 4 +-- tests/overlay-local-store/delete-inner.sh | 39 +++++++++++++++++++++++ tests/overlay-local-store/delete.sh | 5 +++ tests/overlay-local-store/local.mk | 1 + 4 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 tests/overlay-local-store/delete-inner.sh create mode 100755 tests/overlay-local-store/delete.sh diff --git a/tests/hermetic.nix b/tests/hermetic.nix index 4c9d7a51f..c4fbbfa14 100644 --- a/tests/hermetic.nix +++ b/tests/hermetic.nix @@ -37,7 +37,7 @@ let buildCommand = '' echo hi-input3 read x < ${input2} - echo $x BAZ > $out + echo ${input2} $x BAZ > $out ''; }; @@ -51,6 +51,6 @@ in '' read x < ${input1} read y < ${input3} - echo "$x $y" > $out + echo ${input1} ${input3} "$x $y" > $out ''; } diff --git a/tests/overlay-local-store/delete-inner.sh b/tests/overlay-local-store/delete-inner.sh new file mode 100644 index 000000000..0702d1693 --- /dev/null +++ b/tests/overlay-local-store/delete-inner.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +source common.sh + +# Avoid store dir being inside sandbox build-dir +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +storeDirs + +initLowerStore + +mountOverlayfs + +export NIX_REMOTE="$storeB" +stateB="$storeBRoot/nix/var/nix" +hermetic=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg seed 2) +input1=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg seed 2 -A passthru.input1 -j0) +input2=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg seed 2 -A passthru.input2 -j0) +input3=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg seed 2 -A passthru.input3 -j0) + +# Can't delete because referenced +expectStderr 1 nix-store --delete $input1 | grepQuiet "Cannot delete path" +expectStderr 1 nix-store --delete $input2 | grepQuiet "Cannot delete path" +expectStderr 1 nix-store --delete $input3 | grepQuiet "Cannot delete path" + +# These same paths are referenced in the lower layer (by the seed 1 +# build done in `initLowerStore`). +expectStderr 1 nix-store --store "$storeA" --delete $input2 | grepQuiet "Cannot delete path" +expectStderr 1 nix-store --store "$storeA" --delete $input3 | grepQuiet "Cannot delete path" + +# Can delete +nix-store --delete $hermetic + +# Now unreferenced in upper layer, can delete +nix-store --delete $input3 +nix-store --delete $input2 diff --git a/tests/overlay-local-store/delete.sh b/tests/overlay-local-store/delete.sh new file mode 100755 index 000000000..79d67da71 --- /dev/null +++ b/tests/overlay-local-store/delete.sh @@ -0,0 +1,5 @@ +source common.sh + +requireEnvironment +setupConfig +execUnshare ./delete-inner.sh diff --git a/tests/overlay-local-store/local.mk b/tests/overlay-local-store/local.mk index 1f8de8f27..c9cb0b7f2 100644 --- a/tests/overlay-local-store/local.mk +++ b/tests/overlay-local-store/local.mk @@ -4,6 +4,7 @@ overlay-local-store-tests := \ $(d)/build.sh \ $(d)/bad-uris.sh \ $(d)/add-lower.sh \ + $(d)/delete.sh \ $(d)/gc.sh \ $(d)/verify.sh \ $(d)/optimise.sh From 07b34edc449f25e9b4062d7f6d90dad1f6c71760 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 25 Jul 2023 17:09:23 -0400 Subject: [PATCH 088/133] Fix deletion test Lower layer references are ignored for deleting just in the upper layer. --- src/libstore/gc.cc | 2 +- src/libstore/local-overlay-store.cc | 8 ++++++++ src/libstore/local-overlay-store.hh | 9 +++++++++ src/libstore/local-store.hh | 12 ++++++++++++ 4 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index ea2868c58..8f5ce0994 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -741,7 +741,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) auto i = referrersCache.find(*path); if (i == referrersCache.end()) { StorePathSet referrers; - queryReferrers(*path, referrers); + queryGCReferrers(*path, referrers); referrersCache.emplace(*path, std::move(referrers)); i = referrersCache.find(*path); } diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 47d09dc75..e8ca8074e 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -138,6 +138,12 @@ void LocalOverlayStore::queryReferrers(const StorePath & path, StorePathSet & re } +void LocalOverlayStore::queryGCReferrers(const StorePath & path, StorePathSet & referrers) +{ + LocalStore::queryReferrers(path, referrers); +} + + StorePathSet LocalOverlayStore::queryValidDerivers(const StorePath & path) { auto res = LocalStore::queryValidDerivers(path); @@ -188,6 +194,7 @@ void LocalOverlayStore::deleteGCPath(const Path & path, uint64_t & bytesFreed) } } + void LocalOverlayStore::optimiseStore() { Activity act(*logger, actOptimiseStore); @@ -216,6 +223,7 @@ Path LocalOverlayStore::toRealPathForHardLink(const StorePath & path) : Store::toRealPath(path); } + static RegisterStoreImplementation regLocalOverlayStore; } diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 64e2ef488..21658c6ab 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -115,7 +115,16 @@ private: void optimiseStore() override; + /** + * For lower-store paths, we used the lower store location. This avoids the + * wasteful "copying up" that would otherwise happen. + */ Path toRealPathForHardLink(const StorePath & storePath) override; + + /** + * Deletion only effects the upper layer, so we ignore lower-layer referrers. + */ + void queryGCReferrers(const StorePath & path, StorePathSet & referrers) override; }; } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 9a44722d4..c049de02e 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -230,6 +230,18 @@ public: void collectGarbage(const GCOptions & options, GCResults & results) override; + /** + * Called by `collectGarbage` to trace in reverse. + * + * Using this rather than `queryReferrers` directly allows us to + * fine-tune which referrers we consider for garbage collection; + * some store implementations take advantage of this. + */ + virtual void queryGCReferrers(const StorePath & path, StorePathSet & referrers) + { + return queryReferrers(path, referrers); + } + /** * Called by `collectGarbage` to recursively delete a path. * The default implementation simply calls `deletePath`, but it can be From b0877ad3c90a806e70cac3e7f6e5c26151239bb8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 26 Jul 2023 09:29:04 -0400 Subject: [PATCH 089/133] Give test a more specific name --- .../{delete-inner.sh => delete-refs-inner.sh} | 0 tests/overlay-local-store/{delete.sh => delete-refs.sh} | 2 +- tests/overlay-local-store/local.mk | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename tests/overlay-local-store/{delete-inner.sh => delete-refs-inner.sh} (100%) rename tests/overlay-local-store/{delete.sh => delete-refs.sh} (58%) diff --git a/tests/overlay-local-store/delete-inner.sh b/tests/overlay-local-store/delete-refs-inner.sh similarity index 100% rename from tests/overlay-local-store/delete-inner.sh rename to tests/overlay-local-store/delete-refs-inner.sh diff --git a/tests/overlay-local-store/delete.sh b/tests/overlay-local-store/delete-refs.sh similarity index 58% rename from tests/overlay-local-store/delete.sh rename to tests/overlay-local-store/delete-refs.sh index 79d67da71..942d7fbdc 100755 --- a/tests/overlay-local-store/delete.sh +++ b/tests/overlay-local-store/delete-refs.sh @@ -2,4 +2,4 @@ source common.sh requireEnvironment setupConfig -execUnshare ./delete-inner.sh +execUnshare ./delete-refs-inner.sh diff --git a/tests/overlay-local-store/local.mk b/tests/overlay-local-store/local.mk index c9cb0b7f2..8bfb22ae7 100644 --- a/tests/overlay-local-store/local.mk +++ b/tests/overlay-local-store/local.mk @@ -4,7 +4,7 @@ overlay-local-store-tests := \ $(d)/build.sh \ $(d)/bad-uris.sh \ $(d)/add-lower.sh \ - $(d)/delete.sh \ + $(d)/delete-refs.sh \ $(d)/gc.sh \ $(d)/verify.sh \ $(d)/optimise.sh From d9688ba70881f1735cf36161244a9a6b2acf4d48 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Wed, 26 Jul 2023 11:31:26 +0100 Subject: [PATCH 090/133] Add new remount-hook store parameter. --- src/libstore/local-overlay-store.hh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 64e2ef488..f489c6ed6 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -41,6 +41,13 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig default, but can be disabled if needed. )"}; + const PathSetting remountHook{(StoreConfig*) this, "", "remount-hook", + R"( + Script or program to run when overlay filesystem needs remounting. + + TODO: Document this in more detail. + )"}; + const std::string name() override { return "Experimental Local Overlay Store"; } std::string doc() override From cc6f8aa91a1716ed1801f51fb3a6a1fa468f338f Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Wed, 26 Jul 2023 12:33:26 +0100 Subject: [PATCH 091/133] Test that delete works for duplicate file edge case. --- tests/overlay-local-store/delete-inner.sh | 38 +++++++++++++++++++++++ tests/overlay-local-store/delete.sh | 5 +++ tests/overlay-local-store/local.mk | 3 +- 3 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 tests/overlay-local-store/delete-inner.sh create mode 100644 tests/overlay-local-store/delete.sh diff --git a/tests/overlay-local-store/delete-inner.sh b/tests/overlay-local-store/delete-inner.sh new file mode 100644 index 000000000..20b39c7fa --- /dev/null +++ b/tests/overlay-local-store/delete-inner.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +set -x + +source common.sh + +# Avoid store dir being inside sandbox build-dir +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +storeDirs + +initLowerStore + +mountOverlayfs + +# Add to overlay before lower to ensure file is duplicated +upperPath=$(nix-store --store "$storeB" --add delete.sh) +lowerPath=$(nix-store --store "$storeA" --add delete.sh) +[[ "$upperPath" = "$lowerPath" ]] + +# Check there really are two files with different inodes +upperInode=$(stat -c %i "$storeBRoot/$upperPath") +lowerInode=$(stat -c %i "$storeA/$lowerPath") +[[ "$upperInode" != "$lowerInode" ]] + +# Now delete file via the overlay store +nix-store --store "$storeB" --delete "$upperPath" + +# Check there is no longer a file in upper layer +expect 1 stat "$storeBTop/${upperPath##/nix/store/}" + +# Check that overlay file is now the one in lower layer +upperInode=$(stat -c %i "$storeBRoot/$upperPath") +lowerInode=$(stat -c %i "$storeA/$lowerPath") +[[ "$upperInode" = "$lowerInode" ]] diff --git a/tests/overlay-local-store/delete.sh b/tests/overlay-local-store/delete.sh new file mode 100644 index 000000000..79d67da71 --- /dev/null +++ b/tests/overlay-local-store/delete.sh @@ -0,0 +1,5 @@ +source common.sh + +requireEnvironment +setupConfig +execUnshare ./delete-inner.sh diff --git a/tests/overlay-local-store/local.mk b/tests/overlay-local-store/local.mk index 1f8de8f27..9233462a3 100644 --- a/tests/overlay-local-store/local.mk +++ b/tests/overlay-local-store/local.mk @@ -6,6 +6,7 @@ overlay-local-store-tests := \ $(d)/add-lower.sh \ $(d)/gc.sh \ $(d)/verify.sh \ - $(d)/optimise.sh + $(d)/optimise.sh \ + $(d)/delete.sh install-tests-groups += overlay-local-store From 11c493f8fa88a81645318844077392020618887f Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Wed, 26 Jul 2023 13:21:38 +0100 Subject: [PATCH 092/133] Avoid creating whiteout for duplicate store paths. --- src/libstore/local-overlay-store.cc | 22 ++++++++++++++++++++-- src/libstore/local-overlay-store.hh | 2 ++ tests/overlay-local-store/delete-inner.sh | 1 + 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 47d09dc75..38c40fbad 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -183,11 +183,27 @@ void LocalOverlayStore::deleteGCPath(const Path & path, uint64_t & bytesFreed) warn("local-overlay: unexpected gc path '%s' ", path); return; } - if (pathExists(toUpperPath({path.substr(mergedDir.length())}))) { - LocalStore::deleteGCPath(path, bytesFreed); + + StorePath storePath = {path.substr(mergedDir.length())}; + auto upperPath = toUpperPath(storePath); + + if (pathExists(upperPath)) { + std::cerr << " upper exists" << std::endl; + if (lowerStore->isValidPath(storePath)) { + std::cerr << " lower exists" << std::endl; + // Path also exists in lower store. + // We must delete via upper layer to avoid creating a whiteout. + deletePath(upperPath, bytesFreed); + _remountRequired = true; + } else { + // Path does not exist in lower store. + // So we can delete via overlayfs and not need to remount. + LocalStore::deleteGCPath(path, bytesFreed); + } } } + void LocalOverlayStore::optimiseStore() { Activity act(*logger, actOptimiseStore); @@ -209,6 +225,7 @@ void LocalOverlayStore::optimiseStore() } } + Path LocalOverlayStore::toRealPathForHardLink(const StorePath & path) { return lowerStore->isValidPath(path) @@ -216,6 +233,7 @@ Path LocalOverlayStore::toRealPathForHardLink(const StorePath & path) : Store::toRealPath(path); } + static RegisterStoreImplementation regLocalOverlayStore; } diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index f489c6ed6..ffb310b7b 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -123,6 +123,8 @@ private: void optimiseStore() override; Path toRealPathForHardLink(const StorePath & storePath) override; + + bool _remountRequired = false; }; } diff --git a/tests/overlay-local-store/delete-inner.sh b/tests/overlay-local-store/delete-inner.sh index 20b39c7fa..051b59d5f 100644 --- a/tests/overlay-local-store/delete-inner.sh +++ b/tests/overlay-local-store/delete-inner.sh @@ -28,6 +28,7 @@ lowerInode=$(stat -c %i "$storeA/$lowerPath") # Now delete file via the overlay store nix-store --store "$storeB" --delete "$upperPath" +remountOverlayfs # Check there is no longer a file in upper layer expect 1 stat "$storeBTop/${upperPath##/nix/store/}" From 33ebae75ca4a7fd81585928b36a1cc18995f8212 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Wed, 26 Jul 2023 13:29:31 +0100 Subject: [PATCH 093/133] Reuse deletion logic for optimiseStore and rename method. --- src/libstore/gc.cc | 2 +- src/libstore/local-overlay-store.cc | 7 ++++--- src/libstore/local-overlay-store.hh | 2 +- src/libstore/local-store.cc | 2 +- src/libstore/local-store.hh | 2 +- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index ea2868c58..a909d063b 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -653,7 +653,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) results.paths.insert(path); uint64_t bytesFreed; - deleteGCPath(realPath, bytesFreed); + deleteStorePath(realPath, bytesFreed); results.bytesFreed += bytesFreed; diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 38c40fbad..8b89f68bc 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -176,7 +176,7 @@ void LocalOverlayStore::registerValidPaths(const ValidPathInfos & infos) } -void LocalOverlayStore::deleteGCPath(const Path & path, uint64_t & bytesFreed) +void LocalOverlayStore::deleteStorePath(const Path & path, uint64_t & bytesFreed) { auto mergedDir = realStoreDir.get() + "/"; if (path.substr(0, mergedDir.length()) != mergedDir) { @@ -198,7 +198,7 @@ void LocalOverlayStore::deleteGCPath(const Path & path, uint64_t & bytesFreed) } else { // Path does not exist in lower store. // So we can delete via overlayfs and not need to remount. - LocalStore::deleteGCPath(path, bytesFreed); + LocalStore::deleteStorePath(path, bytesFreed); } } } @@ -217,8 +217,9 @@ void LocalOverlayStore::optimiseStore() for (auto & path : paths) { if (lowerStore->isValidPath(path)) { + uint64_t bytesFreed = 0; // Deduplicate store path - deletePath(toUpperPath(path)); + deleteStorePath(Store::toRealPath(path), bytesFreed); } done++; act.progress(done, paths.size()); diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index ffb310b7b..d45f6bfc5 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -118,7 +118,7 @@ private: void queryRealisationUncached(const DrvOutput&, Callback> callback) noexcept override; - void deleteGCPath(const Path & path, uint64_t & bytesFreed) override; + void deleteStorePath(const Path & path, uint64_t & bytesFreed) override; void optimiseStore() override; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 21acb3c38..2c18867f5 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -457,7 +457,7 @@ AutoCloseFD LocalStore::openGCLock() } -void LocalStore::deleteGCPath(const Path & path, uint64_t & bytesFreed) +void LocalStore::deleteStorePath(const Path & path, uint64_t & bytesFreed) { deletePath(path, bytesFreed); } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 9a44722d4..9146f27a5 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -235,7 +235,7 @@ public: * The default implementation simply calls `deletePath`, but it can be * overridden by stores that wish to provide their own deletion behaviour. */ - virtual void deleteGCPath(const Path & path, uint64_t & bytesFreed); + virtual void deleteStorePath(const Path & path, uint64_t & bytesFreed); /** * Optimise the disk space usage of the Nix store by hard-linking From ed1428692409f94cd5553b66c9f5e6aa3c048e86 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Wed, 26 Jul 2023 14:05:54 +0100 Subject: [PATCH 094/133] Invoke remount-hook program when necessary. --- src/libstore/local-overlay-store.cc | 22 ++++++++++++++++++++++ src/libstore/local-overlay-store.hh | 4 ++++ tests/overlay-local-store/delete-inner.sh | 3 +-- tests/overlay-local-store/remount.sh | 2 ++ 4 files changed, 29 insertions(+), 2 deletions(-) create mode 100755 tests/overlay-local-store/remount.sh diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 8b89f68bc..76c3adf5f 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -176,6 +176,14 @@ void LocalOverlayStore::registerValidPaths(const ValidPathInfos & infos) } +void LocalOverlayStore::collectGarbage(const GCOptions & options, GCResults & results) +{ + LocalStore::collectGarbage(options, results); + + remountIfNecessary(); +} + + void LocalOverlayStore::deleteStorePath(const Path & path, uint64_t & bytesFreed) { auto mergedDir = realStoreDir.get() + "/"; @@ -224,6 +232,8 @@ void LocalOverlayStore::optimiseStore() done++; act.progress(done, paths.size()); } + + remountIfNecessary(); } @@ -235,6 +245,18 @@ Path LocalOverlayStore::toRealPathForHardLink(const StorePath & path) } +void LocalOverlayStore::remountIfNecessary() +{ + if (remountHook.get().empty()) { + warn("'%s' needs remounting, set remount-hook to do this automatically", realStoreDir.get()); + } else { + runProgram(remountHook, false, {realStoreDir}); + } + + _remountRequired = false; +} + + static RegisterStoreImplementation regLocalOverlayStore; } diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index d45f6bfc5..a26392523 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -118,12 +118,16 @@ private: void queryRealisationUncached(const DrvOutput&, Callback> callback) noexcept override; + void collectGarbage(const GCOptions & options, GCResults & results) override; + void deleteStorePath(const Path & path, uint64_t & bytesFreed) override; void optimiseStore() override; Path toRealPathForHardLink(const StorePath & storePath) override; + void remountIfNecessary(); + bool _remountRequired = false; }; diff --git a/tests/overlay-local-store/delete-inner.sh b/tests/overlay-local-store/delete-inner.sh index 051b59d5f..f3878f657 100644 --- a/tests/overlay-local-store/delete-inner.sh +++ b/tests/overlay-local-store/delete-inner.sh @@ -27,8 +27,7 @@ lowerInode=$(stat -c %i "$storeA/$lowerPath") [[ "$upperInode" != "$lowerInode" ]] # Now delete file via the overlay store -nix-store --store "$storeB" --delete "$upperPath" -remountOverlayfs +nix-store --store "$storeB&remount-hook=$PWD/remount.sh" --delete "$upperPath" # Check there is no longer a file in upper layer expect 1 stat "$storeBTop/${upperPath##/nix/store/}" diff --git a/tests/overlay-local-store/remount.sh b/tests/overlay-local-store/remount.sh new file mode 100755 index 000000000..ee91c6b43 --- /dev/null +++ b/tests/overlay-local-store/remount.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +mount -o remount "$1" From 6da05c0a11365c8cd138c15600ec966e9c2b5940 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Wed, 26 Jul 2023 14:29:36 +0100 Subject: [PATCH 095/133] Rename test to delete-duplicate. --- .../{delete-inner.sh => delete-duplicate-inner.sh} | 4 ++-- tests/overlay-local-store/{delete.sh => delete-duplicate.sh} | 2 +- tests/overlay-local-store/local.mk | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) rename tests/overlay-local-store/{delete-inner.sh => delete-duplicate-inner.sh} (87%) rename tests/overlay-local-store/{delete.sh => delete-duplicate.sh} (55%) diff --git a/tests/overlay-local-store/delete-inner.sh b/tests/overlay-local-store/delete-duplicate-inner.sh similarity index 87% rename from tests/overlay-local-store/delete-inner.sh rename to tests/overlay-local-store/delete-duplicate-inner.sh index f3878f657..6053451d8 100644 --- a/tests/overlay-local-store/delete-inner.sh +++ b/tests/overlay-local-store/delete-duplicate-inner.sh @@ -17,8 +17,8 @@ initLowerStore mountOverlayfs # Add to overlay before lower to ensure file is duplicated -upperPath=$(nix-store --store "$storeB" --add delete.sh) -lowerPath=$(nix-store --store "$storeA" --add delete.sh) +upperPath=$(nix-store --store "$storeB" --add delete-duplicate.sh) +lowerPath=$(nix-store --store "$storeA" --add delete-duplicate.sh) [[ "$upperPath" = "$lowerPath" ]] # Check there really are two files with different inodes diff --git a/tests/overlay-local-store/delete.sh b/tests/overlay-local-store/delete-duplicate.sh similarity index 55% rename from tests/overlay-local-store/delete.sh rename to tests/overlay-local-store/delete-duplicate.sh index 79d67da71..0c0b1a3b2 100644 --- a/tests/overlay-local-store/delete.sh +++ b/tests/overlay-local-store/delete-duplicate.sh @@ -2,4 +2,4 @@ source common.sh requireEnvironment setupConfig -execUnshare ./delete-inner.sh +execUnshare ./delete-duplicate-inner.sh diff --git a/tests/overlay-local-store/local.mk b/tests/overlay-local-store/local.mk index 9233462a3..7de46ab02 100644 --- a/tests/overlay-local-store/local.mk +++ b/tests/overlay-local-store/local.mk @@ -7,6 +7,6 @@ overlay-local-store-tests := \ $(d)/gc.sh \ $(d)/verify.sh \ $(d)/optimise.sh \ - $(d)/delete.sh + $(d)/delete-duplicate.sh install-tests-groups += overlay-local-store From 5744a500d66214f6d833100924a2f7362b7b82a7 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Wed, 26 Jul 2023 14:33:47 +0100 Subject: [PATCH 096/133] Use debug instead of writing directly to stderr. --- src/libstore/local-overlay-store.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 76c3adf5f..dfd2f0d34 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -196,9 +196,9 @@ void LocalOverlayStore::deleteStorePath(const Path & path, uint64_t & bytesFreed auto upperPath = toUpperPath(storePath); if (pathExists(upperPath)) { - std::cerr << " upper exists" << std::endl; + debug("upper exists: %s", path); if (lowerStore->isValidPath(storePath)) { - std::cerr << " lower exists" << std::endl; + debug("lower exists: %s", storePath.to_string()); // Path also exists in lower store. // We must delete via upper layer to avoid creating a whiteout. deletePath(upperPath, bytesFreed); From ca1a108dad93f00f18929c02bf29bbe5347035ac Mon Sep 17 00:00:00 2001 From: Ben Radford <104896700+benradf@users.noreply.github.com> Date: Wed, 26 Jul 2023 14:37:35 +0100 Subject: [PATCH 097/133] Update tests/overlay-local-store/remount.sh Co-authored-by: John Ericson --- tests/overlay-local-store/remount.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/overlay-local-store/remount.sh b/tests/overlay-local-store/remount.sh index ee91c6b43..0b06debb5 100755 --- a/tests/overlay-local-store/remount.sh +++ b/tests/overlay-local-store/remount.sh @@ -1,2 +1,2 @@ -#!/usr/bin/env bash +#!/bin/sh mount -o remount "$1" From 3a9fe1a085edc8db3e3cd35531829ddab51bcd81 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Wed, 26 Jul 2023 14:44:14 +0100 Subject: [PATCH 098/133] Made remountRequired atomic to avoid concurrency issues. --- src/libstore/local-overlay-store.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index a26392523..66a43d84a 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -128,7 +128,7 @@ private: void remountIfNecessary(); - bool _remountRequired = false; + std::atomic_bool _remountRequired = false; }; } From c2d54496a0c8611a7008181e2dc4fc57d520752f Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Wed, 26 Jul 2023 14:47:44 +0100 Subject: [PATCH 099/133] Forgot to check flag and early out. --- src/libstore/local-overlay-store.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index dfd2f0d34..04a1717cd 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -247,6 +247,8 @@ Path LocalOverlayStore::toRealPathForHardLink(const StorePath & path) void LocalOverlayStore::remountIfNecessary() { + if (!_remountRequired) return; + if (remountHook.get().empty()) { warn("'%s' needs remounting, set remount-hook to do this automatically", realStoreDir.get()); } else { From 50ce8d15eb5fed11080a3b94991c55e880c22252 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Fri, 28 Jul 2023 10:11:45 +0100 Subject: [PATCH 100/133] Preparatory refactor of LocalStore::verifyStore. --- src/libstore/local-store.cc | 51 +++++++++++++++++++++---------------- src/libstore/local-store.hh | 6 +++-- 2 files changed, 33 insertions(+), 24 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 2c18867f5..6c3584b49 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1498,24 +1498,14 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) { printInfo("reading the Nix store..."); - bool errors = false; - /* Acquire the global GC lock to get a consistent snapshot of existing and valid paths. */ auto fdGCLock = openGCLock(); FdLock gcLock(fdGCLock.get(), ltRead, true, "waiting for the big garbage collector lock..."); - StringSet store; - for (auto & i : readDirectory(realStoreDir)) store.insert(i.name); - - /* Check whether all valid paths actually exist. */ - printInfo("checking path existence..."); - StorePathSet validPaths; - PathSet done; - for (auto & i : queryAllValidPaths()) - verifyPath(printStorePath(i), store, done, validPaths, repair, errors); + bool errors = verifyAllValidPaths(repair, validPaths); /* Optionally, check the content hashes (slow). */ if (checkContents) { @@ -1601,28 +1591,45 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) } -void LocalStore::verifyPath(const Path & pathS, const StringSet & store, - PathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors) +bool LocalStore::verifyAllValidPaths(RepairFlag repair, StorePathSet & validPaths) +{ + StringSet store; + for (auto & i : readDirectory(realStoreDir)) store.insert(i.name); + + /* Check whether all valid paths actually exist. */ + printInfo("checking path existence..."); + + StorePathSet done; + bool errors = false; + + auto existsInStoreDir = [&](const StorePath & storePath) { + return store.count(std::string(storePath.to_string())); + }; + + for (auto & i : queryAllValidPaths()) + verifyPath(i, existsInStoreDir, done, validPaths, repair, errors); + + return errors; +} + + +void LocalStore::verifyPath(const StorePath & path, std::function existsInStoreDir, + StorePathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors) { checkInterrupt(); - if (!done.insert(pathS).second) return; + if (!done.insert(path).second) return; - if (!isStorePath(pathS)) { - printError("path '%s' is not in the Nix store", pathS); - return; - } + auto pathS = printStorePath(path); - auto path = parseStorePath(pathS); - - if (!store.count(std::string(path.to_string()))) { + if (!existsInStoreDir(path)) { /* Check any referrers first. If we can invalidate them first, then we can invalidate this path as well. */ bool canInvalidate = true; StorePathSet referrers; queryReferrers(path, referrers); for (auto & i : referrers) if (i != path) { - verifyPath(printStorePath(i), store, done, validPaths, repair, errors); + verifyPath(i, existsInStoreDir, done, validPaths, repair, errors); if (validPaths.count(i)) canInvalidate = false; } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 514ac256b..f9db43517 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -265,6 +265,8 @@ public: bool verifyStore(bool checkContents, RepairFlag repair) override; + virtual bool verifyAllValidPaths(RepairFlag repair, StorePathSet & validPaths); + /** * Register the validity of a path, i.e., that `path` exists, that * the paths referenced by it exists, and in the case of an output @@ -333,8 +335,8 @@ private: */ void invalidatePathChecked(const StorePath & path); - void verifyPath(const Path & path, const StringSet & store, - PathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors); + void verifyPath(const StorePath & path, std::function existsInStoreDir, + StorePathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors); std::shared_ptr queryPathInfoInternal(State & state, const StorePath & path); From 6a8de4c9dcd5c329f6488818517995647c577c87 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Fri, 28 Jul 2023 10:59:16 +0100 Subject: [PATCH 101/133] Avoid enumerating entire overlay store dir upfront. As an optimisation for LocalStore, we read all the store directory entries into a set. Checking for membership of this set is much faster than a stat syscall. However for LocalOverlayStore, the lower store directory is expected to contain a vast number of entries and reading them all can take a very long time. So instead of enumerating them all upfront, we call pathExists as needed. This means making stat syscalls for each store path, but the upper layer is expected to be relatively small compared to the lower store so that should be okay. --- src/libstore/local-overlay-store.cc | 16 ++++++++++++++++ src/libstore/local-overlay-store.hh | 2 ++ src/libstore/local-store.hh | 8 +++++--- tests/overlay-local-store/verify-inner.sh | 3 +++ 4 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 30008de67..4bfad6d32 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -243,6 +243,22 @@ void LocalOverlayStore::optimiseStore() } +bool LocalOverlayStore::verifyAllValidPaths(RepairFlag repair, StorePathSet & validPaths) +{ + StorePathSet done; + bool errors = false; + + auto existsInStoreDir = [&](const StorePath & storePath) { + return pathExists(realStoreDir.get() + "/" + storePath.to_string()); + }; + + for (auto & i : queryAllValidPaths()) + verifyPath(i, existsInStoreDir, done, validPaths, repair, errors); + + return errors; +} + + Path LocalOverlayStore::toRealPathForHardLink(const StorePath & path) { return lowerStore->isValidPath(path) diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 1c1d54a8f..21af396a6 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -124,6 +124,8 @@ private: void optimiseStore() override; + bool verifyAllValidPaths(RepairFlag repair, StorePathSet & validPaths) override; + /** * For lower-store paths, we used the lower store location. This avoids the * wasteful "copying up" that would otherwise happen. diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index f9db43517..322d27932 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -312,6 +312,11 @@ public: std::optional getVersion() override; +protected: + + void verifyPath(const StorePath & path, std::function existsInStoreDir, + StorePathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors); + private: /** @@ -335,9 +340,6 @@ private: */ void invalidatePathChecked(const StorePath & path); - void verifyPath(const StorePath & path, std::function existsInStoreDir, - StorePathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors); - std::shared_ptr queryPathInfoInternal(State & state, const StorePath & path); void updatePathInfo(State & state, const ValidPathInfo & info); diff --git a/tests/overlay-local-store/verify-inner.sh b/tests/overlay-local-store/verify-inner.sh index de40c3d05..64d3b63c1 100755 --- a/tests/overlay-local-store/verify-inner.sh +++ b/tests/overlay-local-store/verify-inner.sh @@ -50,6 +50,9 @@ find "$storeA" -name "*-dummy" -exec truncate -s 0 {} \; # Also truncate the file that only exists in lower store truncate -s 0 "$storeA/$lowerOnlyPath" +# Ensure overlayfs is synchronised +remountOverlayfs + ## Now test that verify and repair work as expected From 1255866e16bbbf2ac93ade99ae5ef2f68f64d8c6 Mon Sep 17 00:00:00 2001 From: Ben Radford <104896700+benradf@users.noreply.github.com> Date: Fri, 28 Jul 2023 12:23:42 +0100 Subject: [PATCH 102/133] Update src/libstore/local-overlay-store.hh Co-authored-by: John Ericson --- src/libstore/local-overlay-store.hh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 21af396a6..b91f68eea 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -124,6 +124,13 @@ private: void optimiseStore() override; + /** + * Check all paths registered in the upper DB. + * + * Note that this includes store objects that reside in either overlayfs layer; just enumerating the contents of the upper layer would skip them. + * + * We don't verify the contents of both layers on the assumption that the lower layer is far bigger, and also the observation that anything not in the upper db the overlayfs doesn't yet care about. + */ bool verifyAllValidPaths(RepairFlag repair, StorePathSet & validPaths) override; /** From c409a753dbc6318f5454f058dbf55786327035b7 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 1 Aug 2023 11:11:16 +0100 Subject: [PATCH 103/133] Fix new lines in comment. --- src/libstore/local-overlay-store.hh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index b91f68eea..874f3b779 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -127,9 +127,11 @@ private: /** * Check all paths registered in the upper DB. * - * Note that this includes store objects that reside in either overlayfs layer; just enumerating the contents of the upper layer would skip them. + * Note that this includes store objects that reside in either overlayfs layer; + * just enumerating the contents of the upper layer would skip them. * - * We don't verify the contents of both layers on the assumption that the lower layer is far bigger, and also the observation that anything not in the upper db the overlayfs doesn't yet care about. + * We don't verify the contents of both layers on the assumption that the lower layer is far bigger, + * and also the observation that anything not in the upper db the overlayfs doesn't yet care about. */ bool verifyAllValidPaths(RepairFlag repair, StorePathSet & validPaths) override; From c712369ec597c706f985f2f998215f5ab913d61a Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 1 Aug 2023 11:55:55 +0100 Subject: [PATCH 104/133] Document remount-hook store parameter. --- src/libstore/local-overlay-store.hh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 874f3b779..c0fa0ffa7 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -43,9 +43,15 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig const PathSetting remountHook{(StoreConfig*) this, "", "remount-hook", R"( - Script or program to run when overlay filesystem needs remounting. + Script or other executable to run when overlay filesystem needs remounting. - TODO: Document this in more detail. + This is occasionally necessary when deleting a store path that exists in both upper and lower layers. + In such a situation, bypassing OverlayFS and deleting the path in the upper layer directly + is the only way to perform the deletion without creating a "whiteout". + However this causes the OverlayFS kernel data structures to get out-of-sync, + and can lead to 'stale file handle' errors; remounting solves the problem. + + The store directory is passed as an argument to the invoked executable. )"}; const std::string name() override { return "Experimental Local Overlay Store"; } From 19164cf727030dc75c11533f11dc84645658b7c3 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 1 Aug 2023 12:49:46 +0100 Subject: [PATCH 105/133] Test that remounting fixes 'stale file handle' errors. --- tests/overlay-local-store/local.mk | 3 +- .../stale-file-handle-inner.sh | 47 +++++++++++++++++++ .../overlay-local-store/stale-file-handle.sh | 5 ++ 3 files changed, 54 insertions(+), 1 deletion(-) create mode 100755 tests/overlay-local-store/stale-file-handle-inner.sh create mode 100755 tests/overlay-local-store/stale-file-handle.sh diff --git a/tests/overlay-local-store/local.mk b/tests/overlay-local-store/local.mk index 19ae6a3d0..34056683d 100644 --- a/tests/overlay-local-store/local.mk +++ b/tests/overlay-local-store/local.mk @@ -8,6 +8,7 @@ overlay-local-store-tests := \ $(d)/delete-duplicate.sh \ $(d)/gc.sh \ $(d)/verify.sh \ - $(d)/optimise.sh + $(d)/optimise.sh \ + $(d)/stale-file-handle.sh install-tests-groups += overlay-local-store diff --git a/tests/overlay-local-store/stale-file-handle-inner.sh b/tests/overlay-local-store/stale-file-handle-inner.sh new file mode 100755 index 000000000..86e39fa40 --- /dev/null +++ b/tests/overlay-local-store/stale-file-handle-inner.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +set -x + +source common.sh + +# Avoid store dir being inside sandbox build-dir +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +storeDirs + +initLowerStore + +mountOverlayfs + +buildInStore () { + nix-build --store "$1" ../hermetic.nix --arg busybox "$busybox" --arg seed 1 --no-out-link +} + +triggerStaleFileHandle () { + # Arrange it so there are duplicate paths + nix-store --store "$storeA" --gc # Clear lower store + buildInStore "$storeB" # Build into upper layer first + buildInStore "$storeA" # Then build in lower store + + # Duplicate paths mean GC will have to delete via upper layer + nix-store --store "$storeB" --gc + + # Clear lower store again to force building in upper layer + nix-store --store "$storeA" --gc + + # Now attempting to build in upper layer will fail + buildInStore "$storeB" +} + +# Without remounting, we should encounter errors +expectStderr 1 triggerStaleFileHandle | grepQuiet 'Stale file handle' + +# Configure remount-hook and reset OverlayFS +storeB="$storeB&remount-hook=$PWD/remount.sh" +remountOverlayfs + +# Now it should succeed +triggerStaleFileHandle diff --git a/tests/overlay-local-store/stale-file-handle.sh b/tests/overlay-local-store/stale-file-handle.sh new file mode 100755 index 000000000..5e75628ca --- /dev/null +++ b/tests/overlay-local-store/stale-file-handle.sh @@ -0,0 +1,5 @@ +source common.sh + +requireEnvironment +setupConfig +execUnshare ./stale-file-handle-inner.sh From 6b297e5895e62d91963be49e5d301604d0b8fd09 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 2 Aug 2023 14:38:22 -0400 Subject: [PATCH 106/133] Make `verifyAllValidPaths` more functional return map rather than mutate one passed in by reference --- src/libstore/local-overlay-store.cc | 8 +++++--- src/libstore/local-overlay-store.hh | 4 ++-- src/libstore/local-store.cc | 12 ++++++------ src/libstore/local-store.hh | 7 ++++++- 4 files changed, 19 insertions(+), 12 deletions(-) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 4bfad6d32..4ba98a02d 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -243,19 +243,21 @@ void LocalOverlayStore::optimiseStore() } -bool LocalOverlayStore::verifyAllValidPaths(RepairFlag repair, StorePathSet & validPaths) +std::pair LocalOverlayStore::verifyAllValidPaths(RepairFlag repair) { StorePathSet done; - bool errors = false; auto existsInStoreDir = [&](const StorePath & storePath) { return pathExists(realStoreDir.get() + "/" + storePath.to_string()); }; + bool errors = false; + StorePathSet validPaths; + for (auto & i : queryAllValidPaths()) verifyPath(i, existsInStoreDir, done, validPaths, repair, errors); - return errors; + return { errors, validPaths }; } diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index c0fa0ffa7..40ee75bca 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -135,11 +135,11 @@ private: * * Note that this includes store objects that reside in either overlayfs layer; * just enumerating the contents of the upper layer would skip them. - * + * * We don't verify the contents of both layers on the assumption that the lower layer is far bigger, * and also the observation that anything not in the upper db the overlayfs doesn't yet care about. */ - bool verifyAllValidPaths(RepairFlag repair, StorePathSet & validPaths) override; + std::pair verifyAllValidPaths(RepairFlag repair) override; /** * For lower-store paths, we used the lower store location. This avoids the diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 2e662ad66..42347d80a 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1503,9 +1503,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) auto fdGCLock = openGCLock(); FdLock gcLock(fdGCLock.get(), ltRead, true, "waiting for the big garbage collector lock..."); - StorePathSet validPaths; - - bool errors = verifyAllValidPaths(repair, validPaths); + auto [errors, validPaths] = verifyAllValidPaths(repair); /* Optionally, check the content hashes (slow). */ if (checkContents) { @@ -1591,7 +1589,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) } -bool LocalStore::verifyAllValidPaths(RepairFlag repair, StorePathSet & validPaths) +std::pair LocalStore::verifyAllValidPaths(RepairFlag repair) { StorePathSet storePathsInStoreDir; /* Why aren't we using `queryAllValidPaths`? Because that would @@ -1613,16 +1611,18 @@ bool LocalStore::verifyAllValidPaths(RepairFlag repair, StorePathSet & validPath printInfo("checking path existence..."); StorePathSet done; - bool errors = false; auto existsInStoreDir = [&](const StorePath & storePath) { return storePathsInStoreDir.count(storePath); }; + bool errors = false; + StorePathSet validPaths; + for (auto & i : queryAllValidPaths()) verifyPath(i, existsInStoreDir, done, validPaths, repair, errors); - return errors; + return { errors, validPaths }; } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 322d27932..88549eea5 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -265,7 +265,12 @@ public: bool verifyStore(bool checkContents, RepairFlag repair) override; - virtual bool verifyAllValidPaths(RepairFlag repair, StorePathSet & validPaths); + /** + * @return A pair of whether any errors were encountered, and a set of + * (so-far) valid paths. The store objects pointed to by those paths are + * suitable for further validation checking. + */ + virtual std::pair verifyAllValidPaths(RepairFlag repair); /** * Register the validity of a path, i.e., that `path` exists, that From 4b9a6218125c23e95b7397069f6fc2796fb70fd2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 2 Aug 2023 20:30:42 -0400 Subject: [PATCH 107/133] Guard the local overlay store behind an experimental feature --- src/libstore/local-overlay-store.hh | 5 +++++ src/libstore/store-api.cc | 5 ++++- src/libutil/experimental-features.cc | 9 ++++++++- src/libutil/experimental-features.hh | 1 + tests/overlay-local-store/common.sh | 2 +- 5 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 40ee75bca..44f0c77c8 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -56,6 +56,11 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig const std::string name() override { return "Experimental Local Overlay Store"; } + std::optional experimentalFeature() const override + { + return ExperimentalFeature::LocalOverlayStore; + } + std::string doc() override { return diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 1fbf8995b..1863a0876 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1444,7 +1444,9 @@ std::shared_ptr openFromNonUri(const std::string & uri, const Store::Para } else if (uri == "local") { return std::make_shared(params); } else if (uri == "local-overlay") { - return std::make_shared(params); + auto store = std::make_shared(params); + experimentalFeatureSettings.require(store->experimentalFeature()); + return store; } else if (isNonUriPath(uri)) { Store::Params params2 = params; params2["root"] = absPath(uri); @@ -1512,6 +1514,7 @@ ref openStore(const std::string & uri_, params.insert(uriParams.begin(), uriParams.end()); if (auto store = openFromNonUri(uri, params)) { + experimentalFeatureSettings.require(store->experimentalFeature()); store->warnUnknownSettings(); return ref(store); } diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 7c4112d32..422d18522 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -12,7 +12,7 @@ struct ExperimentalFeatureDetails std::string_view description; }; -constexpr std::array xpFeatureDetails = {{ +constexpr std::array xpFeatureDetails = {{ { .tag = Xp::CaDerivations, .name = "ca-derivations", @@ -228,6 +228,13 @@ constexpr std::array xpFeatureDetails = {{ Allow the use of the `read-only` parameter in [local store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-store) URIs. )", }, + { + .tag = Xp::LocalOverlayStore, + .name = "local-overlay-store", + .description = R"( + Allow the use of [local overlay store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-overlay-store). + )", + }, }}; static_assert( diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index faf2e9398..c044a858e 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -32,6 +32,7 @@ enum struct ExperimentalFeature DynamicDerivations, ParseTomlTimestamps, ReadOnlyLocalStore, + LocalOverlayStore, }; /** diff --git a/tests/overlay-local-store/common.sh b/tests/overlay-local-store/common.sh index 6bb5bc391..2d614b140 100644 --- a/tests/overlay-local-store/common.sh +++ b/tests/overlay-local-store/common.sh @@ -16,7 +16,7 @@ setupConfig () { addConfig "build-users-group = " } - +enableFeatures "local-overlay-store" storeDirs () { # Attempt to create store dirs on tmpfs volume. From 2556c4d7531c1303ec3e5db5d5dd1b3a9b91a3b4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 2 Aug 2023 20:32:45 -0400 Subject: [PATCH 108/133] Rename test group `overlay-local-store` -> `local-overlay-store` Makes it match the store name (`local-overlay`) and experimental feature name (`local-overlay-store`)._ --- Makefile | 2 +- .../add-lower-inner.sh | 0 .../{overlay-local-store => local-overlay-store}/add-lower.sh | 0 .../{overlay-local-store => local-overlay-store}/bad-uris.sh | 0 .../build-inner.sh | 0 tests/{overlay-local-store => local-overlay-store}/build.sh | 0 .../check-post-init-inner.sh | 0 .../check-post-init.sh | 0 tests/{overlay-local-store => local-overlay-store}/common.sh | 0 .../delete-duplicate-inner.sh | 0 .../delete-duplicate.sh | 0 .../delete-refs-inner.sh | 0 .../delete-refs.sh | 0 .../{overlay-local-store => local-overlay-store}/gc-inner.sh | 0 tests/{overlay-local-store => local-overlay-store}/gc.sh | 0 tests/{overlay-local-store => local-overlay-store}/local.mk | 4 ++-- .../optimise-inner.sh | 0 .../{overlay-local-store => local-overlay-store}/optimise.sh | 0 .../redundant-add-inner.sh | 0 .../redundant-add.sh | 0 tests/{overlay-local-store => local-overlay-store}/remount.sh | 0 .../stale-file-handle-inner.sh | 0 .../stale-file-handle.sh | 0 .../verify-inner.sh | 0 tests/{overlay-local-store => local-overlay-store}/verify.sh | 0 25 files changed, 3 insertions(+), 3 deletions(-) rename tests/{overlay-local-store => local-overlay-store}/add-lower-inner.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/add-lower.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/bad-uris.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/build-inner.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/build.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/check-post-init-inner.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/check-post-init.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/common.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/delete-duplicate-inner.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/delete-duplicate.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/delete-refs-inner.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/delete-refs.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/gc-inner.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/gc.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/local.mk (77%) rename tests/{overlay-local-store => local-overlay-store}/optimise-inner.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/optimise.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/redundant-add-inner.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/redundant-add.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/remount.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/stale-file-handle-inner.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/stale-file-handle.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/verify-inner.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/verify.sh (100%) diff --git a/Makefile b/Makefile index 715830811..ea15b03db 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ makefiles += \ tests/local.mk \ tests/ca/local.mk \ tests/dyn-drv/local.mk \ - tests/overlay-local-store/local.mk \ + tests/local-overlay-store/local.mk \ tests/test-libstoreconsumer/local.mk \ tests/plugins/local.mk else diff --git a/tests/overlay-local-store/add-lower-inner.sh b/tests/local-overlay-store/add-lower-inner.sh similarity index 100% rename from tests/overlay-local-store/add-lower-inner.sh rename to tests/local-overlay-store/add-lower-inner.sh diff --git a/tests/overlay-local-store/add-lower.sh b/tests/local-overlay-store/add-lower.sh similarity index 100% rename from tests/overlay-local-store/add-lower.sh rename to tests/local-overlay-store/add-lower.sh diff --git a/tests/overlay-local-store/bad-uris.sh b/tests/local-overlay-store/bad-uris.sh similarity index 100% rename from tests/overlay-local-store/bad-uris.sh rename to tests/local-overlay-store/bad-uris.sh diff --git a/tests/overlay-local-store/build-inner.sh b/tests/local-overlay-store/build-inner.sh similarity index 100% rename from tests/overlay-local-store/build-inner.sh rename to tests/local-overlay-store/build-inner.sh diff --git a/tests/overlay-local-store/build.sh b/tests/local-overlay-store/build.sh similarity index 100% rename from tests/overlay-local-store/build.sh rename to tests/local-overlay-store/build.sh diff --git a/tests/overlay-local-store/check-post-init-inner.sh b/tests/local-overlay-store/check-post-init-inner.sh similarity index 100% rename from tests/overlay-local-store/check-post-init-inner.sh rename to tests/local-overlay-store/check-post-init-inner.sh diff --git a/tests/overlay-local-store/check-post-init.sh b/tests/local-overlay-store/check-post-init.sh similarity index 100% rename from tests/overlay-local-store/check-post-init.sh rename to tests/local-overlay-store/check-post-init.sh diff --git a/tests/overlay-local-store/common.sh b/tests/local-overlay-store/common.sh similarity index 100% rename from tests/overlay-local-store/common.sh rename to tests/local-overlay-store/common.sh diff --git a/tests/overlay-local-store/delete-duplicate-inner.sh b/tests/local-overlay-store/delete-duplicate-inner.sh similarity index 100% rename from tests/overlay-local-store/delete-duplicate-inner.sh rename to tests/local-overlay-store/delete-duplicate-inner.sh diff --git a/tests/overlay-local-store/delete-duplicate.sh b/tests/local-overlay-store/delete-duplicate.sh similarity index 100% rename from tests/overlay-local-store/delete-duplicate.sh rename to tests/local-overlay-store/delete-duplicate.sh diff --git a/tests/overlay-local-store/delete-refs-inner.sh b/tests/local-overlay-store/delete-refs-inner.sh similarity index 100% rename from tests/overlay-local-store/delete-refs-inner.sh rename to tests/local-overlay-store/delete-refs-inner.sh diff --git a/tests/overlay-local-store/delete-refs.sh b/tests/local-overlay-store/delete-refs.sh similarity index 100% rename from tests/overlay-local-store/delete-refs.sh rename to tests/local-overlay-store/delete-refs.sh diff --git a/tests/overlay-local-store/gc-inner.sh b/tests/local-overlay-store/gc-inner.sh similarity index 100% rename from tests/overlay-local-store/gc-inner.sh rename to tests/local-overlay-store/gc-inner.sh diff --git a/tests/overlay-local-store/gc.sh b/tests/local-overlay-store/gc.sh similarity index 100% rename from tests/overlay-local-store/gc.sh rename to tests/local-overlay-store/gc.sh diff --git a/tests/overlay-local-store/local.mk b/tests/local-overlay-store/local.mk similarity index 77% rename from tests/overlay-local-store/local.mk rename to tests/local-overlay-store/local.mk index 34056683d..6348a4423 100644 --- a/tests/overlay-local-store/local.mk +++ b/tests/local-overlay-store/local.mk @@ -1,4 +1,4 @@ -overlay-local-store-tests := \ +local-overlay-store-tests := \ $(d)/check-post-init.sh \ $(d)/redundant-add.sh \ $(d)/build.sh \ @@ -11,4 +11,4 @@ overlay-local-store-tests := \ $(d)/optimise.sh \ $(d)/stale-file-handle.sh -install-tests-groups += overlay-local-store +install-tests-groups += local-overlay-store diff --git a/tests/overlay-local-store/optimise-inner.sh b/tests/local-overlay-store/optimise-inner.sh similarity index 100% rename from tests/overlay-local-store/optimise-inner.sh rename to tests/local-overlay-store/optimise-inner.sh diff --git a/tests/overlay-local-store/optimise.sh b/tests/local-overlay-store/optimise.sh similarity index 100% rename from tests/overlay-local-store/optimise.sh rename to tests/local-overlay-store/optimise.sh diff --git a/tests/overlay-local-store/redundant-add-inner.sh b/tests/local-overlay-store/redundant-add-inner.sh similarity index 100% rename from tests/overlay-local-store/redundant-add-inner.sh rename to tests/local-overlay-store/redundant-add-inner.sh diff --git a/tests/overlay-local-store/redundant-add.sh b/tests/local-overlay-store/redundant-add.sh similarity index 100% rename from tests/overlay-local-store/redundant-add.sh rename to tests/local-overlay-store/redundant-add.sh diff --git a/tests/overlay-local-store/remount.sh b/tests/local-overlay-store/remount.sh similarity index 100% rename from tests/overlay-local-store/remount.sh rename to tests/local-overlay-store/remount.sh diff --git a/tests/overlay-local-store/stale-file-handle-inner.sh b/tests/local-overlay-store/stale-file-handle-inner.sh similarity index 100% rename from tests/overlay-local-store/stale-file-handle-inner.sh rename to tests/local-overlay-store/stale-file-handle-inner.sh diff --git a/tests/overlay-local-store/stale-file-handle.sh b/tests/local-overlay-store/stale-file-handle.sh similarity index 100% rename from tests/overlay-local-store/stale-file-handle.sh rename to tests/local-overlay-store/stale-file-handle.sh diff --git a/tests/overlay-local-store/verify-inner.sh b/tests/local-overlay-store/verify-inner.sh similarity index 100% rename from tests/overlay-local-store/verify-inner.sh rename to tests/local-overlay-store/verify-inner.sh diff --git a/tests/overlay-local-store/verify.sh b/tests/local-overlay-store/verify.sh similarity index 100% rename from tests/overlay-local-store/verify.sh rename to tests/local-overlay-store/verify.sh From 4f5b01f5cd83f1ce0d36ea8f4be8b06e9526cbe3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 3 Aug 2023 11:59:04 -0400 Subject: [PATCH 109/133] Start to document the local-overlay store --- src/libstore/local-overlay-store.cc | 7 +++++++ src/libstore/local-overlay-store.hh | 9 +-------- src/libstore/local-overlay-store.md | 7 +++++++ 3 files changed, 15 insertions(+), 8 deletions(-) create mode 100644 src/libstore/local-overlay-store.md diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 4ba98a02d..73902e9ba 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -5,6 +5,13 @@ namespace nix { +std::string LocalOverlayStoreConfig::doc() +{ + return + #include "local-overlay-store.md" + ; +} + Path LocalOverlayStoreConfig::toUpperPath(const StorePath & path) { return upperLayer + "/" + path.to_string(); } diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 44f0c77c8..e3fda2f93 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -61,14 +61,7 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig return ExperimentalFeature::LocalOverlayStore; } - std::string doc() override - { - return - "" - // FIXME write docs - //#include "local-overlay-store.md" - ; - } + std::string doc() override; /** * Given a store path, get its location (if it is exists) in the diff --git a/src/libstore/local-overlay-store.md b/src/libstore/local-overlay-store.md new file mode 100644 index 000000000..0b8a1786b --- /dev/null +++ b/src/libstore/local-overlay-store.md @@ -0,0 +1,7 @@ +R"( + +**Store URL format**: `local-overlay` + +This store type is a variation of the [local store](#local-store) designed to leverage overlayfs. + +)" From 4d99e407fd8c2a257304a622b93abd69149413fc Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 3 Aug 2023 12:12:21 -0400 Subject: [PATCH 110/133] Remove FIXME on why something doesn't work I now know it is due to https://github.com/llvm/llvm-project/issues/64108. The workaround is just fine and already in use in this codebase. --- src/libstore/local-overlay-store.hh | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index e3fda2f93..12e49e6d2 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -7,9 +7,6 @@ namespace nix { */ struct LocalOverlayStoreConfig : virtual LocalStoreConfig { - // FIXME why doesn't this work? - // using LocalStoreConfig::LocalStoreConfig; - LocalOverlayStoreConfig(const StringMap & params) : StoreConfig(params) , LocalFSStoreConfig(params) From 7ad16c9d1283c14df410a933eff560da258c11bf Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 3 Aug 2023 16:10:32 -0400 Subject: [PATCH 111/133] Add some docs for the local overlay store --- src/libstore/local-overlay-store.md | 49 ++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/src/libstore/local-overlay-store.md b/src/libstore/local-overlay-store.md index 0b8a1786b..a0b9c3adf 100644 --- a/src/libstore/local-overlay-store.md +++ b/src/libstore/local-overlay-store.md @@ -2,6 +2,53 @@ R"( **Store URL format**: `local-overlay` -This store type is a variation of the [local store](#local-store) designed to leverage overlayfs. +This store type is a variation of the [local store] designed to leverage Linux's [Overlay Filesystem](https://docs.kernel.org/filesystems/overlayfs.html) (OverlayFS for short). +Just as OverlayFS combines a lower and upper filesystem by treating the upper one as a patch against the lower, the local overlay store combines a lower store with an upper almost [local store]. +("almost" because while the upper fileystems for OverlayFS is valid on its own, the upper almost-store is not a valid local store on its own because some references will dangle.) + +### Parts of a local overlay store + +The parts of a local overlay store are as follows: + +- Lower store: + + This is any store implementation that includes a store directory as part of the native operating system filesystem. + For example, this could be a [local store], [local daemon store], or even another local overlay store. + + Specified with the `lower-store` setting. + + - Lower store directory. + This is the directory used/exposed by the lower store. + + Specified with `lower-store.real` setting. + + - Lower abstract read-only metadata source. + This is abstract, just some way to read the metadata of lower store [store objects](@docroot@/glossary.md#gloss-store-object). + For example it could be a SQLite database (for the [local store]), or a socket connection (for the [local daemon store]). + +- Upper almost-store: + + This is a [local store] that by itself would appear corrupted. + But combined with everything else as part of an overlay local store, it is valid. + + - Upper layer directory. + This contains additional [store objects] + (or, strictly speaking, their [file system objects](#gloss-file-system-object)) + that the local overlay store will extend the lower store with. + + Specified with `upper-layer` setting. + + - Upper store directory + The lower store directory and upper layer directory are combined via OverlayFS to create this directory. + This contains all the store objects from each of the two directories. + + Specified with the `real` setting. + + - Upper SQLite database + This contains the metadata of all of the upper layer [store objects]: everything beyond their file system objects, and also duplicate copies of some lower layer ones. + The duplication is so the metadata for the [closure](@docroot@/glossary.md#gloss-closure) of upper layer [store objects] can entirely be found in the upper layer. + This allows us to use the same SQL Schema as the [local store]'s SQLite database, as foreign keys in that schema enforce closure metadata to be self-contained in this way. + + Specified with the `state` setting, is always `${state}/db`. )" From d137002e941ffdaa46efdffbca9fdc6fbb108522 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 3 Aug 2023 17:28:18 -0400 Subject: [PATCH 112/133] Add API docs for all overridden local overlay methods These docs explain the implementation relative to the local store originals. The original declaration of virtual methods can still be consulted for proper interface-level documentation. --- src/libstore/local-overlay-store.hh | 64 +++++++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 3 deletions(-) diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 12e49e6d2..8ed7516db 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -68,7 +68,10 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig }; /** - * Variation of local store using overlayfs for the store dir. + * Variation of local store using OverlayFS for the store directory. + * + * Documentation on overridden methods states how they differ from their + * `LocalStore` counterparts. */ class LocalOverlayStore : public virtual LocalOverlayStoreConfig, public virtual LocalStore { @@ -99,30 +102,77 @@ public: } private: - // Overridden methods… - + /** + * First copy up any lower store realisation with the same key, so we + * merge rather than mask it. + */ void registerDrvOutput(const Realisation & info) override; + /** + * Check lower store if upper DB does not have. + */ void queryPathInfoUncached(const StorePath & path, Callback> callback) noexcept override; + /** + * Check lower store if upper DB does not have. + * + * In addition, copy up metadata for lower store objects (and their + * closure). (I.e. Optimistically cache in the upper DB.) + */ bool isValidPathUncached(const StorePath & path) override; + /** + * Check the lower store and upper DB. + */ void queryReferrers(const StorePath & path, StorePathSet & referrers) override; + /** + * Check the lower store and upper DB. + */ StorePathSet queryValidDerivers(const StorePath & path) override; + /** + * Check lower store if upper DB does not have. + */ std::optional queryPathFromHashPart(const std::string & hashPart) override; + /** + * First copy up any lower store realisation with the same key, so we + * merge rather than mask it. + */ void registerValidPaths(const ValidPathInfos & infos) override; + /** + * Check lower store if upper DB does not have. + */ void queryRealisationUncached(const DrvOutput&, Callback> callback) noexcept override; + /** + * Call `remountIfNecessary` after collecting garbage normally. + */ void collectGarbage(const GCOptions & options, GCResults & results) override; + /** + * Check which layers the store object exists in to try to avoid + * needing to remount. + */ void deleteStorePath(const Path & path, uint64_t & bytesFreed) override; + /** + * Deduplicate by removing store objects from the upper layer that + * are now in the lower layer. + * + * This implementation will not cause duplications, but addition of + * new store objects to the lower layer can instill induce them + * (there is no way to prevent that). This cleans up those + * duplications. + * + * @note We do not yet optomise the upper layer in the normal way + * (hardlink) yet. We would like to, but it requires more + * refactoring of existing code to support this sustainably. + */ void optimiseStore() override; /** @@ -147,8 +197,16 @@ private: */ void queryGCReferrers(const StorePath & path, StorePathSet & referrers) override; + /** + * Call the `remountHook` if we have done something such that the + * OverlayFS needed to be remounted. See that hook's user-facing + * documentation for further details. + */ void remountIfNecessary(); + /** + * State for `remountIfNecessary` + */ std::atomic_bool _remountRequired = false; }; From 6f0a95897cacf6edb8d753e19aedf2f4f1f12108 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 25 Oct 2023 14:20:40 -0400 Subject: [PATCH 113/133] Revert "Fix hard linking issue causing overlay fs copy-ups" This reverts commit 9ef0a9e8aa2e27991434c104ad73b1b95b241f08. Master now has a better solution. --- src/libstore/build/local-derivation-goal.cc | 7 +++---- src/libstore/local-fs-store.hh | 10 ---------- src/libstore/local-overlay-store.cc | 8 -------- src/libstore/local-overlay-store.hh | 6 ------ 4 files changed, 3 insertions(+), 28 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 9da3afffd..40a0385af 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -388,9 +388,8 @@ void LocalDerivationGoal::cleanupPostOutputsRegisteredModeNonCheck() #if __linux__ -static void linkOrCopy(LocalFSStore & store, const StorePath & from_, const Path & to) +static void linkOrCopy(const Path & from, const Path & to) { - auto from = store.toRealPathForHardLink(from_); if (link(from.c_str(), to.c_str()) == -1) { /* Hard-linking fails if we exceed the maximum link count on a file (e.g. 32000 of ext3), which is quite possible after a @@ -715,7 +714,7 @@ void LocalDerivationGoal::startBuilder() if (S_ISDIR(lstat(r).st_mode)) dirsInChroot.insert_or_assign(p, r); else - linkOrCopy(getLocalStore(), i, chrootRootDir + p); + linkOrCopy(r, chrootRootDir + p); } /* If we're repairing, checking or rebuilding part of a @@ -1600,7 +1599,7 @@ void LocalDerivationGoal::addDependency(const StorePath & path) throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path)); } else - linkOrCopy(getLocalStore(), path, target); + linkOrCopy(source, target); #else throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox", diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 19858f5c8..488109501 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -73,16 +73,6 @@ public: return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1); } - /** - * If the real path is hardlinked with something else, we might - * prefer to refer to the other path instead. This is the case with - * overlayfs, for example. - */ - virtual Path toRealPathForHardLink(const StorePath & storePath) - { - return Store::toRealPath(storePath); - } - std::optional getBuildLogExact(const StorePath & path) override; }; diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 73902e9ba..732b4d6ce 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -268,14 +268,6 @@ std::pair LocalOverlayStore::verifyAllValidPaths(RepairFlag } -Path LocalOverlayStore::toRealPathForHardLink(const StorePath & path) -{ - return lowerStore->isValidPath(path) - ? lowerStore->Store::toRealPath(path) - : Store::toRealPath(path); -} - - void LocalOverlayStore::remountIfNecessary() { if (!_remountRequired) return; diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 8ed7516db..ac730c77c 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -186,12 +186,6 @@ private: */ std::pair verifyAllValidPaths(RepairFlag repair) override; - /** - * For lower-store paths, we used the lower store location. This avoids the - * wasteful "copying up" that would otherwise happen. - */ - Path toRealPathForHardLink(const StorePath & storePath) override; - /** * Deletion only effects the upper layer, so we ignore lower-layer referrers. */ From 250c3541bb7ffffae81e873f09f781d3b57e5935 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 25 Oct 2023 15:29:11 -0400 Subject: [PATCH 114/133] Use `local-overlay://` not `local-overlay` for store URL This is a bit uglier, but allows us to avoid an ad-hoc special case in `store-api.cc`. --- src/libstore/local-overlay-store.hh | 9 ++++++--- src/libstore/store-api.cc | 5 ----- tests/functional/local-overlay-store/bad-uris.sh | 6 +++--- tests/functional/local-overlay-store/common.sh | 2 +- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index ac730c77c..060ca37c3 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -90,15 +90,18 @@ public: LocalOverlayStore(std::string scheme, std::string path, const Params & params) : LocalOverlayStore(params) { - throw UnimplementedError("LocalOverlayStore"); + if (!path.empty()) + throw UsageError("local-overlay:// store url doesn't support path part, only scheme and query params"); } static std::set uriSchemes() - { return {}; } + { + return { "local-overlay" }; + } std::string getUri() override { - return "local-overlay"; + return "local-overlay://"; } private: diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 55cdd71a9..069fad420 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -11,7 +11,6 @@ #include "archive.hh" #include "callback.hh" #include "remote-store.hh" -#include "local-overlay-store.hh" // FIXME this should not be here, see TODO below on // `addMultipleToStore`. #include "worker-protocol.hh" @@ -1456,10 +1455,6 @@ std::shared_ptr openFromNonUri(const std::string & uri, const Store::Para return std::make_shared(params); } else if (uri == "local") { return std::make_shared(params); - } else if (uri == "local-overlay") { - auto store = std::make_shared(params); - experimentalFeatureSettings.require(store->experimentalFeature()); - return store; } else if (isNonUriPath(uri)) { Store::Params params2 = params; params2["root"] = absPath(uri); diff --git a/tests/functional/local-overlay-store/bad-uris.sh b/tests/functional/local-overlay-store/bad-uris.sh index 462bf27eb..07b2c6f28 100644 --- a/tests/functional/local-overlay-store/bad-uris.sh +++ b/tests/functional/local-overlay-store/bad-uris.sh @@ -6,9 +6,9 @@ storeDirs mkdir -p $TEST_ROOT/bad_test badTestRoot=$TEST_ROOT/bad_test -storeBadRoot="local-overlay?root=$badTestRoot&lower-store=$storeA&upper-layer=$storeBTop" -storeBadLower="local-overlay?root=$storeBRoot&lower-store=$badTestRoot&upper-layer=$storeBTop" -storeBadUpper="local-overlay?root=$storeBRoot&lower-store=$storeA&upper-layer=$badTestRoot" +storeBadRoot="local-overlay://?root=$badTestRoot&lower-store=$storeA&upper-layer=$storeBTop" +storeBadLower="local-overlay://?root=$storeBRoot&lower-store=$badTestRoot&upper-layer=$storeBTop" +storeBadUpper="local-overlay://?root=$storeBRoot&lower-store=$storeA&upper-layer=$badTestRoot" declare -a storesBad=( "$storeBadRoot" "$storeBadLower" "$storeBadUpper" diff --git a/tests/functional/local-overlay-store/common.sh b/tests/functional/local-overlay-store/common.sh index 2d614b140..bd144c925 100644 --- a/tests/functional/local-overlay-store/common.sh +++ b/tests/functional/local-overlay-store/common.sh @@ -29,7 +29,7 @@ storeDirs () { storeA="$storeVolume/store-a" storeBTop="$storeVolume/store-b" storeBRoot="$storeVolume/merged-store" - storeB="local-overlay?root=$storeBRoot&lower-store=$storeA&upper-layer=$storeBTop" + storeB="local-overlay://?root=$storeBRoot&lower-store=$storeA&upper-layer=$storeBTop" # Creating testing directories mkdir -p "$storeVolume"/{store-a/nix/store,store-b,merged-store/nix/store,workdir} } From c30b5d8a0bc7766e55b5254c75b264597bd2935e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 11 Dec 2023 13:18:34 -0500 Subject: [PATCH 115/133] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com> --- src/libstore/local-overlay-store.hh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 060ca37c3..64ed2f20a 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -24,7 +24,7 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig const PathSetting upperLayer{(StoreConfig*) this, "", "upper-layer", R"( - Must be used as OverlayFS upper layer for this store's store dir. + Directory containing the OverlayFS upper layer for this store's store dir. )"}; Setting checkMount{(StoreConfig*) this, true, "check-mount", @@ -60,6 +60,7 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig std::string doc() override; +protected: /** * Given a store path, get its location (if it is exists) in the * upper layer of the overlayfs. @@ -167,7 +168,7 @@ private: * Deduplicate by removing store objects from the upper layer that * are now in the lower layer. * - * This implementation will not cause duplications, but addition of + * Operations on a layered store will not cause duplications, but addition of * new store objects to the lower layer can instill induce them * (there is no way to prevent that). This cleans up those * duplications. From b21ee60594c637b4ea5ab98d5cc60b34a66bdde4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 11 Dec 2023 13:28:40 -0500 Subject: [PATCH 116/133] Get rid of `verifyAllValidPaths` boolean blindness --- src/libstore/local-overlay-store.cc | 7 +++++-- src/libstore/local-overlay-store.hh | 2 +- src/libstore/local-store.cc | 7 +++++-- src/libstore/local-store.hh | 26 ++++++++++++++++++++++---- 4 files changed, 33 insertions(+), 9 deletions(-) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index f657d8341..598415db8 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -252,7 +252,7 @@ void LocalOverlayStore::optimiseStore() } -std::pair LocalOverlayStore::verifyAllValidPaths(RepairFlag repair) +LocalStore::VerificationResult LocalOverlayStore::verifyAllValidPaths(RepairFlag repair) { StorePathSet done; @@ -266,7 +266,10 @@ std::pair LocalOverlayStore::verifyAllValidPaths(RepairFlag for (auto & i : queryAllValidPaths()) verifyPath(i, existsInStoreDir, done, validPaths, repair, errors); - return { errors, validPaths }; + return { + .errors = errors, + .validPaths = validPaths, + }; } diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 64ed2f20a..a93b069cc 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -188,7 +188,7 @@ private: * We don't verify the contents of both layers on the assumption that the lower layer is far bigger, * and also the observation that anything not in the upper db the overlayfs doesn't yet care about. */ - std::pair verifyAllValidPaths(RepairFlag repair) override; + VerificationResult verifyAllValidPaths(RepairFlag repair) override; /** * Deletion only effects the upper layer, so we ignore lower-layer referrers. diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 8561d58be..b35d59bbb 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1443,7 +1443,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) } -std::pair LocalStore::verifyAllValidPaths(RepairFlag repair) +LocalStore::VerificationResult LocalStore::verifyAllValidPaths(RepairFlag repair) { StorePathSet storePathsInStoreDir; /* Why aren't we using `queryAllValidPaths`? Because that would @@ -1476,7 +1476,10 @@ std::pair LocalStore::verifyAllValidPaths(RepairFlag repair) for (auto & i : queryAllValidPaths()) verifyPath(i, existsInStoreDir, done, validPaths, repair, errors); - return { errors, validPaths }; + return { + .errors = errors, + .validPaths = validPaths, + }; } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 9a9171e2f..dc2d5cb7c 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -264,12 +264,30 @@ public: bool verifyStore(bool checkContents, RepairFlag repair) override; +protected: + /** - * @return A pair of whether any errors were encountered, and a set of - * (so-far) valid paths. The store objects pointed to by those paths are - * suitable for further validation checking. + * Result of `verifyAllValidPaths` */ - virtual std::pair verifyAllValidPaths(RepairFlag repair); + struct VerificationResult { + /** + * Whether any errors were encountered + */ + bool errors; + + /** + * A set of so-far valid paths. The store objects pointed to by + * those paths are suitable for further validation checking. + */ + StorePathSet validPaths; + }; + + /** + * First, unconditional step of `verifyStore` + */ + virtual VerificationResult verifyAllValidPaths(RepairFlag repair); + +public: /** * Register the validity of a path, i.e., that `path` exists, that From bf0bf3d1be47aa1bcf9cb3c8f0923e1304fc7065 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 11 Dec 2023 13:30:40 -0500 Subject: [PATCH 117/133] local-overlay store tests: `storeDirs` -> `setupStoreDirs` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Théophane Hufschmitt --- tests/functional/local-overlay-store/add-lower-inner.sh | 2 +- tests/functional/local-overlay-store/bad-uris.sh | 4 ++-- tests/functional/local-overlay-store/build-inner.sh | 2 +- tests/functional/local-overlay-store/check-post-init-inner.sh | 2 +- tests/functional/local-overlay-store/common.sh | 2 +- .../functional/local-overlay-store/delete-duplicate-inner.sh | 2 +- tests/functional/local-overlay-store/delete-refs-inner.sh | 2 +- tests/functional/local-overlay-store/gc-inner.sh | 2 +- tests/functional/local-overlay-store/optimise-inner.sh | 2 +- tests/functional/local-overlay-store/redundant-add-inner.sh | 2 +- .../functional/local-overlay-store/stale-file-handle-inner.sh | 2 +- tests/functional/local-overlay-store/verify-inner.sh | 2 +- 12 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/functional/local-overlay-store/add-lower-inner.sh b/tests/functional/local-overlay-store/add-lower-inner.sh index ca7db7ab6..4efa7d088 100755 --- a/tests/functional/local-overlay-store/add-lower-inner.sh +++ b/tests/functional/local-overlay-store/add-lower-inner.sh @@ -10,7 +10,7 @@ source common.sh unset NIX_STORE_DIR unset NIX_STATE_DIR -storeDirs +setupStoreDirs initLowerStore diff --git a/tests/functional/local-overlay-store/bad-uris.sh b/tests/functional/local-overlay-store/bad-uris.sh index 07b2c6f28..2517681dd 100644 --- a/tests/functional/local-overlay-store/bad-uris.sh +++ b/tests/functional/local-overlay-store/bad-uris.sh @@ -2,7 +2,7 @@ source common.sh requireEnvironment setupConfig -storeDirs +setupStoreDirs mkdir -p $TEST_ROOT/bad_test badTestRoot=$TEST_ROOT/bad_test @@ -18,7 +18,7 @@ for i in "${storesBad[@]}"; do echo $i unshare --mount --map-root-user bash < Date: Mon, 11 Dec 2023 13:34:09 -0500 Subject: [PATCH 118/133] Update tests/functional/local-overlay-store/redundant-add-inner.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com> --- tests/functional/local-overlay-store/redundant-add-inner.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/functional/local-overlay-store/redundant-add-inner.sh b/tests/functional/local-overlay-store/redundant-add-inner.sh index 71899a9a7..0fb3af2a6 100755 --- a/tests/functional/local-overlay-store/redundant-add-inner.sh +++ b/tests/functional/local-overlay-store/redundant-add-inner.sh @@ -19,9 +19,13 @@ mountOverlayfs ### Do a redundant add # upper layer should not have it +path=$(nix-store --store "$storeA" ../dummy) + expect 1 stat $(toRealPath "$storeBTop/nix/store" "$path") -path=$(nix-store --store "$storeB" --add ../dummy) +pathFromB=$(nix-store --store "$storeB" --add ../dummy) + +[[ $path == $pathFromB ]] # lower store should have it from before stat $(toRealPath "$storeA/nix/store" "$path") From c93f78f6fabdc4f01d3af98c68c4225be9d2c546 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 11 Dec 2023 13:36:53 -0500 Subject: [PATCH 119/133] Fix test a bit from previous commit --- tests/functional/local-overlay-store/redundant-add-inner.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/functional/local-overlay-store/redundant-add-inner.sh b/tests/functional/local-overlay-store/redundant-add-inner.sh index 0fb3af2a6..d2c95b187 100755 --- a/tests/functional/local-overlay-store/redundant-add-inner.sh +++ b/tests/functional/local-overlay-store/redundant-add-inner.sh @@ -18,9 +18,10 @@ mountOverlayfs ### Do a redundant add -# upper layer should not have it -path=$(nix-store --store "$storeA" ../dummy) +# (Already done in `initLowerStore`, but repeated here for clarity.) +path=$(nix-store --store "$storeA" --add ../dummy) +# upper layer should not have it expect 1 stat $(toRealPath "$storeBTop/nix/store" "$path") pathFromB=$(nix-store --store "$storeB" --add ../dummy) From b3bdd70ea2a69ec2ddd547ee475219f5b38b8f59 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 11 Dec 2023 13:43:17 -0500 Subject: [PATCH 120/133] Clarify `toUpperPath` docs We're just mapping store paths to host OS paths, there is no checking what is actually at this location. --- src/libstore/local-overlay-store.hh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index a93b069cc..2c24285dd 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -62,8 +62,12 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig protected: /** - * Given a store path, get its location (if it is exists) in the - * upper layer of the overlayfs. + * @return The host OS path corresponding to the store path for the + * upper layer. + * + * @note The there is no guarantee a store object is actually stored + * at that file path. It might be stored in the lower layer instead, + * or it might not be part of this store at all. */ Path toUpperPath(const StorePath & path); }; From c90e46d3f0c6614921b42ba5506a409ca11b5c30 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 11 Dec 2023 13:45:46 -0500 Subject: [PATCH 121/133] Update tests/functional/local-overlay-store/common.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com> --- tests/functional/local-overlay-store/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/local-overlay-store/common.sh b/tests/functional/local-overlay-store/common.sh index 0de688eb4..e1da00b42 100644 --- a/tests/functional/local-overlay-store/common.sh +++ b/tests/functional/local-overlay-store/common.sh @@ -66,7 +66,7 @@ initLowerStore () { # Build something in lower store drvPath=$(nix-instantiate --store $storeA ../hermetic.nix --arg busybox "$busybox" --arg seed 1) - path=$(nix-store --store "$storeA" --realise $drvPath) + pathInLowerStore=$(nix-store --store "$storeA" --realise $drvPath) } execUnshare () { From 8d0a03b5a22df6f764686386edde6b90d290b8e4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 11 Dec 2023 13:48:42 -0500 Subject: [PATCH 122/133] Fix tests after last rename (`path` -> `pathInLowerStore`) --- .../check-post-init-inner.sh | 20 +++++++++---------- .../redundant-add-inner.sh | 10 +++++----- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/functional/local-overlay-store/check-post-init-inner.sh b/tests/functional/local-overlay-store/check-post-init-inner.sh index ccc0461a3..ac2499002 100755 --- a/tests/functional/local-overlay-store/check-post-init-inner.sh +++ b/tests/functional/local-overlay-store/check-post-init-inner.sh @@ -19,13 +19,13 @@ mountOverlayfs ### Check status # Checking for path in lower layer -stat $(toRealPath "$storeA/nix/store" "$path") +stat $(toRealPath "$storeA/nix/store" "$pathInLowerStore") # Checking for path in upper layer (should fail) -expect 1 stat $(toRealPath "$storeBTop" "$path") +expect 1 stat $(toRealPath "$storeBTop" "$pathInLowerStore") # Checking for path in overlay store matching lower layer -diff $(toRealPath "$storeA/nix/store" "$path") $(toRealPath "$storeBRoot/nix/store" "$path") +diff $(toRealPath "$storeA/nix/store" "$pathInLowerStore") $(toRealPath "$storeBRoot/nix/store" "$pathInLowerStore") # Checking requisites query agreement [[ \ @@ -44,9 +44,9 @@ busyboxStore=$(nix store --store $storeA add-path $busybox) # Checking derivers query agreement [[ \ - $(nix-store --store $storeA --query --deriver $path) \ + $(nix-store --store $storeA --query --deriver $pathInLowerStore) \ == \ - $(nix-store --store $storeB --query --deriver $path) \ + $(nix-store --store $storeB --query --deriver $pathInLowerStore) \ ]] # Checking outputs query agreement @@ -57,15 +57,15 @@ busyboxStore=$(nix store --store $storeA add-path $busybox) ]] # Verifying path in lower layer -nix-store --verify-path --store "$storeA" "$path" +nix-store --verify-path --store "$storeA" "$pathInLowerStore" # Verifying path in merged-store -nix-store --verify-path --store "$storeB" "$path" +nix-store --verify-path --store "$storeB" "$pathInLowerStore" -hashPart=$(echo $path | sed "s^${NIX_STORE_DIR:-/nix/store}/^^" | sed 's/-.*//') +hashPart=$(echo $pathInLowerStore | sed "s^${NIX_STORE_DIR:-/nix/store}/^^" | sed 's/-.*//') # Lower store can find from hash part -[[ $(nix store --store $storeA path-from-hash-part $hashPart) == $path ]] +[[ $(nix store --store $storeA path-from-hash-part $hashPart) == $pathInLowerStore ]] # merged store can find from hash part -[[ $(nix store --store $storeB path-from-hash-part $hashPart) == $path ]] +[[ $(nix store --store $storeB path-from-hash-part $hashPart) == $pathInLowerStore ]] diff --git a/tests/functional/local-overlay-store/redundant-add-inner.sh b/tests/functional/local-overlay-store/redundant-add-inner.sh index d2c95b187..e37ef90e5 100755 --- a/tests/functional/local-overlay-store/redundant-add-inner.sh +++ b/tests/functional/local-overlay-store/redundant-add-inner.sh @@ -19,17 +19,17 @@ mountOverlayfs ### Do a redundant add # (Already done in `initLowerStore`, but repeated here for clarity.) -path=$(nix-store --store "$storeA" --add ../dummy) +pathInLowerStore=$(nix-store --store "$storeA" --add ../dummy) # upper layer should not have it -expect 1 stat $(toRealPath "$storeBTop/nix/store" "$path") +expect 1 stat $(toRealPath "$storeBTop/nix/store" "$pathInLowerStore") pathFromB=$(nix-store --store "$storeB" --add ../dummy) -[[ $path == $pathFromB ]] +[[ $pathInLowerStore == $pathFromB ]] # lower store should have it from before -stat $(toRealPath "$storeA/nix/store" "$path") +stat $(toRealPath "$storeA/nix/store" "$pathInLowerStore") # upper layer should still not have it (no redundant copy) -expect 1 stat $(toRealPath "$storeBTop" "$path") +expect 1 stat $(toRealPath "$storeBTop" "$pathInLowerStore") From 4a2cee8e6cf2cfe0975b11fbccf6beb36b99cc60 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Mon, 11 Dec 2023 18:55:39 +0000 Subject: [PATCH 123/133] Document expected filesystem layout and OverlayFS mount command. --- src/libstore/local-overlay-store.md | 42 +++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/libstore/local-overlay-store.md b/src/libstore/local-overlay-store.md index a0b9c3adf..813efc3e9 100644 --- a/src/libstore/local-overlay-store.md +++ b/src/libstore/local-overlay-store.md @@ -5,6 +5,7 @@ R"( This store type is a variation of the [local store] designed to leverage Linux's [Overlay Filesystem](https://docs.kernel.org/filesystems/overlayfs.html) (OverlayFS for short). Just as OverlayFS combines a lower and upper filesystem by treating the upper one as a patch against the lower, the local overlay store combines a lower store with an upper almost [local store]. ("almost" because while the upper fileystems for OverlayFS is valid on its own, the upper almost-store is not a valid local store on its own because some references will dangle.) +To use this store, you will first need to configure an OverlayFS mountpoint [appropriately](#example-filesystem-layout) as Nix will not do this for you (though it will verify the mountpoint is configured correctly). ### Parts of a local overlay store @@ -15,6 +16,9 @@ The parts of a local overlay store are as follows: This is any store implementation that includes a store directory as part of the native operating system filesystem. For example, this could be a [local store], [local daemon store], or even another local overlay store. + The lower store must not change while it is mounted as part of an overlay store. + To ensure it does not, you might want to mount the store directory read-only (which then requires the [read-only] parameter to be set to `true`). + Specified with the `lower-store` setting. - Lower store directory. @@ -51,4 +55,42 @@ The parts of a local overlay store are as follows: Specified with the `state` setting, is always `${state}/db`. + +### Example filesystem layout + +Say we have the following paths: + +- `/mnt/example/merged-store/nix/store` + +- `/mnt/example/store-a/nix/store` + +- `/mnt/example/store-b` + + +Then the following store URI can be used to access a local-overlay store at `/mnt/example/merged-store`: + +``` + local-overlay://?root=/mnt/example/merged-store&lower-store=/mnt/example/store-a&upper-layer=/mnt/example/store-b +``` + +The lower store is located at `/mnt/example/store-a/nix/store`, while the upper layer is at `/mnt/example/store-b`. + +Before accessing the overlay store you will need to ensure the OverlayFS mount is set up correctly: + +``` + mount -t overlay overlay \ + -o lowerdir="/mnt/example/store-a/nix/store" \ + -o upperdir="/mnt/example/store-b" \ + -o workdir="/mnt/example/workdir" \ + "/mnt/example/merged-store/nix/store" \ +``` + +Note that OverlayFS requires `/mnt/example/workdir` to be on the same volume as the `upperdir`. + +By default, Nix will check that the mountpoint as been set up correctly and fail with an error if it has not. +You can override this behaviour by passing [`check-mount=false`](???) if you need to. + + + + )" From eae2717e00e82a61884b3f53a50db94272a1fad1 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 11 Dec 2023 13:55:43 -0500 Subject: [PATCH 124/133] tests: Use `cp -ar` instead of tar-untar pipe --- tests/functional/local-overlay-store/verify-inner.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/local-overlay-store/verify-inner.sh b/tests/functional/local-overlay-store/verify-inner.sh index c98ea1f96..659f2ae50 100755 --- a/tests/functional/local-overlay-store/verify-inner.sh +++ b/tests/functional/local-overlay-store/verify-inner.sh @@ -34,7 +34,7 @@ nix-store --store "$storeB" --verify --check-contents # Make a backup so we can repair later backupStore="$storeVolume/backup" mkdir "$backupStore" -tar -cC "$storeBRoot" nix | tar -xC "$backupStore" +cp -ar "$storeBRoot/nix" "$backupStore" ## Deliberately corrupt store paths From dc439eaf236ea1ef5c291b50de25817df8103041 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 1 Feb 2024 11:20:19 -0500 Subject: [PATCH 125/133] Fill in missing markdown link dest --- src/libstore/local-overlay-store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-overlay-store.md b/src/libstore/local-overlay-store.md index 813efc3e9..882d2c5ce 100644 --- a/src/libstore/local-overlay-store.md +++ b/src/libstore/local-overlay-store.md @@ -88,7 +88,7 @@ Before accessing the overlay store you will need to ensure the OverlayFS mount i Note that OverlayFS requires `/mnt/example/workdir` to be on the same volume as the `upperdir`. By default, Nix will check that the mountpoint as been set up correctly and fail with an error if it has not. -You can override this behaviour by passing [`check-mount=false`](???) if you need to. +You can override this behaviour by passing [`check-mount=false`](#store-experimental-local-overlay-store-check-mount) if you need to. From 9b506ff0c1e48fa805eab86eb43bd99ac1ae24a3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 29 Feb 2024 10:06:53 -0500 Subject: [PATCH 126/133] Activate `hermetic.nix` variation only for new layered store tests --- tests/functional/hermetic.nix | 9 +++++++-- tests/functional/local-overlay-store/common.sh | 2 +- .../functional/local-overlay-store/delete-refs-inner.sh | 8 ++++---- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/functional/hermetic.nix b/tests/functional/hermetic.nix index 810180ac6..d1dccdff3 100644 --- a/tests/functional/hermetic.nix +++ b/tests/functional/hermetic.nix @@ -1,4 +1,9 @@ -{ busybox, seed }: +{ busybox +, seed +# If we want the final derivation output to have references to its +# dependencies. Some tests need/want this, other don't. +, withFinalRefs ? false +}: with import ./config.nix; @@ -54,6 +59,6 @@ in '' read x < ${input1} read y < ${input3} - echo ${input1} ${input3} "$x $y" > $out + echo ${if (builtins.trace withFinalRefs withFinalRefs) then "${input1} ${input3}" else ""} "$x $y" > $out ''; } diff --git a/tests/functional/local-overlay-store/common.sh b/tests/functional/local-overlay-store/common.sh index e1da00b42..13d921c5e 100644 --- a/tests/functional/local-overlay-store/common.sh +++ b/tests/functional/local-overlay-store/common.sh @@ -65,7 +65,7 @@ initLowerStore () { nix-store --store "$storeA" --add ../dummy # Build something in lower store - drvPath=$(nix-instantiate --store $storeA ../hermetic.nix --arg busybox "$busybox" --arg seed 1) + drvPath=$(nix-instantiate --store $storeA ../hermetic.nix --arg withFinalRefs true --arg busybox "$busybox" --arg seed 1) pathInLowerStore=$(nix-store --store "$storeA" --realise $drvPath) } diff --git a/tests/functional/local-overlay-store/delete-refs-inner.sh b/tests/functional/local-overlay-store/delete-refs-inner.sh index e24f34a85..385eeadc9 100644 --- a/tests/functional/local-overlay-store/delete-refs-inner.sh +++ b/tests/functional/local-overlay-store/delete-refs-inner.sh @@ -16,10 +16,10 @@ mountOverlayfs export NIX_REMOTE="$storeB" stateB="$storeBRoot/nix/var/nix" -hermetic=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg seed 2) -input1=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg seed 2 -A passthru.input1 -j0) -input2=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg seed 2 -A passthru.input2 -j0) -input3=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg seed 2 -A passthru.input3 -j0) +hermetic=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg withFinalRefs true --arg seed 2) +input1=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg withFinalRefs true --arg seed 2 -A passthru.input1 -j0) +input2=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg withFinalRefs true --arg seed 2 -A passthru.input2 -j0) +input3=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg withFinalRefs true --arg seed 2 -A passthru.input3 -j0) # Can't delete because referenced expectStderr 1 nix-store --delete $input1 | grepQuiet "Cannot delete path" From bcd6b33dbcbd2e9c2ef3eaf50b9339fe9a33a49d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 29 Feb 2024 11:58:53 -0500 Subject: [PATCH 127/133] Polish local overlay store docs --- src/libstore/local-overlay-store.md | 95 ++++++++++++++++++----------- 1 file changed, 61 insertions(+), 34 deletions(-) diff --git a/src/libstore/local-overlay-store.md b/src/libstore/local-overlay-store.md index 882d2c5ce..b85f69205 100644 --- a/src/libstore/local-overlay-store.md +++ b/src/libstore/local-overlay-store.md @@ -3,61 +3,92 @@ R"( **Store URL format**: `local-overlay` This store type is a variation of the [local store] designed to leverage Linux's [Overlay Filesystem](https://docs.kernel.org/filesystems/overlayfs.html) (OverlayFS for short). -Just as OverlayFS combines a lower and upper filesystem by treating the upper one as a patch against the lower, the local overlay store combines a lower store with an upper almost [local store]. +Just as OverlayFS combines a lower and upper filesystem by treating the upper one as a patch against the lower, the local overlay store combines a lower store with an upper almost-[local store]. ("almost" because while the upper fileystems for OverlayFS is valid on its own, the upper almost-store is not a valid local store on its own because some references will dangle.) To use this store, you will first need to configure an OverlayFS mountpoint [appropriately](#example-filesystem-layout) as Nix will not do this for you (though it will verify the mountpoint is configured correctly). -### Parts of a local overlay store +### Conceptual parts of a local overlay store + +*This is a more abstract/conceptual description of the parts of a layered store, an authoritative reference. +For more "practical" instructions, see the worked-out example in the next subsection.* The parts of a local overlay store are as follows: -- Lower store: +- **Lower store**: This is any store implementation that includes a store directory as part of the native operating system filesystem. For example, this could be a [local store], [local daemon store], or even another local overlay store. + The local overlay store never tries to modify the lower store in any way. + Something else could modify the lower store, but there are restrictions on this + Nix itself requires that this store only grow, and not change in other ways. + For example, new store objects can be added, but deleting or modifying store objects is not allowed in general, because that will confuse and corrupt any local overlay store using those objects. + (In addition, the underlying filesystem overlay mechanism may imposed additional restrictions, see below.) + The lower store must not change while it is mounted as part of an overlay store. To ensure it does not, you might want to mount the store directory read-only (which then requires the [read-only] parameter to be set to `true`). - Specified with the `lower-store` setting. + Specified with the [`lower-store`](#store-experimental-local-overlay-store-lower-store) setting. + + - **Lower store directory**: - - Lower store directory. This is the directory used/exposed by the lower store. Specified with `lower-store.real` setting. - - Lower abstract read-only metadata source. - This is abstract, just some way to read the metadata of lower store [store objects](@docroot@/glossary.md#gloss-store-object). + As specified above, Nix requires the local store can only grow not change in other ways. + Linux's OverlayFS in addition imposes the further requirement that this directory cannot change at all. + That means that, while any local overlay store exists that is using this store as a lower store, this directory must not change. + + - **Lower metadata source**: + + This is abstract, just some way to read the metadata of lower store [store objects][store object]. For example it could be a SQLite database (for the [local store]), or a socket connection (for the [local daemon store]). -- Upper almost-store: + This need not be writable. + As stated above a local overlay store never tries to modify its lower store. + The lower store's metadata is considered part of the lower store, just as the store's [file system objects][file system object] that appear in the store directory are. - This is a [local store] that by itself would appear corrupted. +- **Upper almost-store**: + + This is almost but not quite just a [local store]. + That is because taken in isolation, not as part of a local overlay store, by itself, it would appear corrupted. But combined with everything else as part of an overlay local store, it is valid. - - Upper layer directory. - This contains additional [store objects] - (or, strictly speaking, their [file system objects](#gloss-file-system-object)) - that the local overlay store will extend the lower store with. + - **Upper layer directory**: - Specified with `upper-layer` setting. + This contains additional [store objects][store object] + (or, strictly speaking, their [file system objects][file system object] that the local overlay store will extend the lower store with). + + Specified with [`upper-layer`](#store-experimental-local-overlay-store-upper-layer) setting. + + - **Upper store directory**: - - Upper store directory - The lower store directory and upper layer directory are combined via OverlayFS to create this directory. This contains all the store objects from each of the two directories. - Specified with the `real` setting. + The lower store directory and upper layer directory are combined via OverlayFS to create this directory. + Nix doesn't do this itself, because it typically wouldn't have the permissions to do so, so it is the responsibility of the user to set this up first. + Nix can, however, optionally check that that the OverlayFS mount settings appear as expected, matching Nix's own settings. - - Upper SQLite database - This contains the metadata of all of the upper layer [store objects]: everything beyond their file system objects, and also duplicate copies of some lower layer ones. - The duplication is so the metadata for the [closure](@docroot@/glossary.md#gloss-closure) of upper layer [store objects] can entirely be found in the upper layer. - This allows us to use the same SQL Schema as the [local store]'s SQLite database, as foreign keys in that schema enforce closure metadata to be self-contained in this way. + Specified with the [`real`](#store-experimental-local-overlay-store-real) setting. - Specified with the `state` setting, is always `${state}/db`. + - **Upper SQLite database**: + + This contains the metadata of all of the upper layer [store objects][store object] (everything beyond their file system objects), and also duplicate copies of some lower layer store object's metadta. + The duplication is so the metadata for the [closure](@docroot@/glossary.md#gloss-closure) of upper layer [store objects][store object] can be found entirely within the upper layer. + (This allows us to use the same SQL Schema as the [local store]'s SQLite database, as foreign keys in that schema enforce closure metadata to be self-contained in this way.) + + The location of the database is directly specified, but depends on the [`state`](#store-experimental-local-overlay-store-state) setting. + It is is always `${state}/db`. + +[file system object]: @docroot@/store/file-system-object.md +[store object]: @docroot@/store/store-object.md ### Example filesystem layout +Here is a worked out example of usage, following the concepts in the previous section. + Say we have the following paths: - `/mnt/example/merged-store/nix/store` @@ -66,23 +97,22 @@ Say we have the following paths: - `/mnt/example/store-b` - Then the following store URI can be used to access a local-overlay store at `/mnt/example/merged-store`: ``` - local-overlay://?root=/mnt/example/merged-store&lower-store=/mnt/example/store-a&upper-layer=/mnt/example/store-b +local-overlay://?root=/mnt/example/merged-store&lower-store=/mnt/example/store-a&upper-layer=/mnt/example/store-b ``` -The lower store is located at `/mnt/example/store-a/nix/store`, while the upper layer is at `/mnt/example/store-b`. +The lower store directory is located at `/mnt/example/store-a/nix/store`, while the upper layer is at `/mnt/example/store-b`. Before accessing the overlay store you will need to ensure the OverlayFS mount is set up correctly: -``` - mount -t overlay overlay \ - -o lowerdir="/mnt/example/store-a/nix/store" \ - -o upperdir="/mnt/example/store-b" \ - -o workdir="/mnt/example/workdir" \ - "/mnt/example/merged-store/nix/store" \ +```shell +mount -t overlay overlay \ + -o lowerdir="/mnt/example/store-a/nix/store" \ + -o upperdir="/mnt/example/store-b" \ + -o workdir="/mnt/example/workdir" \ + "/mnt/example/merged-store/nix/store" ``` Note that OverlayFS requires `/mnt/example/workdir` to be on the same volume as the `upperdir`. @@ -90,7 +120,4 @@ Note that OverlayFS requires `/mnt/example/workdir` to be on the same volume as By default, Nix will check that the mountpoint as been set up correctly and fail with an error if it has not. You can override this behaviour by passing [`check-mount=false`](#store-experimental-local-overlay-store-check-mount) if you need to. - - - )" From d6d7d2cb4617c571e1ee7c152818f3b7a07a5f81 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 29 Feb 2024 14:39:29 -0500 Subject: [PATCH 128/133] Revert "Merge pull request #9546 from NixOS/nixos-23.11" This reverts commit 587c7dcb2bc4e504159f527c7e776ede231bd0db, reversing changes made to 864fc85fc88ff092725ba99907611b2b8d2205fb. --- Makefile | 1 - flake.lock | 8 ++++---- flake.nix | 26 +++++++++++++++----------- package.nix | 2 +- src/libstore/local-store.cc | 2 +- tests/functional/git-hashing/common.sh | 2 +- tests/functional/impure-env.sh | 2 +- 7 files changed, 23 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index af00e58b2..a33614d26 100644 --- a/Makefile +++ b/Makefile @@ -69,7 +69,6 @@ ifeq ($(OPTIMIZE), 1) GLOBAL_LDFLAGS += $(CXXLTO) else GLOBAL_CXXFLAGS += -O0 -U_FORTIFY_SOURCE - unexport NIX_HARDENING_ENABLE endif include mk/platform.mk diff --git a/flake.lock b/flake.lock index bb2e400c0..f0efb4036 100644 --- a/flake.lock +++ b/flake.lock @@ -34,16 +34,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1709083642, - "narHash": "sha256-7kkJQd4rZ+vFrzWu8sTRtta5D1kBG0LSRYAfhtmMlSo=", + "lastModified": 1705033721, + "narHash": "sha256-K5eJHmL1/kev6WuqyqqbS1cdNnSidIZ3jeqJ7GbrYnQ=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "b550fe4b4776908ac2a861124307045f8e717c8e", + "rev": "a1982c92d8980a0114372973cbdfe0a307f1bdea", "type": "github" }, "original": { "owner": "NixOS", - "ref": "release-23.11", + "ref": "nixos-23.05-small", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index 42aaace67..0bc70768e 100644 --- a/flake.nix +++ b/flake.nix @@ -1,9 +1,7 @@ { description = "The purely functional package manager"; - # TODO switch to nixos-23.11-small - # https://nixpk.gs/pr-tracker.html?pr=291954 - inputs.nixpkgs.url = "github:NixOS/nixpkgs/release-23.11"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; }; @@ -12,10 +10,20 @@ let inherit (nixpkgs) lib; - inherit (lib) fileset; + + # Experimental fileset library: https://github.com/NixOS/nixpkgs/pull/222981 + # Not an "idiomatic" flake input because: + # - Propagation to dependent locks: https://github.com/NixOS/nix/issues/7730 + # - Subflake would download redundant and huge parent flake + # - No git tree hash support: https://github.com/NixOS/nix/issues/6044 + inherit (import (builtins.fetchTarball { url = "https://github.com/NixOS/nix/archive/1bdcd7fc8a6a40b2e805bad759b36e64e911036b.tar.gz"; sha256 = "sha256:14ljlpdsp4x7h1fkhbmc4bd3vsqnx8zdql4h3037wh09ad6a0893"; })) + fileset; officialRelease = false; + # Set to true to build the release notes for the next release. + buildUnreleasedNotes = false; + version = lib.fileContents ./.version + versionSuffix; versionSuffix = if officialRelease @@ -396,11 +404,8 @@ # Make bash completion work. XDG_DATA_DIRS+=:$out/share ''; - nativeBuildInputs = attrs.nativeBuildInputs or [] - # TODO: Remove the darwin check once - # https://github.com/NixOS/nixpkgs/pull/291814 is available - ++ lib.optional (stdenv.cc.isClang && !stdenv.buildPlatform.isDarwin) pkgs.buildPackages.bear + ++ lib.optional stdenv.cc.isClang pkgs.buildPackages.bear ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) pkgs.buildPackages.clang-tools; }); in @@ -412,9 +417,8 @@ (forAllStdenvs (stdenvName: makeShell pkgs pkgs.${stdenvName})); in (makeShells "native" nixpkgsFor.${system}.native) // - (lib.optionalAttrs (!nixpkgsFor.${system}.native.stdenv.isDarwin) - (makeShells "static" nixpkgsFor.${system}.static)) // - (lib.genAttrs shellCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv)) // + (makeShells "static" nixpkgsFor.${system}.static) // + (lib.genAttrs shellCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv)) // { default = self.devShells.${system}.native-stdenvPackages; } diff --git a/package.nix b/package.nix index 20796a386..1f895e301 100644 --- a/package.nix +++ b/package.nix @@ -154,7 +154,7 @@ in { in fileset.toSource { root = ./.; - fileset = fileset.intersection baseFiles (fileset.unions ([ + fileset = fileset.intersect baseFiles (fileset.unions ([ # For configure ./.version ./configure.ac diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index a7c34ab4a..0d9faa7c6 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1208,7 +1208,7 @@ StorePath LocalStore::addToStoreFromDump( Path tempDir; AutoCloseFD tempDirFd; - bool methodsMatch = ContentAddressMethod(FileIngestionMethod(dumpMethod)) == hashMethod; + bool methodsMatch = (FileIngestionMethod) dumpMethod == hashMethod; /* If the methods don't match, our streaming hash of the dump is the wrong sort, and we need to rehash. */ diff --git a/tests/functional/git-hashing/common.sh b/tests/functional/git-hashing/common.sh index 572cea438..5de96e74f 100644 --- a/tests/functional/git-hashing/common.sh +++ b/tests/functional/git-hashing/common.sh @@ -4,7 +4,7 @@ clearStore clearCache # Need backend to support git-hashing too -requireDaemonNewerThan "2.19" +requireDaemonNewerThan "2.18.0pre20230908" enableFeatures "git-hashing" diff --git a/tests/functional/impure-env.sh b/tests/functional/impure-env.sh index cfea4cae9..d9e4a34a2 100644 --- a/tests/functional/impure-env.sh +++ b/tests/functional/impure-env.sh @@ -1,7 +1,7 @@ source common.sh # Needs the config option 'impure-env' to work -requireDaemonNewerThan "2.19.0" +requireDaemonNewerThan "2.18.0pre20230816" enableFeatures "configurable-impure-env" restartDaemon From 5a2985431c8b03be7d434a39ea62a8b3d26e4f51 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 29 Feb 2024 14:52:31 -0500 Subject: [PATCH 129/133] Revert "Revert "Merge pull request #9546 from NixOS/nixos-23.11"" This reverts commit d6d7d2cb4617c571e1ee7c152818f3b7a07a5f81. --- Makefile | 1 + flake.lock | 8 ++++---- flake.nix | 26 +++++++++++--------------- package.nix | 2 +- src/libstore/local-store.cc | 2 +- tests/functional/git-hashing/common.sh | 2 +- tests/functional/impure-env.sh | 2 +- 7 files changed, 20 insertions(+), 23 deletions(-) diff --git a/Makefile b/Makefile index a33614d26..af00e58b2 100644 --- a/Makefile +++ b/Makefile @@ -69,6 +69,7 @@ ifeq ($(OPTIMIZE), 1) GLOBAL_LDFLAGS += $(CXXLTO) else GLOBAL_CXXFLAGS += -O0 -U_FORTIFY_SOURCE + unexport NIX_HARDENING_ENABLE endif include mk/platform.mk diff --git a/flake.lock b/flake.lock index f0efb4036..bb2e400c0 100644 --- a/flake.lock +++ b/flake.lock @@ -34,16 +34,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1705033721, - "narHash": "sha256-K5eJHmL1/kev6WuqyqqbS1cdNnSidIZ3jeqJ7GbrYnQ=", + "lastModified": 1709083642, + "narHash": "sha256-7kkJQd4rZ+vFrzWu8sTRtta5D1kBG0LSRYAfhtmMlSo=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "a1982c92d8980a0114372973cbdfe0a307f1bdea", + "rev": "b550fe4b4776908ac2a861124307045f8e717c8e", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-23.05-small", + "ref": "release-23.11", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index 0bc70768e..42aaace67 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,9 @@ { description = "The purely functional package manager"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small"; + # TODO switch to nixos-23.11-small + # https://nixpk.gs/pr-tracker.html?pr=291954 + inputs.nixpkgs.url = "github:NixOS/nixpkgs/release-23.11"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; }; @@ -10,20 +12,10 @@ let inherit (nixpkgs) lib; - - # Experimental fileset library: https://github.com/NixOS/nixpkgs/pull/222981 - # Not an "idiomatic" flake input because: - # - Propagation to dependent locks: https://github.com/NixOS/nix/issues/7730 - # - Subflake would download redundant and huge parent flake - # - No git tree hash support: https://github.com/NixOS/nix/issues/6044 - inherit (import (builtins.fetchTarball { url = "https://github.com/NixOS/nix/archive/1bdcd7fc8a6a40b2e805bad759b36e64e911036b.tar.gz"; sha256 = "sha256:14ljlpdsp4x7h1fkhbmc4bd3vsqnx8zdql4h3037wh09ad6a0893"; })) - fileset; + inherit (lib) fileset; officialRelease = false; - # Set to true to build the release notes for the next release. - buildUnreleasedNotes = false; - version = lib.fileContents ./.version + versionSuffix; versionSuffix = if officialRelease @@ -404,8 +396,11 @@ # Make bash completion work. XDG_DATA_DIRS+=:$out/share ''; + nativeBuildInputs = attrs.nativeBuildInputs or [] - ++ lib.optional stdenv.cc.isClang pkgs.buildPackages.bear + # TODO: Remove the darwin check once + # https://github.com/NixOS/nixpkgs/pull/291814 is available + ++ lib.optional (stdenv.cc.isClang && !stdenv.buildPlatform.isDarwin) pkgs.buildPackages.bear ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) pkgs.buildPackages.clang-tools; }); in @@ -417,8 +412,9 @@ (forAllStdenvs (stdenvName: makeShell pkgs pkgs.${stdenvName})); in (makeShells "native" nixpkgsFor.${system}.native) // - (makeShells "static" nixpkgsFor.${system}.static) // - (lib.genAttrs shellCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv)) // + (lib.optionalAttrs (!nixpkgsFor.${system}.native.stdenv.isDarwin) + (makeShells "static" nixpkgsFor.${system}.static)) // + (lib.genAttrs shellCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv)) // { default = self.devShells.${system}.native-stdenvPackages; } diff --git a/package.nix b/package.nix index 1f895e301..20796a386 100644 --- a/package.nix +++ b/package.nix @@ -154,7 +154,7 @@ in { in fileset.toSource { root = ./.; - fileset = fileset.intersect baseFiles (fileset.unions ([ + fileset = fileset.intersection baseFiles (fileset.unions ([ # For configure ./.version ./configure.ac diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 0d9faa7c6..a7c34ab4a 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1208,7 +1208,7 @@ StorePath LocalStore::addToStoreFromDump( Path tempDir; AutoCloseFD tempDirFd; - bool methodsMatch = (FileIngestionMethod) dumpMethod == hashMethod; + bool methodsMatch = ContentAddressMethod(FileIngestionMethod(dumpMethod)) == hashMethod; /* If the methods don't match, our streaming hash of the dump is the wrong sort, and we need to rehash. */ diff --git a/tests/functional/git-hashing/common.sh b/tests/functional/git-hashing/common.sh index 5de96e74f..572cea438 100644 --- a/tests/functional/git-hashing/common.sh +++ b/tests/functional/git-hashing/common.sh @@ -4,7 +4,7 @@ clearStore clearCache # Need backend to support git-hashing too -requireDaemonNewerThan "2.18.0pre20230908" +requireDaemonNewerThan "2.19" enableFeatures "git-hashing" diff --git a/tests/functional/impure-env.sh b/tests/functional/impure-env.sh index d9e4a34a2..cfea4cae9 100644 --- a/tests/functional/impure-env.sh +++ b/tests/functional/impure-env.sh @@ -1,7 +1,7 @@ source common.sh # Needs the config option 'impure-env' to work -requireDaemonNewerThan "2.18.0pre20230816" +requireDaemonNewerThan "2.19.0" enableFeatures "configurable-impure-env" restartDaemon From fe42a0ead70db8f5b82dec42a14682bbe4414577 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= <7226587+thufschmitt@users.noreply.github.com> Date: Wed, 6 Mar 2024 09:10:32 +0100 Subject: [PATCH 130/133] Documentation typo --- src/libstore/local-overlay-store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-overlay-store.md b/src/libstore/local-overlay-store.md index b85f69205..cc310bc7f 100644 --- a/src/libstore/local-overlay-store.md +++ b/src/libstore/local-overlay-store.md @@ -23,7 +23,7 @@ The parts of a local overlay store are as follows: Something else could modify the lower store, but there are restrictions on this Nix itself requires that this store only grow, and not change in other ways. For example, new store objects can be added, but deleting or modifying store objects is not allowed in general, because that will confuse and corrupt any local overlay store using those objects. - (In addition, the underlying filesystem overlay mechanism may imposed additional restrictions, see below.) + (In addition, the underlying filesystem overlay mechanism may impose additional restrictions, see below.) The lower store must not change while it is mounted as part of an overlay store. To ensure it does not, you might want to mount the store directory read-only (which then requires the [read-only] parameter to be set to `true`). From cd35e0010322bbf30b967cb7991e84dd740e43df Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 18 Mar 2024 16:41:16 -0400 Subject: [PATCH 131/133] Adding missing tracking URL for local overlay store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com> --- src/libutil/experimental-features.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 595dd4301..e1a8b5b9d 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -260,6 +260,7 @@ constexpr std::array xpFeatureDetails .description = R"( Allow the use of [local overlay store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-overlay-store). )", + .trackingUrl = ""https://github.com/NixOS/nix/milestone/50", }, { .tag = Xp::ConfigurableImpureEnv, From e73dc0e938c59b071eb24252980935b51cb17106 Mon Sep 17 00:00:00 2001 From: cidkidnix Date: Fri, 5 Apr 2024 16:43:14 -0500 Subject: [PATCH 132/133] Use LIBMOUNT_FORCE_MOUNT2=always to workaround new mount API issues --- tests/functional/local-overlay-store/common.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/functional/local-overlay-store/common.sh b/tests/functional/local-overlay-store/common.sh index 13d921c5e..1c70f947f 100644 --- a/tests/functional/local-overlay-store/common.sh +++ b/tests/functional/local-overlay-store/common.sh @@ -1,5 +1,7 @@ source ../common.sh +export LIBMOUNT_FORCE_MOUNT2=always + requireEnvironment () { requireSandboxSupport [[ $busybox =~ busybox ]] || skipTest "no busybox" From bd7c26bc7bb7af37c21d46faf28aa0e88606b98f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 7 Apr 2024 21:37:30 -0400 Subject: [PATCH 133/133] Add comment explaining `LIBMOUNT_FORCE_MOUNT2=always` --- .../functional/local-overlay-store/common.sh | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/functional/local-overlay-store/common.sh b/tests/functional/local-overlay-store/common.sh index 1c70f947f..2634f8c8f 100644 --- a/tests/functional/local-overlay-store/common.sh +++ b/tests/functional/local-overlay-store/common.sh @@ -1,5 +1,26 @@ source ../common.sh +# The new Linux mount interface does not seem to support remounting +# OverlayFS mount points. +# +# It is not clear whether this is intentional or not: +# +# The kernel source code [1] would seem to indicate merely remounting +# while *changing* mount options is now an error because it erroneously +# succeeded (by ignoring those new options) before. However, we are +# *not* trying to remount with changed options, and are still hitting +# the failure when using the new interface. +# +# For further details, see these `util-linux` issues: +# +# - https://github.com/util-linux/util-linux/issues/2528 +# - https://github.com/util-linux/util-linux/issues/2576 +# +# In the meantime, setting this environment variable to "always" will +# force the use of the old mount interface, keeping the remounting +# working and these tests passing. +# +# [1]: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/fs/overlayfs/params.c?id=3006adf3be79cde4d14b1800b963b82b6e5572e0#n549 export LIBMOUNT_FORCE_MOUNT2=always requireEnvironment () {