From 398c4136c2a963a38b8d7664c47f2bd8c60e86c3 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 31 Mar 2025 21:35:15 -0400 Subject: [PATCH 1/8] Improve and fix the error message when a file is not tracked by Git (cherry picked from commit 62e2304891375f642ac7b52358d36455ce99171a) --- src/libfetchers/git.cc | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index f46334d30..5684583cd 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -534,11 +534,21 @@ struct GitInputScheme : InputScheme static MakeNotAllowedError makeNotAllowedError(std::string url) { - return [url{std::move(url)}](const CanonPath & path) -> RestrictedPathError - { - if (nix::pathExists(path.abs())) - return RestrictedPathError("access to path '%s' is forbidden because it is not under Git control; maybe you should 'git add' it to the repository '%s'?", path, url); - else + return [url{std::move(url)}](const CanonPath & path) -> RestrictedPathError { + if (nix::pathExists(url + "/" + path.abs())) { + auto relativePath = path.rel(); // .makeRelative(CanonPath("/")); + + return RestrictedPathError( + "'%s' is not tracked by Git.\n" + "\n" + "To use '%s', stage it in the Git repository at '%s':\n" + "\n" + "git add %s", + relativePath, + relativePath, + url, + relativePath); + } else return RestrictedPathError("path '%s' does not exist in Git repository '%s'", path, url); }; } From d653225a24c80def265dc813a3a4fbab652763f1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 1 Apr 2025 15:14:20 +0200 Subject: [PATCH 2/8] Tweak error message (cherry picked from commit 277c29a64b379d66fe17a0c68260481a63fdcdd2) --- src/libfetchers/git.cc | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 5684583cd..6b82d9ae3 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -532,24 +532,20 @@ struct GitInputScheme : InputScheme return *head; } - static MakeNotAllowedError makeNotAllowedError(std::string url) + static MakeNotAllowedError makeNotAllowedError(std::filesystem::path repoPath) { - return [url{std::move(url)}](const CanonPath & path) -> RestrictedPathError { - if (nix::pathExists(url + "/" + path.abs())) { - auto relativePath = path.rel(); // .makeRelative(CanonPath("/")); - + return [repoPath{std::move(repoPath)}](const CanonPath & path) -> RestrictedPathError { + if (nix::pathExists(repoPath / path.rel())) return RestrictedPathError( - "'%s' is not tracked by Git.\n" + "File '%1%' in the repository %2% is not tracked by Git.\n" "\n" - "To use '%s', stage it in the Git repository at '%s':\n" + "To make it visible to Nix, run:\n" "\n" - "git add %s", - relativePath, - relativePath, - url, - relativePath); - } else - return RestrictedPathError("path '%s' does not exist in Git repository '%s'", path, url); + "git -C %2% add \"%1%\"", + path.rel(), + repoPath); + else + return RestrictedPathError("path '%s' does not exist in Git repository %s", path, repoPath); }; } @@ -757,7 +753,7 @@ struct GitInputScheme : InputScheme ref accessor = repo->getAccessor(repoInfo.workdirInfo, exportIgnore, - makeNotAllowedError(repoInfo.locationToArg())); + makeNotAllowedError(repoPath)); /* If the repo has submodules, return a mounted input accessor consisting of the accessor for the top-level repo and the From 2b4ddbbf47320bb9a0b01a74c1ec1d30687d79ee Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 1 Apr 2025 22:56:14 +0200 Subject: [PATCH 3/8] Make Git error messages more consistent (cherry picked from commit f15681df26bbbf246c226530d1ab814a172a7e87) --- src/libfetchers/git.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 6b82d9ae3..6f6fdd3a3 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -537,7 +537,7 @@ struct GitInputScheme : InputScheme return [repoPath{std::move(repoPath)}](const CanonPath & path) -> RestrictedPathError { if (nix::pathExists(repoPath / path.rel())) return RestrictedPathError( - "File '%1%' in the repository %2% is not tracked by Git.\n" + "Path '%1%' in the repository %2% is not tracked by Git.\n" "\n" "To make it visible to Nix, run:\n" "\n" @@ -545,7 +545,7 @@ struct GitInputScheme : InputScheme path.rel(), repoPath); else - return RestrictedPathError("path '%s' does not exist in Git repository %s", path, repoPath); + return RestrictedPathError("Path '%s' does not exist in Git repository %s.", path.rel(), repoPath); }; } From 7ef0eddfdcbe8c372b1de1ec7a9057ae9e6779bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 2 Apr 2025 17:59:30 +0200 Subject: [PATCH 4/8] decrease connect-timeout to 5s For people self-hosting caches that can be occasionally down, the default timeout is very long. This is annoying if you are trying to update your binary cache at the same time you are trying to update another machine. Same if cachix has one of its rare hiccups. We tested this value of 5s in srvos now for years and we like to travel around the world with shitty internet, so it should be still reasonable high. (cherry picked from commit bef91a618af5f1eb88757e52fad703c6145c3d5f) --- src/libstore/filetransfer.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index 0ecc7f376..47e767664 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -30,7 +30,7 @@ struct FileTransferSettings : Config {"binary-caches-parallel-connections"}}; Setting connectTimeout{ - this, 0, "connect-timeout", + this, 5, "connect-timeout", R"( The timeout (in seconds) for establishing connections in the binary cache substituter. It corresponds to `curl`’s From f1cb0e6ddbb8c52b8b80657ff5e84ca44d1cf6d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 1 Apr 2025 19:04:45 +0200 Subject: [PATCH 5/8] libgit2: use upstream version if possible we don't seem to use libgit2 for fetching via ssh, hence it shouldn't matter if it's using libssh or the ssh binary. (cherry picked from commit 0b61b758fb6c26f0cd3052ccbd442247c0bbb86d) --- packaging/dependencies.nix | 68 ++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 35 deletions(-) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 535b3ff37..0af670bfb 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -65,39 +65,37 @@ scope: { installPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.installPhase; }); - libgit2 = pkgs.libgit2.overrideAttrs ( - attrs: - { - cmakeFlags = attrs.cmakeFlags or [ ] ++ [ "-DUSE_SSH=exec" ]; - } - # libgit2: Nixpkgs 24.11 has < 1.9.0, which needs our patches - // lib.optionalAttrs (!lib.versionAtLeast pkgs.libgit2.version "1.9.0") { - nativeBuildInputs = - attrs.nativeBuildInputs or [ ] - # gitMinimal does not build on Windows. See packbuilder patch. - ++ lib.optionals (!stdenv.hostPlatform.isWindows) [ - # Needed for `git apply`; see `prePatch` - pkgs.buildPackages.gitMinimal - ]; - # Only `git apply` can handle git binary patches - prePatch = - attrs.prePatch or "" - + lib.optionalString (!stdenv.hostPlatform.isWindows) '' - patch() { - git apply - } - ''; - patches = - attrs.patches or [ ] - ++ [ - ./patches/libgit2-mempack-thin-packfile.patch - ] - # gitMinimal does not build on Windows, but fortunately this patch only - # impacts interruptibility - ++ lib.optionals (!stdenv.hostPlatform.isWindows) [ - # binary patch; see `prePatch` - ./patches/libgit2-packbuilder-callback-interruptible.patch - ]; - } - ); + libgit2 = + if lib.versionAtLeast pkgs.libgit2.version "1.9.0" then + pkgs.libgit2 + else + pkgs.libgit2.overrideAttrs (attrs: { + # libgit2: Nixpkgs 24.11 has < 1.9.0, which needs our patches + nativeBuildInputs = + attrs.nativeBuildInputs or [ ] + # gitMinimal does not build on Windows. See packbuilder patch. + ++ lib.optionals (!stdenv.hostPlatform.isWindows) [ + # Needed for `git apply`; see `prePatch` + pkgs.buildPackages.gitMinimal + ]; + # Only `git apply` can handle git binary patches + prePatch = + attrs.prePatch or "" + + lib.optionalString (!stdenv.hostPlatform.isWindows) '' + patch() { + git apply + } + ''; + patches = + attrs.patches or [ ] + ++ [ + ./patches/libgit2-mempack-thin-packfile.patch + ] + # gitMinimal does not build on Windows, but fortunately this patch only + # impacts interruptibility + ++ lib.optionals (!stdenv.hostPlatform.isWindows) [ + # binary patch; see `prePatch` + ./patches/libgit2-packbuilder-callback-interruptible.patch + ]; + }); } From a6a621c8ff378903863fb89bf95781884918065e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 1 Apr 2025 19:17:05 +0200 Subject: [PATCH 6/8] remove obsolete stdenv darwinMinVersion override we are more up-to-date now: nix-repl> stdenv.hostPlatform.darwinMinVersion "11.3" (cherry picked from commit d91310bb32b9efca2f1e1a6a767cbe5b0a7f072c) --- flake.nix | 2 +- packaging/dependencies.nix | 19 ------------------- 2 files changed, 1 insertion(+), 20 deletions(-) diff --git a/flake.nix b/flake.nix index e3cea08bf..c2d2385c5 100644 --- a/flake.nix +++ b/flake.nix @@ -170,7 +170,7 @@ { otherSplices = final.generateSplicesForMkScope "nixDependencies"; f = import ./packaging/dependencies.nix { - inherit inputs stdenv; + inherit stdenv; pkgs = final; }; }; diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 0af670bfb..f06b65dee 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -1,33 +1,14 @@ # These overrides are applied to the dependencies of the Nix components. { - # Flake inputs; used for sources - inputs, - # The raw Nixpkgs, not affected by this scope pkgs, stdenv, }: -let - prevStdenv = stdenv; -in - let inherit (pkgs) lib; - - stdenv = if prevStdenv.isDarwin && prevStdenv.isx86_64 then darwinStdenv else prevStdenv; - - # Fix the following error with the default x86_64-darwin SDK: - # - # error: aligned allocation function of type 'void *(std::size_t, std::align_val_t)' is only available on macOS 10.13 or newer - # - # Despite the use of the 10.13 deployment target here, the aligned - # allocation function Clang uses with this setting actually works - # all the way back to 10.6. - darwinStdenv = pkgs.overrideSDK prevStdenv { darwinMinVersion = "10.13"; }; - in scope: { inherit stdenv; From cb3c8bba7024c99d2556ced72128afe550599c5c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 2 Apr 2025 23:53:21 +0200 Subject: [PATCH 7/8] Revert "decrease connect-timeout to 5s (backport #12876)" --- src/libstore/filetransfer.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index 47e767664..0ecc7f376 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -30,7 +30,7 @@ struct FileTransferSettings : Config {"binary-caches-parallel-connections"}}; Setting connectTimeout{ - this, 5, "connect-timeout", + this, 0, "connect-timeout", R"( The timeout (in seconds) for establishing connections in the binary cache substituter. It corresponds to `curl`’s From fa140349b86c1ddbde0e21c22ff01af785997736 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 2 Apr 2025 21:22:43 +0200 Subject: [PATCH 8/8] symlink_exists: wrap exceptions into nix exception (cherry picked from commit 779687854f62adfdf448f4ccb37b33887f368621) --- src/libutil/file-system.cc | 12 +++++++++++- src/libutil/file-system.hh | 5 ++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 0adafc0e4..a609aa083 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -29,7 +29,17 @@ namespace nix { -namespace fs { using namespace std::filesystem; } +namespace fs { + using namespace std::filesystem; + + bool symlink_exists(const std::filesystem::path & path) { + try { + return std::filesystem::exists(std::filesystem::symlink_status(path)); + } catch (const std::filesystem::filesystem_error & e) { + throw SysError("cannot check existence of %1%", path); + } + } +} bool isAbsolute(PathView path) { diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh index 49d120cb7..61bfe6b30 100644 --- a/src/libutil/file-system.hh +++ b/src/libutil/file-system.hh @@ -134,6 +134,7 @@ bool pathExists(const Path & path); namespace fs { /** + * TODO: we may actually want to use pathExists instead of this function * ``` * symlink_exists(p) = std::filesystem::exists(std::filesystem::symlink_status(p)) * ``` @@ -142,9 +143,7 @@ namespace fs { * std::filesystem::exists(p) = std::filesystem::exists(std::filesystem::status(p)) * ``` */ -inline bool symlink_exists(const std::filesystem::path & path) { - return std::filesystem::exists(std::filesystem::symlink_status(path)); -} +bool symlink_exists(const std::filesystem::path & path); } // namespace fs