From 91508de3152b4448b44d9e48b749570077ff473f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 24 Feb 2025 17:46:43 +0100 Subject: [PATCH 1/4] nix flake archive: Recurse into relative path inputs We can't ignore them entirely, since we do want to archive their transitive inputs. Fixes #12438. (cherry picked from commit 14c9755462cc8ee61ba7a34da48fcfc34d3b110c) --- src/nix/flake.cc | 22 +++++++++++----------- tests/functional/flakes/common.sh | 14 +++++++++++--- tests/functional/flakes/relative-paths.sh | 14 ++++++++++++-- 3 files changed, 34 insertions(+), 16 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 87eaafd15..9259743f4 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1088,21 +1088,21 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun nlohmann::json jsonObj2 = json ? json::object() : nlohmann::json(nullptr); for (auto & [inputName, input] : node.inputs) { if (auto inputNode = std::get_if<0>(&input)) { - if ((*inputNode)->lockedRef.input.isRelative()) - continue; - auto storePath = - dryRun - ? (*inputNode)->lockedRef.input.computeStorePath(*store) - : (*inputNode)->lockedRef.input.fetchToStore(store).first; + std::optional storePath; + if (!(*inputNode)->lockedRef.input.isRelative()) { + storePath = + dryRun + ? (*inputNode)->lockedRef.input.computeStorePath(*store) + : (*inputNode)->lockedRef.input.fetchToStore(store).first; + sources.insert(*storePath); + } if (json) { auto & jsonObj3 = jsonObj2[inputName]; - jsonObj3["path"] = store->printStorePath(storePath); - sources.insert(std::move(storePath)); + if (storePath) + jsonObj3["path"] = store->printStorePath(*storePath); jsonObj3["inputs"] = traverse(**inputNode); - } else { - sources.insert(std::move(storePath)); + } else traverse(**inputNode); - } } } return jsonObj2; diff --git a/tests/functional/flakes/common.sh b/tests/functional/flakes/common.sh index b1c3988e3..06e414e9d 100644 --- a/tests/functional/flakes/common.sh +++ b/tests/functional/flakes/common.sh @@ -99,6 +99,16 @@ writeTrivialFlake() { EOF } +initGitRepo() { + local repo="$1" + local extraArgs="${2-}" + + # shellcheck disable=SC2086 # word splitting of extraArgs is intended + git -C "$repo" init $extraArgs + git -C "$repo" config user.email "foobar@example.com" + git -C "$repo" config user.name "Foobar" +} + createGitRepo() { local repo="$1" local extraArgs="${2-}" @@ -107,7 +117,5 @@ createGitRepo() { mkdir -p "$repo" # shellcheck disable=SC2086 # word splitting of extraArgs is intended - git -C "$repo" init $extraArgs - git -C "$repo" config user.email "foobar@example.com" - git -C "$repo" config user.name "Foobar" + initGitRepo "$repo" $extraArgs } diff --git a/tests/functional/flakes/relative-paths.sh b/tests/functional/flakes/relative-paths.sh index ac4b07eb2..3f7ca3f46 100644 --- a/tests/functional/flakes/relative-paths.sh +++ b/tests/functional/flakes/relative-paths.sh @@ -45,7 +45,7 @@ EOF [[ $(nix eval "$rootFlake?dir=sub1#y") = 6 ]] -git init "$rootFlake" +initGitRepo "$rootFlake" git -C "$rootFlake" add flake.nix sub0/flake.nix sub1/flake.nix [[ $(nix eval "$subflake1#y") = 6 ]] @@ -77,7 +77,17 @@ fi (! grep narHash "$subflake2/flake.lock") # Test `nix flake archive` with relative path flakes. -nix flake archive --json "$rootFlake" +git -C "$rootFlake" add flake.lock +git -C "$rootFlake" commit -a -m Foo + +json=$(nix flake archive --json "$rootFlake" --to "$TEST_ROOT/store2") +[[ $(echo "$json" | jq .inputs.sub0.inputs) = {} ]] +[[ -n $(echo "$json" | jq .path) ]] + +nix flake prefetch --out-link "$TEST_ROOT/result" "$rootFlake" +outPath=$(readlink "$TEST_ROOT/result") + +[ -e "$TEST_ROOT/store2/nix/store/$(basename "$outPath")" ] # Test circular relative path flakes. FIXME: doesn't work at the moment. if false; then From 827f760ad7e12dd006e834045d46645869cd4c74 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 24 Feb 2025 23:00:07 +0100 Subject: [PATCH 2/4] Fix test We didn't backport `nix flake prefetch --out-link`. --- tests/functional/flakes/relative-paths.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/functional/flakes/relative-paths.sh b/tests/functional/flakes/relative-paths.sh index 3f7ca3f46..9c0e6fd41 100644 --- a/tests/functional/flakes/relative-paths.sh +++ b/tests/functional/flakes/relative-paths.sh @@ -84,10 +84,10 @@ json=$(nix flake archive --json "$rootFlake" --to "$TEST_ROOT/store2") [[ $(echo "$json" | jq .inputs.sub0.inputs) = {} ]] [[ -n $(echo "$json" | jq .path) ]] -nix flake prefetch --out-link "$TEST_ROOT/result" "$rootFlake" -outPath=$(readlink "$TEST_ROOT/result") +#nix flake prefetch --out-link "$TEST_ROOT/result" "$rootFlake" +#outPath=$(readlink "$TEST_ROOT/result") -[ -e "$TEST_ROOT/store2/nix/store/$(basename "$outPath")" ] +#[ -e "$TEST_ROOT/store2/nix/store/$(basename "$outPath")" ] # Test circular relative path flakes. FIXME: doesn't work at the moment. if false; then From d8606f96eebc18947c5e5318162726c1ba225cc5 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 28 Feb 2025 17:40:32 +0100 Subject: [PATCH 3/4] packaging/everything.nix: Use a multi-output derivation This should fix a few packaging regressions. `dev` also includes a merged `includes/`, which may be helpful until inter-component includes are fixed properly. (cherry picked from commit 41085295ab3717b5ec8d348307dd4c9c1d378846) --- packaging/everything.nix | 200 ++++++++++++++++++++++++--------------- 1 file changed, 122 insertions(+), 78 deletions(-) diff --git a/packaging/everything.nix b/packaging/everything.nix index 0974a34df..c9ad26823 100644 --- a/packaging/everything.nix +++ b/packaging/everything.nix @@ -1,6 +1,7 @@ { lib, stdenv, + lndir, buildEnv, nix-util, @@ -38,7 +39,6 @@ nix-perl-bindings, testers, - runCommand, }: let @@ -119,92 +119,136 @@ let }; in -(buildEnv { - name = "nix-${nix-cli.version}"; - paths = [ - nix-cli - nix-manual.man +stdenv.mkDerivation (finalAttrs: { + pname = "nix"; + version = nix-cli.version; + + /** + This package uses a multi-output derivation, even though some outputs could + have been provided directly by the constituent component that provides it. + + This is because not all tooling handles packages composed of arbitrary + outputs yet. This includes nix itself, https://github.com/NixOS/nix/issues/6507. + + `devdoc` is also available, but not listed here, because this attribute is + not an output of the same derivation that provides `out`, `dev`, etc. + */ + outputs = [ + "out" + "dev" + "doc" + "man" ]; - meta.mainProgram = "nix"; -}).overrideAttrs - ( - finalAttrs: prevAttrs: { - doCheck = true; - doInstallCheck = true; + /** + Unpacking is handled in this package's constituent components + */ + dontUnpack = true; + /** + Building is handled in this package's constituent components + */ + dontBuild = true; - checkInputs = - [ - # Make sure the unit tests have passed - nix-util-tests.tests.run - nix-store-tests.tests.run - nix-expr-tests.tests.run - nix-fetchers-tests.tests.run - nix-flake-tests.tests.run + /** + `doCheck` controles whether tests are added as build gate for the combined package. + This includes both the unit tests and the functional tests, but not the + integration tests that run in CI (the flake's `hydraJobs` and some of the `checks`). + */ + doCheck = true; - # Make sure the functional tests have passed - nix-functional-tests + /** + `fixupPhase` currently doesn't understand that a symlink output isn't writable. - # dev bundle is ok - # (checkInputs must be empty paths??) - (runCommand "check-pkg-config" { checked = dev.tests.pkg-config; } "mkdir $out") - ] - ++ lib.optionals - (!stdenv.hostPlatform.isStatic && stdenv.buildPlatform.canExecute stdenv.hostPlatform) - [ - # Perl currently fails in static build - # TODO: Split out tests into a separate derivation? - nix-perl-bindings - ]; - passthru = prevAttrs.passthru // { - inherit (nix-cli) version; + We don't compile or link anything in this derivation, so fixups aren't needed. + */ + dontFixup = true; - /** - These are the libraries that are part of the Nix project. They are used - by the Nix CLI and other tools. + checkInputs = + [ + # Make sure the unit tests have passed + nix-util-tests.tests.run + nix-store-tests.tests.run + nix-expr-tests.tests.run + nix-fetchers-tests.tests.run + nix-flake-tests.tests.run - If you need to use these libraries in your project, we recommend to use - the `-c` C API libraries exclusively, if possible. + # Make sure the functional tests have passed + nix-functional-tests + ] + ++ lib.optionals + (!stdenv.hostPlatform.isStatic && stdenv.buildPlatform.canExecute stdenv.hostPlatform) + [ + # Perl currently fails in static build + # TODO: Split out tests into a separate derivation? + nix-perl-bindings + ]; - We also recommend that you build the complete package to ensure that the unit tests pass. - You could do this in CI, or by passing it in an unused environment variable. e.g in a `mkDerivation` call: + nativeBuildInputs = [ + lndir + ]; - ```nix - buildInputs = [ nix.libs.nix-util-c nix.libs.nix-store-c ]; - # Make sure the nix libs we use are ok - unusedInputsForTests = [ nix ]; - disallowedReferences = nix.all; - ``` - */ - inherit libs; + installPhase = + let + devPaths = lib.mapAttrsToList (_k: lib.getDev) finalAttrs.finalPackage.libs; + in + '' + mkdir -p $out $dev $doc $man - tests = prevAttrs.passthru.tests or { } // { - # TODO: create a proper fixpoint and: - # pkg-config = - # testers.hasPkgConfigModules { - # package = finalPackage; - # }; - }; + # Merged outputs + lndir ${nix-cli} $out + for lib in ${lib.escapeShellArgs devPaths}; do + lndir $lib $dev + done - /** - A derivation referencing the `dev` outputs of the Nix libraries. - */ - inherit dev; - inherit devdoc; - doc = nix-manual; - outputs = [ - "out" - "dev" - "devdoc" - "doc" - ]; - all = lib.attrValues ( - lib.genAttrs finalAttrs.passthru.outputs (outName: finalAttrs.finalPackage.${outName}) - ); + # Forwarded outputs + ln -s ${nix-manual} $doc + ln -s ${nix-manual.man} $man + ''; + + passthru = { + inherit (nix-cli) version; + + /** + These are the libraries that are part of the Nix project. They are used + by the Nix CLI and other tools. + + If you need to use these libraries in your project, we recommend to use + the `-c` C API libraries exclusively, if possible. + + We also recommend that you build the complete package to ensure that the unit tests pass. + You could do this in CI, or by passing it in an unused environment variable. e.g in a `mkDerivation` call: + + ```nix + buildInputs = [ nix.libs.nix-util-c nix.libs.nix-store-c ]; + # Make sure the nix libs we use are ok + unusedInputsForTests = [ nix ]; + disallowedReferences = nix.all; + ``` + */ + inherit libs; + + /** + Developer documentation for `nix`, in `share/doc/nix/{internal,external}-api/`. + + This is not a proper output; see `outputs` for context. + */ + inherit devdoc; + + /** + Extra tests that test this package, but do not run as part of the build. + See + */ + tests = { + pkg-config = testers.hasPkgConfigModules { + package = finalAttrs.finalPackage; }; - meta = prevAttrs.meta // { - description = "The Nix package manager"; - pkgConfigModules = dev.meta.pkgConfigModules; - }; - } - ) + }; + }; + + meta = { + mainProgram = "nix"; + description = "The Nix package manager"; + pkgConfigModules = dev.meta.pkgConfigModules; + }; + +}) From 87bf338612376fa15f6fa0f60ac7d4e79b612901 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 4 Mar 2025 17:21:56 +0100 Subject: [PATCH 4/4] packaging: Typo in setVersionLayer / preConfigure Apparently dead code in our use case, but good to keep nonetheless. Credit: ztzg in https://github.com/NixOS/nix/pull/12498#pullrequestreview-2658031853 (cherry picked from commit dcaea8cb1c6482f0c64649fb8dc99a020351b53a) --- packaging/components.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/components.nix b/packaging/components.nix index 9da864887..991d54241 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -51,7 +51,7 @@ let setVersionLayer = finalAttrs: prevAttrs: { preConfigure = - prevAttrs.prevAttrs or "" + prevAttrs.preConfigure or "" + # Update the repo-global .version file. # Symlink ./.version points there, but by default only workDir is writable.