Merge remote-tracking branch 'origin/master' into lazy-trees

doc/manual/src/release-notes/rl-2.12.md

@@ -17,12 +17,12 @@
   The `uid-range` [system feature] requires the [`auto-allocate-uids`]
   setting to be enabled.
 
-  [system feature]: (../command-ref/conf-file.md#conf-system-features)
+  [system feature]: ../command-ref/conf-file.md#conf-system-features
 
 * Nix can now automatically pick UIDs for builds, removing the need to
   create `nixbld*` user accounts. See [`auto-allocate-uids`].
 
-  [`auto-allocate-uids`]: (../command-ref/conf-file.md#conf-auto-allocate-uids)
+  [`auto-allocate-uids`]: ../command-ref/conf-file.md#conf-auto-allocate-uids
 
 * On Linux, Nix has experimental support for running builds inside a
   cgroup. See

doc/manual/src/release-notes/rl-next.md

@@ -14,3 +14,14 @@
       '<nixpkgs>' -A hello
   # NIX_PATH=nixpkgs=flake:nixpkgs nix-build '<nixpkgs>' -A hello
   ```
+
+* Allow explicitly selecting outputs in a store derivation installable, just like we can do with other sorts of installables.
+  For example,
+  ```shell-session
+  $ nix-build /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^dev`
+  ```
+  now works just as
+  ```shell-session
+  $ nix-build glibc^dev`
+  ```
+  does already.

flake.nix

@@ -425,6 +425,8 @@
         buildCross = nixpkgs.lib.genAttrs crossSystems (crossSystem:
           nixpkgs.lib.genAttrs ["x86_64-linux"] (system: self.packages.${system}."nix-${crossSystem}"));
 
+        buildNoGc = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix.overrideAttrs (a: { configureFlags = (a.configureFlags or []) ++ ["--enable-gc=no"];}));
+
         # Perl bindings for various platforms.
         perlBindings = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix.perl-bindings);
 

src/libcmd/common-eval-args.cc

@@ -79,7 +79,7 @@ MixEvalArgs::MixEvalArgs()
   branch in the `nixpkgs` repository.
 
   The URLs of the tarballs from the official `nixos.org` channels
-  (see [the manual page for `nix-channel`](nix-channel.md)) can be
+  (see [the manual page for `nix-channel`](../nix-channel.md)) can be
   abbreviated as `channel:<channel-name>`.  For instance, the
   following two flags are equivalent:
 
@@ -88,7 +88,7 @@ MixEvalArgs::MixEvalArgs()
   -I nixpkgs=https://nixos.org/channels/nixos-21.05/nixexprs.tar.xz
   ```
 
-  You can also fetch source trees using flake URLs and add them to the
+  You can also fetch source trees using [flake URLs](./nix3-flake.md#url-like-syntax) and add them to the
   search path. For instance,
 
   ```

src/libcmd/installables.cc

@@ -400,44 +400,56 @@ static StorePath getDeriver(
 struct InstallableStorePath : Installable
 {
     ref<Store> store;
-    StorePath storePath;
+    DerivedPath req;
 
     InstallableStorePath(ref<Store> store, StorePath && storePath)
-        : store(store), storePath(std::move(storePath)) { }
+        : store(store),
+          req(storePath.isDerivation()
+              ? (DerivedPath) DerivedPath::Built {
+                  .drvPath = std::move(storePath),
+                  .outputs = {},
+              }
+              : (DerivedPath) DerivedPath::Opaque {
+                  .path = std::move(storePath),
+              })
+    { }
 
-    std::string what() const override { return store->printStorePath(storePath); }
+    InstallableStorePath(ref<Store> store, DerivedPath && req)
+        : store(store), req(std::move(req))
+    { }
+
+    std::string what() const override
+    {
+        return req.to_string(*store);
+    }
 
     DerivedPaths toDerivedPaths() override
     {
-        if (storePath.isDerivation()) {
+        return { req };
-            auto drv = store->readDerivation(storePath);
-            return {
-                DerivedPath::Built {
-                    .drvPath = storePath,
-                    .outputs = drv.outputNames(),
-                }
-            };
-        } else {
-            return {
-                DerivedPath::Opaque {
-                    .path = storePath,
-                }
-            };
-        }
     }
 
     StorePathSet toDrvPaths(ref<Store> store) override
     {
-        if (storePath.isDerivation()) {
+        return std::visit(overloaded {
-            return {storePath};
+            [&](const DerivedPath::Built & bfd) -> StorePathSet {
-        } else {
+                return { bfd.drvPath };
-            return {getDeriver(store, *this, storePath)};
+            },
-        }
+            [&](const DerivedPath::Opaque & bo) -> StorePathSet {
+                return { getDeriver(store, *this, bo.path) };
+            },
+        }, req.raw());
     }
 
     std::optional<StorePath> getStorePath() override
     {
-        return storePath;
+        return std::visit(overloaded {
+            [&](const DerivedPath::Built & bfd) {
+                return bfd.drvPath;
+            },
+            [&](const DerivedPath::Opaque & bo) {
+                return bo.path;
+            },
+        }, req.raw());
     }
 };
 
@@ -804,7 +816,22 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
         for (auto & s : ss) {
             std::exception_ptr ex;
 
-            if (s.find('/') != std::string::npos) {
+            auto found = s.rfind('^');
+            if (found != std::string::npos) {
+                try {
+                    result.push_back(std::make_shared<InstallableStorePath>(
+                        store,
+                        DerivedPath::Built::parse(*store, s.substr(0, found), s.substr(found + 1))));
+                    continue;
+                } catch (BadStorePath &) {
+                } catch (...) {
+                    if (!ex)
+                        ex = std::current_exception();
+                }
+            }
+
+            found = s.find('/');
+            if (found != std::string::npos) {
                 try {
                     result.push_back(std::make_shared<InstallableStorePath>(store, store->followLinksToStorePath(s)));
                     continue;

src/libexpr/eval.cc

@@ -46,7 +46,7 @@ static char * allocString(size_t size)
 #if HAVE_BOEHMGC
     t = (char *) GC_MALLOC_ATOMIC(size);
 #else
-    t = malloc(size);
+    t = (char *) malloc(size);
 #endif
     if (!t) throw std::bad_alloc();
     return t;
@@ -494,9 +494,6 @@ EvalState::EvalState(
 #if HAVE_BOEHMGC
     , valueAllocCache(std::allocate_shared<void *>(traceable_allocator<void *>(), nullptr))
     , env1AllocCache(std::allocate_shared<void *>(traceable_allocator<void *>(), nullptr))
-#else
-    , valueAllocCache(std::make_shared<void *>(nullptr))
-    , env1AllocCache(std::make_shared<void *>(nullptr))
 #endif
     , virtualPathMarker(settings.nixStore + "/virtual00000000000000000")
     , baseEnv(allocEnv(128))
@@ -1617,7 +1614,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                             ? concatStrings("'", symbols[lambda.name], "'")
                             : "anonymous lambda"));
                     if (pos != noPos)
-                        addErrorTrace(e, pos, "from call site", "");
+                        addErrorTrace(e, pos, "while evaluating call site%s", "");
                 }
                 throw;
             }

src/libexpr/flake/flake.cc

@@ -89,7 +89,7 @@ static FlakeInput parseFlakeInput(
         } catch (Error & e) {
             e.addTrace(
                 state.positions[attr.pos],
-                hintfmt("in flake attribute '%s'", state.symbols[attr.name]));
+                hintfmt("while evaluating flake attribute '%s'", state.symbols[attr.name]));
             throw;
         }
     }
@@ -98,7 +98,7 @@ static FlakeInput parseFlakeInput(
         try {
             input.ref = FlakeRef::fromAttrs(attrs);
         } catch (Error & e) {
-            e.addTrace(state.positions[pos], hintfmt("in flake input"));
+            e.addTrace(state.positions[pos], hintfmt("while evaluating flake input"));
             throw;
         }
     else {

src/libstore/derived-path.cc

@@ -78,15 +78,16 @@ DerivedPath::Opaque DerivedPath::Opaque::parse(const Store & store, std::string_
     return {store.parseStorePath(s)};
 }
 
-DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_view s)
+DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_view drvS, std::string_view outputsS)
 {
-    size_t n = s.find("!");
+    auto drvPath = store.parseStorePath(drvS);
-    assert(n != s.npos);
-    auto drvPath = store.parseStorePath(s.substr(0, n));
-    auto outputsS = s.substr(n + 1);
     std::set<std::string> outputs;
-    if (outputsS != "*")
+    if (outputsS != "*") {
         outputs = tokenizeString<std::set<std::string>>(outputsS, ",");
+        if (outputs.empty())
+            throw Error(
+                 "Explicit list of wanted outputs '%s' must not be empty. Consider using '*' as a wildcard meaning all outputs if no output in particular is wanted.", outputsS);
+	}
     return {drvPath, outputs};
 }
 
@@ -95,7 +96,7 @@ DerivedPath DerivedPath::parse(const Store & store, std::string_view s)
     size_t n = s.find("!");
     return n == s.npos
         ? (DerivedPath) DerivedPath::Opaque::parse(store, s)
-        : (DerivedPath) DerivedPath::Built::parse(store, s);
+        : (DerivedPath) DerivedPath::Built::parse(store, s.substr(0, n), s.substr(n + 1));
 }
 
 RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const

src/libstore/derived-path.hh

@@ -47,7 +47,7 @@ struct DerivedPathBuilt {
     std::set<std::string> outputs;
 
     std::string to_string(const Store & store) const;
-    static DerivedPathBuilt parse(const Store & store, std::string_view);
+    static DerivedPathBuilt parse(const Store & store, std::string_view, std::string_view);
     nlohmann::json toJSON(ref<Store> store) const;
 
     bool operator < (const DerivedPathBuilt & b) const

src/libstore/globals.hh

@@ -281,7 +281,10 @@ public:
           `NIX_REMOTE` is empty, the uid under which the Nix daemon runs if
           `NIX_REMOTE` is `daemon`). Obviously, this should not be used in
           multi-user settings with untrusted users.
-        )"};
+
+          Defaults to `nixbld` when running as root, *empty* otherwise.
+        )",
+        {}, false};
 
     Setting<bool> autoAllocateUids{this, false, "auto-allocate-uids",
         R"(

src/libstore/lock.cc

@@ -185,7 +185,7 @@ std::unique_ptr<UserLock> acquireUserLock(uid_t nrIds, bool useChroot)
 bool useBuildUsers()
 {
     #if __linux__
-    static bool b = (settings.buildUsersGroup != "" || settings.startId.get() != 0) && getuid() == 0;
+    static bool b = (settings.buildUsersGroup != "" || settings.autoAllocateUids) && getuid() == 0;
     return b;
     #elif __APPLE__
     static bool b = settings.buildUsersGroup != "" && getuid() == 0;

src/libutil/error.cc

@@ -197,10 +197,30 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s
         prefix += ":" ANSI_NORMAL " ";
 
     std::ostringstream oss;
-    oss << einfo.msg << "\n";
 
     auto noSource = ANSI_ITALIC " (source not available)" ANSI_NORMAL "\n";
 
+    // traces
+    if (showTrace && !einfo.traces.empty()) {
+        for (const auto & trace : einfo.traces) {
+            oss << "\n" << "… " << trace.hint.str() << "\n";
+
+            if (trace.pos) {
+                oss << "\n" << ANSI_BLUE << "at " ANSI_WARNING << *trace.pos << ANSI_NORMAL << ":";
+
+                if (auto loc = trace.pos->getCodeLines()) {
+                    oss << "\n";
+                    printCodeLines(oss, "", *trace.pos, *loc);
+                    oss << "\n";
+                } else
+                    oss << noSource;
+            }
+        }
+        oss << "\n" << prefix;
+    }
+
+    oss << einfo.msg << "\n";
+
     if (einfo.errPos) {
         oss << "\n" << ANSI_BLUE << "at " ANSI_WARNING << *einfo.errPos << ANSI_NORMAL << ":";
 
@@ -219,24 +239,6 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s
             "?" << std::endl;
     }
 
-    // traces
-    if (showTrace && !einfo.traces.empty()) {
-        for (auto iter = einfo.traces.rbegin(); iter != einfo.traces.rend(); ++iter) {
-            oss << "\n" << "… " << iter->hint.str() << "\n";
-
-            if (iter->pos) {
-                oss << "\n" << ANSI_BLUE << "at " ANSI_WARNING << *iter->pos << ANSI_NORMAL << ":";
-
-                if (auto loc = iter->pos->getCodeLines()) {
-                    oss << "\n";
-                    printCodeLines(oss, "", *iter->pos, *loc);
-                    oss << "\n";
-                } else
-                    oss << noSource;
-            }
-        }
-    }
-
     out << indent(prefix, std::string(filterANSIEscapes(prefix, true).size(), ' '), chomp(oss.str()));
 
     return out;

src/nix/flake.md

@@ -18,7 +18,10 @@ values such as packages or NixOS modules provided by the flake).
 Flake references (*flakerefs*) are a way to specify the location of a
 flake. These have two different forms:
 
-* An attribute set representation, e.g.
+
+## Attribute set representation
+
+Example:
 
 ```nix
 {
@@ -31,7 +34,9 @@ flake. These have two different forms:
 The only required attribute is `type`. The supported types are
 listed below.
 
-* A URL-like syntax, e.g.
+## URL-like syntax
+
+Example:
 
 ```
 github:NixOS/nixpkgs
@@ -62,7 +67,7 @@ flake. These have two different forms:
 };
 ```
 
-## Examples
+### Examples
 
 Here are some examples of flake references in their URL-like representation:
 

src/nix/nix.md

@@ -164,6 +164,13 @@ operate are determined as follows:
   …
   ```
 
+  and likewise, using a store path to a "drv" file to specify the derivation:
+
+  ```console
+  # nix build '/nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^dev,static'
+  …
+  ```
+
 * You can also specify that *all* outputs should be used using the
   syntax *installable*`^*`. For example, the following shows the size
   of all outputs of the `glibc` package in the binary cache:
@@ -177,6 +184,12 @@ operate are determined as follows:
   /nix/store/q6580lr01jpcsqs4r5arlh4ki2c1m9rv-glibc-2.33-123-dev             44200560
   ```
 
+  and likewise, using a store path to a "drv" file to specify the derivation:
+
+  ```console
+  # nix path-info -S '/nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^*'
+  …
+  ```
 * If you didn't specify the desired outputs, but the derivation has an
   attribute `meta.outputsToInstall`, Nix will use those outputs. For
   example, since the package `nixpkgs#libxml2` has this attribute:
@@ -189,6 +202,9 @@ operate are determined as follows:
   a command like `nix shell nixpkgs#libxml2` will provide only those
   two outputs by default.
 
+  Note that a store derivation (given by `.drv` file store path) doesn't have
+  any attributes like `meta`, and thus this case doesn't apply to it.
+
 * Otherwise, Nix will use all outputs of the derivation.
 
 # Nix stores

tests/build.sh

@@ -8,13 +8,15 @@ set -o pipefail
 nix build -f multiple-outputs.nix --json a b --no-link | jq --exit-status '
   (.[0] |
     (.drvPath | match(".*multiple-outputs-a.drv")) and
-    (.outputs | keys | length == 2) and
+    (.outputs |
-    (.outputs.first | match(".*multiple-outputs-a-first")) and
+      (keys | length == 2) and
-    (.outputs.second | match(".*multiple-outputs-a-second")))
+      (.first | match(".*multiple-outputs-a-first")) and
+      (.second | match(".*multiple-outputs-a-second"))))
   and (.[1] |
     (.drvPath | match(".*multiple-outputs-b.drv")) and
-    (.outputs | keys | length == 1) and
+    (.outputs |
-    (.outputs.out | match(".*multiple-outputs-b")))
+      (keys | length == 1) and
+      (.out | match(".*multiple-outputs-b"))))
 '
 
 # Test output selection using the '^' syntax.
@@ -56,6 +58,48 @@ nix build -f multiple-outputs.nix --json 'e^*' --no-link | jq --exit-status '
     (.outputs | keys == ["a", "b", "c"]))
 '
 
+# Test building from raw store path to drv not expression.
+
+drv=$(nix eval -f multiple-outputs.nix --raw a.drvPath)
+if nix build "$drv^not-an-output" --no-link --json; then
+    fail "'not-an-output' should fail to build"
+fi
+
+if nix build "$drv^" --no-link --json; then
+    fail "'empty outputs list' should fail to build"
+fi
+
+if nix build "$drv^*nope" --no-link --json; then
+    fail "'* must be entire string' should fail to build"
+fi
+
+nix build "$drv^first" --no-link --json | jq --exit-status '
+  (.[0] |
+    (.drvPath | match(".*multiple-outputs-a.drv")) and
+    (.outputs |
+      (keys | length == 1) and
+      (.first | match(".*multiple-outputs-a-first")) and
+      (has("second") | not)))
+'
+
+nix build "$drv^first,second" --no-link --json | jq --exit-status '
+  (.[0] |
+    (.drvPath | match(".*multiple-outputs-a.drv")) and
+    (.outputs |
+      (keys | length == 2) and
+      (.first | match(".*multiple-outputs-a-first")) and
+	  (.second | match(".*multiple-outputs-a-second"))))
+'
+
+nix build "$drv^*" --no-link --json | jq --exit-status '
+  (.[0] |
+    (.drvPath | match(".*multiple-outputs-a.drv")) and
+    (.outputs |
+      (keys | length == 2) and
+      (.first | match(".*multiple-outputs-a-first")) and
+	  (.second | match(".*multiple-outputs-a-second"))))
+'
+
 # Make sure that `--impure` works (regression test for https://github.com/NixOS/nix/issues/6488)
 nix build --impure -f multiple-outputs.nix --json e --no-link | jq --exit-status '
   (.[0] |