Merge remote-tracking branch 'origin/master' into lazy-trees

.github/stale.yml

@@ -1,10 +1,9 @@
 # Configuration for probot-stale - https://github.com/probot/stale
 daysUntilStale: 180
-daysUntilClose: 365
+daysUntilClose: false
 exemptLabels:
   - "critical"
+  - "never-stale"
 staleLabel: "stale"
-markComment: |
+markComment: false
-  I marked this as stale due to inactivity. → [More info](https://github.com/NixOS/nix/blob/master/.github/STALE-BOT.md)
+closeComment: false
-closeComment: |
-  I closed this issue due to inactivity. → [More info](https://github.com/NixOS/nix/blob/master/.github/STALE-BOT.md)

.github/workflows/ci.yml

@@ -4,6 +4,8 @@ on:
   pull_request:
   push:
 
+permissions: read-all
+
 jobs:
 
   tests:
@@ -28,6 +30,8 @@ jobs:
     - run: nix --experimental-features 'nix-command flakes' flake check -L
 
   check_cachix:
+    permissions:
+      contents: none
     name: Cachix secret present for installer tests
     runs-on: ubuntu-latest
     outputs:

.github/workflows/hydra_status.yml

@@ -1,8 +1,12 @@
 name: Hydra status
+
+permissions: read-all
+
 on:
   schedule:
     - cron: "12,42 * * * *"
   workflow_dispatch:
+
 jobs:
   check_hydra_status:
     name: Check Hydra status

doc/manual/src/SUMMARY.md.in

@@ -72,6 +72,7 @@
   - [CLI guideline](contributing/cli-guideline.md)
 - [Release Notes](release-notes/release-notes.md)
   - [Release X.Y (202?-??-??)](release-notes/rl-next.md)
+  - [Release 2.10 (2022-07-11)](release-notes/rl-2.10.md)
   - [Release 2.9 (2022-05-30)](release-notes/rl-2.9.md)
   - [Release 2.8 (2022-04-19)](release-notes/rl-2.8.md)
   - [Release 2.7 (2022-03-07)](release-notes/rl-2.7.md)

doc/manual/src/command-ref/nix-env.md

@@ -31,7 +31,7 @@ subcommand to be performed. These are documented below.
 Several commands, such as `nix-env -q` and `nix-env -i`, take a list of
 arguments that specify the packages on which to operate. These are
 extended regular expressions that must match the entire name of the
-package. (For details on regular expressions, see regex7.) The match is
+package. (For details on regular expressions, see **regex**(7).) The match is
 case-sensitive. The regular expression can optionally be followed by a
 dash and a version number; if omitted, any version of the package will
 match. Here are some examples:
@@ -412,7 +412,7 @@ The upgrade operation determines whether a derivation `y` is an upgrade
 of a derivation `x` by looking at their respective `name` attributes.
 The names (e.g., `gcc-3.3.1` are split into two parts: the package name
 (`gcc`), and the version (`3.3.1`). The version part starts after the
-first dash not followed by a letter. `x` is considered an upgrade of `y`
+first dash not followed by a letter. `y` is considered an upgrade of `x`
 if their package names match, and the version of `y` is higher than that
 of `x`.
 

doc/manual/src/release-notes/rl-2.10.md

@@ -0,0 +1,31 @@
+# Release 2.10 (2022-07-11)
+
+* `nix repl` now takes installables on the command line, unifying the usage
+  with other commands that use `--file` and `--expr`. Primary breaking change
+  is for the common usage of `nix repl '<nixpkgs>'` which can be recovered with
+  `nix repl --file '<nixpkgs>'` or `nix repl --expr 'import <nixpkgs>{}'`.
+
+  This is currently guarded by the `repl-flake` experimental feature.
+
+* A new function `builtins.traceVerbose` is available. It is similar
+  to `builtins.trace` if the `trace-verbose` setting is set to true,
+  and it is a no-op otherwise.
+
+* `nix search` has a new flag `--exclude` to filter out packages.
+
+* On Linux, if `/nix` doesn't exist and cannot be created and you're
+  not running as root, Nix will automatically use
+  `~/.local/share/nix/root` as a chroot store. This enables non-root
+  users to download the statically linked Nix binary and have it work
+  out of the box, e.g.
+
+  ```
+  # ~/nix run nixpkgs#hello
+  warning: '/nix' does not exists, so Nix will use '/home/ubuntu/.local/share/nix/root' as a chroot store
+  Hello, world!
+  ```
+
+* `flake-registry.json` is now fetched from `channels.nixos.org`.
+
+* Nix can now be built with LTO by passing `--enable-lto` to `configure`.
+  LTO is currently only supported when building with GCC.

doc/manual/src/release-notes/rl-next.md

@@ -1,4 +1,2 @@
 # Release X.Y (202?-??-??)
 
-* Nix can now be built with LTO by passing `--enable-lto` to `configure`.
-  LTO is currently only supported when building with GCC.

docker.nix

@@ -4,6 +4,8 @@
 , tag ? "latest"
 , channelName ? "nixpkgs"
 , channelURL ? "https://nixos.org/channels/nixpkgs-unstable"
+, extraPkgs ? []
+, maxLayers ? 100
 }:
 let
   defaultPkgs = with pkgs; [
@@ -23,7 +25,7 @@ let
     iana-etc
     git
     openssh
-  ];
+  ] ++ extraPkgs;
 
   users = {
 
@@ -229,7 +231,7 @@ let
 in
 pkgs.dockerTools.buildLayeredImageWithNixDb {
 
-  inherit name tag;
+  inherit name tag maxLayers;
 
   contents = [ baseSystem ];
 

flake.nix

@@ -54,7 +54,7 @@
         # we want most of the time and for backwards compatibility
         forAllSystems (system: stdenvsPackages.${system} // stdenvsPackages.${system}.stdenvPackages);
 
-      commonDeps = pkgs: with pkgs; rec {
+      commonDeps = { pkgs, isStatic ? false }: with pkgs; rec {
         # Use "busybox-sandbox-shell" if present,
         # if not (legacy) fallback and hope it's sufficient.
         sh = pkgs.busybox-sandbox-shell or (busybox.override {
@@ -85,6 +85,8 @@
           lib.optionals stdenv.isLinux [
             "--with-boost=${boost}/lib"
             "--with-sandbox-shell=${sh}/bin/busybox"
+          ]
+          ++ lib.optionals (stdenv.isLinux && !(isStatic && stdenv.system == "aarch64-linux")) [
             "LDFLAGS=-fuse-ld=gold"
           ];
 
@@ -174,7 +176,7 @@
             echo "file installer $out/install" >> $out/nix-support/hydra-build-products
           '';
 
-      testNixVersions = pkgs: client: daemon: with commonDeps pkgs; with pkgs.lib; pkgs.stdenv.mkDerivation {
+      testNixVersions = pkgs: client: daemon: with commonDeps { inherit pkgs; }; with pkgs.lib; pkgs.stdenv.mkDerivation {
         NIX_DAEMON_PACKAGE = daemon;
         NIX_CLIENT_PACKAGE = client;
         name =
@@ -285,7 +287,7 @@
           # Forward from the previous stage as we don’t want it to pick the lowdown override
           nixUnstable = prev.nixUnstable;
 
-          nix = with final; with commonDeps pkgs; currentStdenv.mkDerivation {
+          nix = with final; with commonDeps { inherit pkgs; }; currentStdenv.mkDerivation {
             name = "nix-${version}";
             inherit version;
 
@@ -452,7 +454,7 @@
         # Line coverage analysis.
         coverage =
           with nixpkgsFor.x86_64-linux;
-          with commonDeps pkgs;
+          with commonDeps { inherit pkgs; };
 
           releaseTools.coverageAnalysis {
             name = "nix-coverage-${version}";
@@ -563,7 +565,7 @@
       } // (nixpkgs.lib.optionalAttrs (builtins.elem system linux64BitSystems) {
         nix-static = let
           nixpkgs = nixpkgsFor.${system}.pkgsStatic;
-        in with commonDeps nixpkgs; nixpkgs.stdenv.mkDerivation {
+        in with commonDeps { pkgs = nixpkgs; isStatic = true; }; nixpkgs.stdenv.mkDerivation {
           name = "nix-${version}";
 
           src = self;
@@ -634,7 +636,7 @@
             inherit system crossSystem;
             overlays = [ self.overlays.default ];
           };
-        in with commonDeps nixpkgsCross; nixpkgsCross.stdenv.mkDerivation {
+        in with commonDeps { pkgs = nixpkgsCross; }; nixpkgsCross.stdenv.mkDerivation {
           name = "nix-${version}";
 
           src = self;
@@ -677,7 +679,7 @@
       devShells = forAllSystems (system:
         forAllStdenvs (stdenv:
           with nixpkgsFor.${system};
-          with commonDeps pkgs;
+          with commonDeps { inherit pkgs; };
           nixpkgsFor.${system}.${stdenv}.mkDerivation {
             name = "nix";
 

scripts/install-nix-from-closure.sh

@@ -148,7 +148,9 @@ if ! [ -w "$dest" ]; then
     exit 1
 fi
 
-mkdir -p "$dest/store"
+# The auto-chroot code in openFromNonUri() checks for the
+# non-existence of /nix/var/nix, so we need to create it here.
+mkdir -p "$dest/store" "$dest/var/nix"
 
 printf "copying Nix to %s..." "${dest}/store" >&2
 # Insert a newline if no progress is shown.

src/libcmd/command.hh

@@ -58,6 +58,7 @@ struct CopyCommand : virtual StoreCommand
 struct EvalCommand : virtual StoreCommand, MixEvalArgs
 {
     bool startReplOnEvalErrors = false;
+    bool ignoreExceptionsDuringTry = false;
 
     EvalCommand();
 
@@ -77,10 +78,16 @@ struct MixFlakeOptions : virtual Args, EvalCommand
 {
     flake::LockFlags lockFlags;
 
+    std::optional<std::string> needsFlakeInputCompletion = {};
+
     MixFlakeOptions();
 
-    virtual std::optional<FlakeRef> getFlakeRefForCompletion()
+    virtual std::vector<std::string> getFlakesForCompletion()
     { return {}; }
+
+    void completeFlakeInput(std::string_view prefix);
+
+    void completionHook() override;
 };
 
 struct SourceExprCommand : virtual Args, MixFlakeOptions
@@ -116,12 +123,13 @@ struct InstallablesCommand : virtual Args, SourceExprCommand
     InstallablesCommand();
 
     void prepare() override;
+    Installables load();
 
     virtual bool useDefaultInstallables() { return true; }
 
-    std::optional<FlakeRef> getFlakeRefForCompletion() override;
+    std::vector<std::string> getFlakesForCompletion() override;
 
-private:
+protected:
 
     std::vector<std::string> _installables;
 };
@@ -135,9 +143,9 @@ struct InstallableCommand : virtual Args, SourceExprCommand
 
     void prepare() override;
 
-    std::optional<FlakeRef> getFlakeRefForCompletion() override
+    std::vector<std::string> getFlakesForCompletion() override
     {
-        return parseFlakeRefWithFragment(_installable, absPath(".")).first;
+        return {_installable};
     }
 
 private:

src/libcmd/installables.cc

@@ -23,17 +23,6 @@
 
 namespace nix {
 
-void completeFlakeInputPath(
-    ref<EvalState> evalState,
-    const FlakeRef & flakeRef,
-    std::string_view prefix)
-{
-    auto flake = flake::getFlake(*evalState, flakeRef, true);
-    for (auto & input : flake.inputs)
-        if (hasPrefix(input.first, prefix))
-            completions->add(input.first);
-}
-
 MixFlakeOptions::MixFlakeOptions()
 {
     auto category = "Common flake-related options";
@@ -86,8 +75,7 @@ MixFlakeOptions::MixFlakeOptions()
             lockFlags.inputUpdates.insert(flake::parseInputPath(s));
         }},
         .completer = {[&](size_t, std::string_view prefix) {
-            if (auto flakeRef = getFlakeRefForCompletion())
+            needsFlakeInputCompletion = {std::string(prefix)};
-                completeFlakeInputPath(getEvalState(), *flakeRef, prefix);
         }}
     });
 
@@ -103,12 +91,10 @@ MixFlakeOptions::MixFlakeOptions()
                 parseFlakeRef(flakeRef, absPath("."), true));
         }},
         .completer = {[&](size_t n, std::string_view prefix) {
-            if (n == 0) {
+            if (n == 0)
-                if (auto flakeRef = getFlakeRefForCompletion())
+                needsFlakeInputCompletion = {std::string(prefix)};
-                    completeFlakeInputPath(getEvalState(), *flakeRef, prefix);
+            else if (n == 1)
-            } else if (n == 1) {
                 completeFlakeRef(getEvalState()->store, prefix);
-            }
         }}
     });
 
@@ -139,6 +125,24 @@ MixFlakeOptions::MixFlakeOptions()
     });
 }
 
+void MixFlakeOptions::completeFlakeInput(std::string_view prefix)
+{
+    auto evalState = getEvalState();
+    for (auto & flakeRefS : getFlakesForCompletion()) {
+        auto flakeRef = parseFlakeRefWithFragment(expandTilde(flakeRefS), absPath(".")).first;
+        auto flake = flake::getFlake(*evalState, flakeRef, true);
+        for (auto & input : flake.inputs)
+            if (hasPrefix(input.first, prefix))
+                completions->add(input.first);
+    }
+}
+
+void MixFlakeOptions::completionHook()
+{
+    if (auto & prefix = needsFlakeInputCompletion)
+        completeFlakeInput(*prefix);
+}
+
 SourceExprCommand::SourceExprCommand(bool supportReadOnlyMode)
 {
     addFlag({
@@ -1041,21 +1045,26 @@ InstallablesCommand::InstallablesCommand()
 
 void InstallablesCommand::prepare()
 {
+    installables = load();
+}
+
+Installables InstallablesCommand::load() {
+    Installables installables;
     if (_installables.empty() && useDefaultInstallables())
         // FIXME: commands like "nix profile install" should not have a
         // default, probably.
         _installables.push_back(".");
-    installables = parseInstallables(getStore(), _installables);
+    return parseInstallables(getStore(), _installables);
 }
 
-std::optional<FlakeRef> InstallablesCommand::getFlakeRefForCompletion()
+std::vector<std::string> InstallablesCommand::getFlakesForCompletion()
 {
     if (_installables.empty()) {
         if (useDefaultInstallables())
-            return parseFlakeRefWithFragment(".", absPath(".")).first;
+            return {"."};
         return {};
     }
-    return parseFlakeRefWithFragment(_installables.front(), absPath(".")).first;
+    return _installables;
 }
 
 InstallableCommand::InstallableCommand(bool supportReadOnlyMode)

src/libcmd/installables.hh

@@ -132,6 +132,8 @@ struct Installable
         const std::vector<std::shared_ptr<Installable>> & installables);
 };
 
+typedef std::vector<std::shared_ptr<Installable>> Installables;
+
 struct InstallableValue : Installable
 {
     ref<EvalState> state;

src/libcmd/repl.cc

@@ -22,6 +22,7 @@ extern "C" {
 #include "ansicolor.hh"
 #include "shared.hh"
 #include "eval.hh"
+#include "eval-cache.hh"
 #include "eval-inline.hh"
 #include "attr-path.hh"
 #include "store-api.hh"
@@ -54,6 +55,8 @@ struct NixRepl
     size_t debugTraceIndex;
 
     Strings loadedFiles;
+    typedef std::vector<std::pair<Value*,std::string>> AnnotatedValues;
+    std::function<AnnotatedValues()> getValues;
 
     const static int envSize = 32768;
     std::shared_ptr<StaticEnv> staticEnv;
@@ -63,13 +66,15 @@ struct NixRepl
 
     const Path historyFile;
 
-    NixRepl(ref<EvalState> state);
+    NixRepl(const Strings & searchPath, nix::ref<Store> store,ref<EvalState> state,
+            std::function<AnnotatedValues()> getValues);
     ~NixRepl();
-    void mainLoop(const std::vector<std::string> & files);
+    void mainLoop();
     StringSet completePrefix(const std::string & prefix);
     bool getLine(std::string & input, const std::string & prompt);
     StorePath getDerivationPath(Value & v);
     bool processLine(std::string line);
+
     void loadFile(const Path & path);
     void loadFlake(const std::string & flakeRef);
     void initEnv();
@@ -96,9 +101,11 @@ std::string removeWhitespace(std::string s)
 }
 
 
-NixRepl::NixRepl(ref<EvalState> state)
+NixRepl::NixRepl(const Strings & searchPath, nix::ref<Store> store, ref<EvalState> state,
+            std::function<NixRepl::AnnotatedValues()> getValues)
     : state(state)
     , debugTraceIndex(0)
+    , getValues(getValues)
     , staticEnv(new StaticEnv(false, state->staticBaseEnv.get()))
     , historyFile(getDataDir() + "/nix/repl-history")
 {
@@ -111,23 +118,20 @@ NixRepl::~NixRepl()
     write_history(historyFile.c_str());
 }
 
-std::string runNix(Path program, const Strings & args,
+void runNix(Path program, const Strings & args,
     const std::optional<std::string> & input = {})
 {
     auto subprocessEnv = getEnv();
     subprocessEnv["NIX_CONFIG"] = globalConfig.toKeyValue();
 
-    auto res = runProgram(RunOptions {
+    runProgram2(RunOptions {
         .program = settings.nixBinDir+ "/" + program,
         .args = args,
         .environment = subprocessEnv,
         .input = input,
     });
 
-    if (!statusOk(res.first))
+    return;
-        throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first));
-
-    return res.second;
 }
 
 static NixRepl * curRepl; // ugly
@@ -226,18 +230,12 @@ static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positi
     return out;
 }
 
-void NixRepl::mainLoop(const std::vector<std::string> & files)
+void NixRepl::mainLoop()
 {
     std::string error = ANSI_RED "error:" ANSI_NORMAL " ";
     notice("Welcome to Nix " + nixVersion + ". Type :? for help.\n");
 
-    if (!files.empty()) {
-        for (auto & i : files)
-            loadedFiles.push_back(i);
-    }
-
     loadFiles();
-    if (!loadedFiles.empty()) notice("");
 
     // Allow nix-repl specific settings in .inputrc
     rl_readline_name = "nix-repl";
@@ -749,7 +747,6 @@ bool NixRepl::processLine(std::string line)
     return true;
 }
 
-
 void NixRepl::loadFile(const Path & path)
 {
     loadedFiles.remove(path);
@@ -809,13 +806,15 @@ void NixRepl::loadFiles()
     Strings old = loadedFiles;
     loadedFiles.clear();
 
-    bool first = true;
     for (auto & i : old) {
-        if (!first) notice("");
-        first = false;
         notice("Loading '%1%'...", i);
         loadFile(i);
     }
+
+    for (auto & [i, what] : getValues()) {
+        notice("Loading installable '%1%'...", what);
+        addAttrsToScope(*i);
+    }
 }
 
 
@@ -1015,7 +1014,17 @@ void runRepl(
     ref<EvalState>evalState,
     const ValMap & extraEnv)
 {
-    auto repl = std::make_unique<NixRepl>(evalState);
+    auto getValues = [&]()->NixRepl::AnnotatedValues{
+        NixRepl::AnnotatedValues values;
+        return values;
+    };
+    const Strings & searchPath = {};
+    auto repl = std::make_unique<NixRepl>(
+            searchPath,
+            openStore(),
+            evalState,
+            getValues
+        );
 
     repl->initEnv();
 
@@ -1023,20 +1032,35 @@ void runRepl(
     for (auto & [name, value] : extraEnv)
         repl->addVarToScope(repl->state->symbols.create(name), *value);
 
-    repl->mainLoop({});
+    repl->mainLoop();
 }
 
-struct CmdRepl : StoreCommand, MixEvalArgs
+struct CmdRepl : InstallablesCommand
 {
-    std::vector<std::string> files;
+    CmdRepl(){
-
+        evalSettings.pureEval = false;
-    CmdRepl()
+    }
+    void prepare()
     {
-        expectArgs({
+        if (!settings.isExperimentalFeatureEnabled(Xp::ReplFlake) && !(file) && this->_installables.size() >= 1) {
-            .label = "files",
+            warn("future versions of Nix will require using `--file` to load a file");
-            .handler = {&files},
+            if (this->_installables.size() > 1)
-            .completer = completePath
+                warn("more than one input file is not currently supported");
-        });
+            auto filePath = this->_installables[0].data();
+            file = std::optional(filePath);
+            _installables.front() = _installables.back();
+            _installables.pop_back();
+        }
+        installables = InstallablesCommand::load();
+    }
+    std::vector<std::string> files;
+    Strings getDefaultFlakeAttrPaths() override
+    {
+        return {""};
+    }
+    virtual bool useDefaultInstallables() override
+    {
+        return file.has_value() or expr.has_value();
     }
 
     bool forceImpureByDefault() override
@@ -1058,12 +1082,37 @@ struct CmdRepl : StoreCommand, MixEvalArgs
 
     void run(ref<Store> store) override
     {
-        auto evalState = make_ref<EvalState>(searchPath, store);
+        auto state = getEvalState();
-
+        auto getValues = [&]()->NixRepl::AnnotatedValues{
-        auto repl = std::make_unique<NixRepl>(evalState);
+            auto installables = load();
+            NixRepl::AnnotatedValues values;
+            for (auto & installable: installables){
+                auto what = installable->what();
+                if (file){
+                    auto [val, pos] = installable->toValue(*state);
+                    auto what = installable->what();
+                    state->forceValue(*val, pos);
+                    auto autoArgs = getAutoArgs(*state);
+                    auto valPost = state->allocValue();
+                    state->autoCallFunction(*autoArgs, *val, *valPost);
+                    state->forceValue(*valPost, pos);
+                    values.push_back( {valPost, what });
+                } else {
+                    auto [val, pos] = installable->toValue(*state);
+                    values.push_back( {val, what} );
+                }
+            }
+            return values;
+        };
+        auto repl = std::make_unique<NixRepl>(
+            searchPath,
+            openStore(),
+            state,
+            getValues
+        );
         repl->autoArgs = getAutoArgs(*repl->state);
         repl->initEnv();
-        repl->mainLoop(files);
+        repl->mainLoop();
     }
 };
 

src/libexpr/eval-cache.cc

@@ -489,7 +489,7 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErro
                         return nullptr;
                     else if (std::get_if<failed_t>(&attr->second)) {
                         if (forceErrors)
-                            debug("reevaluating failed cached attribute '%s'");
+                            debug("reevaluating failed cached attribute '%s'", getAttrPathStr(name));
                         else
                             throw CachedEvalError("cached failure of attribute '%s'", getAttrPathStr(name));
                     } else

src/libexpr/eval.cc

@@ -481,9 +481,10 @@ EvalState::EvalState(
     )}
     , store(store)
     , buildStore(buildStore ? buildStore : store)
-    , debugRepl(0)
+    , debugRepl(nullptr)
     , debugStop(false)
     , debugQuit(false)
+    , trylevel(0)
     , regexCache(makeRegexCache())
 #if HAVE_BOEHMGC
     , valueAllocCache(std::allocate_shared<void *>(traceable_allocator<void *>(), nullptr))
@@ -788,7 +789,14 @@ void EvalState::runDebugRepl(const Error * error, const Env & env, const Expr &
         : nullptr;
 
     if (error)
-        printError("%s\n\n" ANSI_BOLD "Starting REPL to allow you to inspect the current state of the evaluator.\n" ANSI_NORMAL, error->what());
+    {
+        printError("%s\n\n", error->what());
+
+        if (trylevel > 0 && error->info().level != lvlInfo)
+            printError("This exception occurred in a 'tryEval' call. Use " ANSI_GREEN "--ignore-try" ANSI_NORMAL " to skip these.\n");
+
+        printError(ANSI_BOLD "Starting REPL to allow you to inspect the current state of the evaluator.\n" ANSI_NORMAL);
+    }
 
     auto se = getStaticEnv(expr);
     if (se) {

src/libexpr/eval.hh

@@ -128,6 +128,7 @@ public:
     void (* debugRepl)(ref<EvalState> es, const ValMap & extraEnv);
     bool debugStop;
     bool debugQuit;
+    int trylevel;
     std::list<DebugTrace> debugTraces;
     std::map<const Expr*, const std::shared_ptr<const StaticEnv>> exprEnvs;
     const std::shared_ptr<const StaticEnv> getStaticEnv(const Expr & expr) const
@@ -643,6 +644,15 @@ struct EvalSettings : Config
 
     Setting<bool> useEvalCache{this, true, "eval-cache",
         "Whether to use the flake evaluation cache."};
+
+    Setting<bool> ignoreExceptionsDuringTry{this, false, "ignore-try",
+        R"(
+          If set to true, ignore exceptions inside 'tryEval' calls when evaluating nix expressions in
+          debug mode (using the --debugger flag). By default the debugger will pause on all exceptions.
+        )"};
+
+    Setting<bool> traceVerbose{this, false, "trace-verbose",
+        "Whether `builtins.traceVerbose` should trace its first argument when evaluated."};
 };
 
 extern EvalSettings evalSettings;

src/libexpr/flake/flake.cc

@@ -324,6 +324,39 @@ LockedFlake lockFlake(
 
         std::vector<FlakeRef> parents;
 
+        std::function<void(
+            const InputPath & inputPathPrefix,
+            const FlakeInputs & flakeInputs
+            )>
+            checkFollowsDeclarations;
+
+        checkFollowsDeclarations = [&](
+            const InputPath & inputPathPrefix,
+            const FlakeInputs & flakeInputs
+        ) {
+            for (auto [inputPath, inputOverride] : overrides) {
+                auto inputPath2(inputPath);
+                auto follow = inputPath2.back();
+                inputPath2.pop_back();
+                if (inputPath2 == inputPathPrefix
+                    && flakeInputs.find(follow) == flakeInputs.end()
+                ) {
+                    std::string root;
+                    for (auto & element : inputPath2) {
+                        root.append(element);
+                        if (element != inputPath2.back()) {
+                            root.append(".inputs.");
+                        }
+                    }
+                    warn(
+                        "%s has a `follows'-declaration for a non-existent input %s!",
+                        root,
+                        follow
+                    );
+                }
+            }
+        };
+
         std::function<void(
             const FlakeInputs & flakeInputs,
             std::shared_ptr<Node> node,
@@ -356,6 +389,8 @@ LockedFlake lockFlake(
         {
             debug("computing lock file node '%s'", printInputPath(inputPathPrefix));
 
+            checkFollowsDeclarations(inputPathPrefix, flakeInputs);
+
             /* Get the overrides (i.e. attributes of the form
                'inputs.nixops.inputs.nixpkgs.url = ...'). */
             for (auto & [id, input] : flakeInputs) {

src/libexpr/primops.cc

@@ -853,6 +853,18 @@ static RegisterPrimOp primop_floor({
 static void prim_tryEval(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
     auto attrs = state.buildBindings(2);
+
+    /* increment state.trylevel, and decrement it when this function returns. */
+    MaintainCount trylevel(state.trylevel);
+
+    void (* savedDebugRepl)(ref<EvalState> es, const ValMap & extraEnv) = nullptr;
+    if (state.debugRepl && evalSettings.ignoreExceptionsDuringTry)
+    {
+        /* to prevent starting the repl from exceptions withing a tryEval, null it. */
+        savedDebugRepl = state.debugRepl;
+        state.debugRepl = nullptr;
+    }
+
     try {
         state.forceValue(*args[0], pos);
         attrs.insert(state.sValue, args[0]);
@@ -861,6 +873,11 @@ static void prim_tryEval(EvalState & state, const PosIdx pos, Value * * args, Va
         attrs.alloc(state.sValue).mkBool(false);
         attrs.alloc("success").mkBool(false);
     }
+
+    // restore the debugRepl pointer if we saved it earlier.
+    if (savedDebugRepl)
+        state.debugRepl = savedDebugRepl;
+
     v.mkAttrs(attrs);
 }
 
@@ -972,6 +989,15 @@ static RegisterPrimOp primop_trace({
 });
 
 
+/* Takes two arguments and evaluates to the second one. Used as the
+ * builtins.traceVerbose implementation when --trace-verbose is not enabled
+ */
+static void prim_second(EvalState & state, const PosIdx pos, Value * * args, Value & v)
+{
+    state.forceValue(*args[1], pos);
+    v = *args[1];
+}
+
 /*************************************************************
  * Derivations
  *************************************************************/
@@ -3952,6 +3978,18 @@ void EvalState::createBaseEnv()
         addPrimOp("__exec", 1, prim_exec);
     }
 
+    addPrimOp({
+        .fun = evalSettings.traceVerbose ? prim_trace : prim_second,
+        .arity = 2,
+        .name = "__traceVerbose",
+        .args = { "e1", "e2" },
+        .doc = R"(
+          Evaluate *e1* and print its abstract syntax representation on standard
+          error if `--trace-verbose` is enabled. Then return *e2*. This function
+          is useful for debugging.
+        )",
+    });
+
     /* Add a value containing the current Nix expression search path. */
     mkList(v, searchPath.size());
     int n = 0;

src/libstore/store-api.cc

@@ -1330,9 +1330,14 @@ std::shared_ptr<Store> openFromNonUri(const std::string & uri, const Store::Para
                we're not root, then automatically set up a chroot
                store in ~/.local/share/nix/root. */
             auto chrootStore = getDataDir() + "/nix/root";
-            if (!pathExists(chrootStore))
+            if (!pathExists(chrootStore)) {
+                try {
+                    createDirs(chrootStore);
+                } catch (Error & e) {
+                    return std::make_shared<LocalStore>(params);
+                }
                 warn("'/nix' does not exist, so Nix will use '%s' as a chroot store", chrootStore);
-            else
+            } else
                 debug("'/nix' does not exist, so Nix will use '%s' as a chroot store", chrootStore);
             Store::Params params2;
             params2["root"] = chrootStore;

src/libutil/args.cc

@@ -124,7 +124,7 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
         bool anyCompleted = false;
         for (size_t n = 0 ; n < flag.handler.arity; ++n) {
             if (pos == end) {
-                if (flag.handler.arity == ArityAny) break;
+                if (flag.handler.arity == ArityAny || anyCompleted) break;
                 throw UsageError("flag '%s' requires %d argument(s)", name, flag.handler.arity);
             }
             if (auto prefix = needsCompletion(*pos)) {
@@ -362,6 +362,14 @@ bool MultiCommand::processArgs(const Strings & args, bool finish)
         return Args::processArgs(args, finish);
 }
 
+void MultiCommand::completionHook()
+{
+    if (command)
+        return command->second->completionHook();
+    else
+        return Args::completionHook();
+}
+
 nlohmann::json MultiCommand::toJSON()
 {
     auto cmds = nlohmann::json::object();

src/libutil/args.hh

@@ -148,6 +148,11 @@ protected:
        argument (if any) have been processed. */
     virtual void initialFlagsProcessed() {}
 
+    /* Called after the command line has been processed if we need to generate
+       completions. Useful for commands that need to know the whole command line
+       in order to know what completions to generate. */
+    virtual void completionHook() { }
+
 public:
 
     void addFlag(Flag && flag);
@@ -223,6 +228,8 @@ public:
 
     bool processArgs(const Strings & args, bool finish) override;
 
+    void completionHook() override;
+
     nlohmann::json toJSON() override;
 };
 

src/libutil/experimental-features.cc

@@ -13,6 +13,7 @@ std::map<ExperimentalFeature, std::string> stringifiedXpFeatures = {
     { Xp::RecursiveNix, "recursive-nix" },
     { Xp::NoUrlLiterals, "no-url-literals" },
     { Xp::FetchClosure, "fetch-closure" },
+    { Xp::ReplFlake, "repl-flake" },
 };
 
 const std::optional<ExperimentalFeature> parseExperimentalFeature(const std::string_view & name)

src/libutil/experimental-features.hh

@@ -22,6 +22,7 @@ enum struct ExperimentalFeature
     RecursiveNix,
     NoUrlLiterals,
     FetchClosure,
+    ReplFlake,
 };
 
 /**

src/nix-build/nix-build.cc

@@ -257,11 +257,12 @@ static void main_nix_build(int argc, char * * argv)
 
     auto autoArgs = myArgs.getAutoArgs(*state);
 
+    auto autoArgsWithInNixShell = autoArgs;
     if (runEnv) {
-        auto newArgs = state->buildBindings(autoArgs->size() + 1);
+        auto newArgs = state->buildBindings(autoArgsWithInNixShell->size() + 1);
         newArgs.alloc("inNixShell").mkBool(true);
         for (auto & i : *autoArgs) newArgs.insert(i);
-        autoArgs = newArgs.finish();
+        autoArgsWithInNixShell = newArgs.finish();
     }
 
     if (packages) {
@@ -319,10 +320,39 @@ static void main_nix_build(int argc, char * * argv)
         Value vRoot;
         state->eval(e, vRoot);
 
+        std::function<bool(const Value & v)> takesNixShellAttr;
+        takesNixShellAttr = [&](const Value & v) {
+            if (!runEnv) {
+                return false;
+            }
+            bool add = false;
+            if (v.type() == nFunction && v.lambda.fun->hasFormals()) {
+                for (auto & i : v.lambda.fun->formals->formals) {
+                    if (state->symbols[i.name] == "inNixShell") {
+                        add = true;
+                        break;
+                    }
+                }
+            }
+            return add;
+        };
+
         for (auto & i : attrPaths) {
-            Value & v(*findAlongAttrPath(*state, i, *autoArgs, vRoot).first);
+            Value & v(*findAlongAttrPath(
+                *state,
+                i,
+                takesNixShellAttr(vRoot) ? *autoArgsWithInNixShell : *autoArgs,
+                vRoot
+            ).first);
             state->forceValue(v, [&]() { return v.determinePos(noPos); });
-            getDerivations(*state, v, "", *autoArgs, drvs, false);
+            getDerivations(
+                *state,
+                v,
+                "",
+                takesNixShellAttr(v) ? *autoArgsWithInNixShell : *autoArgs,
+                drvs,
+                false
+            );
         }
     }
 

src/nix/develop.cc

@@ -276,15 +276,25 @@ struct Common : InstallableCommand, MixProfile
         const BuildEnvironment & buildEnvironment,
         const Path & outputsDir = absPath(".") + "/outputs")
     {
+        // A list of colon-separated environment variables that should be
+        // prepended to, rather than overwritten, in order to keep the shell usable.
+        // Please keep this list minimal in order to avoid impurities.
+        static const char * const savedVars[] = {
+            "PATH",          // for commands
+            "XDG_DATA_DIRS", // for loadable completion
+        };
+
         std::ostringstream out;
 
         out << "unset shellHook\n";
 
-        out << "nix_saved_PATH=\"$PATH\"\n";
+        for (auto & var : savedVars)
+            out << fmt("nix_saved_%s=\"$%s\"\n", var, var);
 
         buildEnvironment.toBash(out, ignoreVars);
 
-        out << "PATH=\"$PATH:$nix_saved_PATH\"\n";
+        for (auto & var : savedVars)
+            out << fmt("%s=\"$%s:$nix_saved_%s\"\n", var, var, var);
 
         out << "export NIX_BUILD_TOP=\"$(mktemp -d -t nix-shell.XXXXXX)\"\n";
         for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"})

src/nix/flake.cc

@@ -50,9 +50,9 @@ public:
         return flake::lockFlake(*getEvalState(), getFlakeRef(), lockFlags);
     }
 
-    std::optional<FlakeRef> getFlakeRefForCompletion() override
+    std::vector<std::string> getFlakesForCompletion() override
     {
-        return getFlakeRef();
+        return {flakeUrl};
     }
 };
 
@@ -731,7 +731,8 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand
         PathSet context;
         auto templateDir = evalState->coerceToPath(noPos, templateDirAttr, context);
 
-        std::vector<CanonPath> files;
+        std::vector<CanonPath> changedFiles;
+        std::vector<CanonPath> conflictedFiles;
 
         std::function<void(const SourcePath & from, const CanonPath & to)> copyDir;
         copyDir = [&](const SourcePath & from, const CanonPath & to)
@@ -748,31 +749,41 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand
                     auto contents = from2.readFile();
                     if (pathExists(to2.abs())) {
                         auto contents2 = readFile(to2.abs());
-                        if (contents != contents2)
+                        if (contents != contents2) {
-                            throw Error("refusing to overwrite existing file '%s'", to2);
+                            printError("refusing to overwrite existing file '%s'\nplease merge it manually with '%s'", to2, from2);
+                            conflictedFiles.push_back(to2);
+                        } else {
+                            notice("skipping identical file: %s", from2);
+                        }
+                        continue;
                     } else
                         writeFile(to2.abs(), contents);
                 }
                 else if (st.type == InputAccessor::tSymlink) {
                     auto target = from2.readLink();
                     if (pathExists(to2.abs())) {
-                        if (readLink(to2.abs()) != target)
+                        if (readLink(to2.abs()) != target) {
-                            throw Error("refusing to overwrite existing symlink '%s'", to2);
+                            printError("refusing to overwrite existing file '%s'\n please merge it manually with '%s'", to2, from2);
+                            conflictedFiles.push_back(to2);
+                        } else {
+                            notice("skipping identical file: %s", from2);
+                        }
+                        continue;
                     } else
                         createSymlink(target, to2.abs());
                 }
                 else
                     throw Error("file '%s' has unsupported type", from2);
-                files.push_back(to2);
+                changedFiles.push_back(to2);
                 notice("wrote: %s", to2);
             }
         };
 
         copyDir(templateDir, CanonPath(flakeDir));
 
-        if (pathExists(flakeDir + "/.git")) {
+        if (!changedFiles.empty() && pathExists(flakeDir + "/.git")) {
             Strings args = { "-C", flakeDir, "add", "--intent-to-add", "--force", "--" };
-            for (auto & s : files) args.push_back(s.abs());
+            for (auto & s : changedFiles) args.push_back(s.abs());
             runProgram("git", true, args);
         }
 
@@ -780,6 +791,9 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand
             notice("\n");
             notice(renderMarkdownToTerminal(welcomeText->getString()));
         }
+
+        if (!conflictedFiles.empty())
+            throw Error("encountered %d conflicts - see above", conflictedFiles.size());
     }
 };
 

src/nix/main.cc

@@ -344,7 +344,10 @@ void mainWrapped(int argc, char * * argv)
         if (!completions) throw;
     }
 
-    if (completions) return;
+    if (completions) {
+        args.completionHook();
+        return;
+    }
 
     if (args.showVersion) {
         printVersion(programName);

src/nix/repl.md

@@ -24,10 +24,34 @@ R""(
 * Interact with Nixpkgs in the REPL:
 
   ```console
-  # nix repl '<nixpkgs>'
+  # nix repl --file example.nix
+  Loading Installable ''...
+  Added 3 variables.
 
-  Loading '<nixpkgs>'...
+  # nix repl --expr '{a={b=3;c=4;};}'
-  Added 12428 variables.
+  Loading Installable ''...
+  Added 1 variables.
+
+  # nix repl --expr '{a={b=3;c=4;};}' a
+  Loading Installable ''...
+  Added 1 variables.
+
+  # nix repl --extra_experimental_features 'flakes repl-flake' nixpkgs
+  Loading Installable 'flake:nixpkgs#'...
+  Added 5 variables.
+
+  nix-repl> legacyPackages.x86_64-linux.emacs.name
+  "emacs-27.1"
+
+  nix-repl> legacyPackages.x86_64-linux.emacs.name
+  "emacs-27.1"
+
+  nix-repl> :q
+
+  # nix repl --expr 'import <nixpkgs>{}'
+
+  Loading Installable ''...
+  Added 12439 variables.
 
   nix-repl> emacs.name
   "emacs-27.1"

tests/ca-shell.nix

@@ -1 +1 @@
-{ ... }@args: import ./shell.nix (args // { contentAddressed = true; })
+{ inNixShell ? false, ... }@args: import ./shell.nix (args // { contentAddressed = true; })

tests/flakes.sh

@@ -410,8 +410,10 @@ cat > $templatesDir/trivial/flake.nix <<EOF
   };
 }
 EOF
+echo a > $templatesDir/trivial/a
+echo b > $templatesDir/trivial/b
 
-git -C $templatesDir add flake.nix trivial/flake.nix
+git -C $templatesDir add flake.nix trivial/
 git -C $templatesDir commit -m 'Initial'
 
 nix flake check templates
@@ -426,6 +428,18 @@ nix flake show $flake7Dir
 nix flake show $flake7Dir --json | jq
 git -C $flake7Dir commit -a -m 'Initial'
 
+# Test 'nix flake init' with benign conflicts
+rm -rf $flake7Dir && mkdir $flake7Dir && git -C $flake7Dir init
+echo a > $flake7Dir/a
+(cd $flake7Dir && nix flake init) # check idempotence
+
+# Test 'nix flake init' with conflicts
+rm -rf $flake7Dir && mkdir $flake7Dir && git -C $flake7Dir init
+echo b > $flake7Dir/a
+pushd $flake7Dir
+(! nix flake init) |& grep "refusing to overwrite existing file '$flake7Dir/a'"
+popd
+
 # Test 'nix flake new'.
 rm -rf $flake6Dir
 nix flake new -t templates#trivial $flake6Dir
@@ -793,6 +807,8 @@ nix flake metadata $flakeFollowsA
 
 nix flake update $flakeFollowsA
 
+nix flake lock $flakeFollowsA
+
 oldLock="$(cat "$flakeFollowsA/flake.lock")"
 
 # Ensure that locking twice doesn't change anything
@@ -815,7 +831,6 @@ cat > $flakeFollowsA/flake.nix <<EOF
     inputs = {
         B = {
             url = "path:./flakeB";
-            inputs.nonFlake.follows = "D";
         };
         D.url = "path:./flakeD";
     };
@@ -851,3 +866,21 @@ nix store delete $(nix store add-path $badFlakeDir)
 
 [[ $(nix path-info      $(nix store add-path $flake1Dir)) =~ flake1 ]]
 [[ $(nix path-info path:$(nix store add-path $flake1Dir)) =~ simple ]]
+
+# Non-existant follows causes an error
+
+cat >$flakeFollowsA/flake.nix <<EOF
+{
+    description = "Flake A";
+    inputs.B = {
+        url = "path:./flakeB";
+        inputs.invalid.follows = "D";
+    };
+    inputs.D.url = "path:./flakeD";
+    outputs = { ... }: {};
+}
+EOF
+
+git -C $flakeFollowsA add flake.nix
+
+nix flake lock $flakeFollowsA 2>&1 | grep "warning: B has a \`follows'-declaration for a non-existant input invalid!"

tests/lang.sh

@@ -5,6 +5,8 @@ export NIX_REMOTE=dummy://
 
 nix-instantiate --eval -E 'builtins.trace "Hello" 123' 2>&1 | grep -q Hello
 nix-instantiate --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1
+nix-instantiate --trace-verbose --eval -E 'builtins.traceVerbose "Hello" 123' 2>&1 | grep -q Hello
+(! nix-instantiate --eval -E 'builtins.traceVerbose "Hello" 123' 2>&1 | grep -q Hello)
 (! nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1 | grep -q Hello)
 nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" (throw "Foo")' 2>&1 | grep -q Hello
 

tests/nix-shell.sh

@@ -102,3 +102,11 @@ source <(nix print-dev-env -f "$shellDotNix" shellDrv)
 [[ ${arr2[1]} = $'\n' ]]
 [[ ${arr2[2]} = $'x\ny' ]]
 [[ $(fun) = blabla ]]
+
+# Test nix-shell with ellipsis and no `inNixShell` argument (for backwards compat with old nixpkgs)
+cat >$TEST_ROOT/shell-ellipsis.nix <<EOF
+{ system ? "x86_64-linux", ... }@args:
+assert (!(args ? inNixShell));
+(import $shellDotNix { }).shellDrv
+EOF
+nix-shell $TEST_ROOT/shell-ellipsis.nix --run "true"

tests/repl.sh

@@ -55,15 +55,17 @@ testRepl
 testRepl --store "$TEST_ROOT/store?real=$NIX_STORE_DIR"
 
 testReplResponse () {
-    local response="$(nix repl <<< "$1")"
+    local commands="$1"; shift
-    echo "$response" | grep -qs "$2" \
+    local expectedResponse="$1"; shift
+    local response="$(nix repl "$@" <<< "$commands")"
+    echo "$response" | grep -qs "$expectedResponse" \
       || fail "repl command set:
 
-$1
+$commands
 
 does not respond with:
 
-$2
+$expectedResponse
 
 but with:
 
@@ -76,3 +78,48 @@ testReplResponse '
 :a { a = "2"; }
 "result: ${a}"
 ' "result: 2"
+
+testReplResponse '
+drvPath
+' '".*-simple.drv"' \
+$testDir/simple.nix
+
+testReplResponse '
+drvPath
+' '".*-simple.drv"' \
+--file $testDir/simple.nix --experimental-features 'ca-derivations'
+
+testReplResponse '
+drvPath
+' '".*-simple.drv"' \
+--file $testDir/simple.nix --extra-experimental-features 'repl-flake ca-derivations'
+
+mkdir -p flake && cat <<EOF > flake/flake.nix
+{
+    outputs = { self }: {
+        foo = 1;
+        bar.baz = 2;
+
+        changingThing = "beforeChange";
+    };
+}
+EOF
+testReplResponse '
+foo + baz
+' "3" \
+    ./flake ./flake\#bar --experimental-features 'flakes repl-flake'
+
+# Test the `:reload` mechansim with flakes:
+# - Eval `./flake#changingThing`
+# - Modify the flake
+# - Re-eval it
+# - Check that the result has changed
+replResult=$( (
+echo "changingThing"
+sleep 1 # Leave the repl the time to eval 'foo'
+sed -i 's/beforeChange/afterChange/' flake/flake.nix
+echo ":reload"
+echo "changingThing"
+) | nix repl ./flake --experimental-features 'flakes repl-flake')
+echo "$replResult" | grep -qs beforeChange
+echo "$replResult" | grep -qs afterChange