Merge remote-tracking branch 'origin/master' into no-flake-substitution

src/build-remote/build-remote.cc

@@ -11,11 +11,13 @@
 
 #include "machines.hh"
 #include "shared.hh"
+#include "plugin.hh"
 #include "pathlocks.hh"
 #include "globals.hh"
 #include "serialise.hh"
 #include "build-result.hh"
 #include "store-api.hh"
+#include "strings.hh"
 #include "derivations.hh"
 #include "local-store.hh"
 #include "legacy.hh"
@@ -37,7 +39,7 @@ static std::string currentLoad;
 
 static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot)
 {
-    return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true);
+    return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri.render()), slot), true);
 }
 
 static bool allSupportedLocally(Store & store, const std::set<std::string>& requiredFeatures) {
@@ -135,7 +137,7 @@ static int main_build_remote(int argc, char * * argv)
                 Machine * bestMachine = nullptr;
                 uint64_t bestLoad = 0;
                 for (auto & m : machines) {
-                    debug("considering building on remote machine '%s'", m.storeUri);
+                    debug("considering building on remote machine '%s'", m.storeUri.render());
 
                     if (m.enabled &&
                         m.systemSupported(neededSystem) &&
@@ -232,17 +234,16 @@ static int main_build_remote(int argc, char * * argv)
                 lock = -1;
 
                 try {
+                    storeUri = bestMachine->storeUri.render();
 
-                    Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri));
+                    Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", storeUri));
 
                     sshStore = bestMachine->openStore();
                     sshStore->connect();
-                    storeUri = bestMachine->storeUri;
-
                 } catch (std::exception & e) {
                     auto msg = chomp(drainFD(5, false));
                     printError("cannot build on '%s': %s%s",
-                        bestMachine->storeUri, e.what(),
+                        storeUri, e.what(),
                         msg.empty() ? "" : ": " + msg);
                     bestMachine->enabled = false;
                     continue;
@@ -262,7 +263,20 @@ connected:
         auto inputs = readStrings<PathSet>(source);
         auto wantedOutputs = readStrings<StringSet>(source);
 
-        AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true);
+        AutoCloseFD uploadLock;
+        {
+            auto setUpdateLock = [&](auto && fileName){
+                uploadLock = openLockFile(currentLoad + "/" + escapeUri(fileName) + ".upload-lock", true);
+            };
+            try {
+                setUpdateLock(storeUri);
+            } catch (SysError & e) {
+                if (e.errNo != ENAMETOOLONG) throw;
+                // Try again hashing the store URL so we have a shorter path
+                auto h = hashString(HashAlgorithm::MD5, storeUri);
+                setUpdateLock(h.to_string(HashFormat::Base64, false));
+            }
+        }
 
         {
             Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri));

src/external-api-docs/.gitignore

@@ -0,0 +1,3 @@
+/doxygen.cfg
+/html
+/latex

src/external-api-docs/.version

@@ -0,0 +1 @@
+../../.version

src/external-api-docs/README.md

@@ -0,0 +1,121 @@
+# Getting started
+
+> **Warning** These bindings are **experimental**, which means they can change
+> at any time or be removed outright; nevertheless the plan is to provide a
+> stable external C API to the Nix language and the Nix store.
+
+The language library allows evaluating Nix expressions and interacting with Nix
+language values. The Nix store API is still rudimentary, and only allows
+initialising and connecting to a store for the Nix language evaluator to
+interact with.
+
+Currently there are two ways to interface with the Nix language evaluator
+programmatically:
+
+1. Embedding the evaluator
+2. Writing language plug-ins
+
+Embedding means you link the Nix C libraries in your program and use them from
+there. Adding a plug-in means you make a library that gets loaded by the Nix
+language evaluator, specified through a configuration option.
+
+Many of the components and mechanisms involved are not yet documented, therefore
+please refer to the [Nix source code](https://github.com/NixOS/nix/) for
+details. Additions to in-code documentation and the reference manual are highly
+appreciated.
+
+The following examples, for simplicity, don't include error handling. See the
+[Handling errors](@ref errors) section for more information.
+
+# Embedding the Nix Evaluator{#nix_evaluator_example}
+
+In this example we programmatically start the Nix language evaluator with a
+dummy store (that has no store paths and cannot be written to), and evaluate the
+Nix expression `builtins.nixVersion`.
+
+**main.c:**
+
+```C
+#include <nix_api_util.h>
+#include <nix_api_expr.h>
+#include <nix_api_value.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+// NOTE: This example lacks all error handling. Production code must check for
+// errors, as some return values will be undefined.
+
+void my_get_string_cb(const char * start, unsigned int n, void * user_data)
+{
+    *((char **) user_data) = strdup(start);
+}
+
+int main()
+{
+    nix_libexpr_init(NULL);
+
+    Store * store = nix_store_open(NULL, "dummy://", NULL);
+    EvalState * state = nix_state_create(NULL, NULL, store); // empty search path (NIX_PATH)
+    Value * value = nix_alloc_value(NULL, state);
+
+    nix_expr_eval_from_string(NULL, state, "builtins.nixVersion", ".", value);
+    nix_value_force(NULL, state, value);
+
+    char * version;
+    nix_get_string(NULL, value, my_get_string_cb, &version);
+    printf("Nix version: %s\n", version);
+
+    free(version);
+    nix_gc_decref(NULL, value);
+    nix_state_free(state);
+    nix_store_free(store);
+    return 0;
+}
+```
+
+**Usage:**
+
+```ShellSession
+$ gcc main.c $(pkg-config nix-expr-c --libs --cflags) -o main
+$ ./main
+Nix version: 2.17
+```
+
+# Writing a Nix language plug-in
+
+In this example we add a custom primitive operation (_primop_) to `builtins`. It
+will increment the argument if it is an integer and throw an error otherwise.
+
+**plugin.c:**
+
+```C
+#include <nix_api_util.h>
+#include <nix_api_expr.h>
+#include <nix_api_value.h>
+
+void increment(void* user_data, nix_c_context* ctx, EvalState* state, Value** args, Value* v) {
+    nix_value_force(NULL, state, args[0]);
+    if (nix_get_type(NULL, args[0]) == NIX_TYPE_INT) {
+      nix_init_int(NULL, v, nix_get_int(NULL, args[0]) + 1);
+    } else {
+      nix_set_err_msg(ctx, NIX_ERR_UNKNOWN, "First argument should be an integer.");
+    }
+}
+
+void nix_plugin_entry() {
+  const char* args[] = {"n", NULL};
+  PrimOp *p = nix_alloc_primop(NULL, increment, 1, "increment", args, "Example custom built-in function: increments an integer", NULL);
+  nix_register_primop(NULL, p);
+  nix_gc_decref(NULL, p);
+}
+```
+
+**Usage:**
+
+```ShellSession
+$ gcc plugin.c $(pkg-config nix-expr-c --libs --cflags) -shared -o plugin.so
+$ nix --plugin-files ./plugin.so repl
+nix-repl> builtins.increment 1
+2
+```

src/external-api-docs/doxygen.cfg.in

@@ -0,0 +1,58 @@
+# Doxyfile 1.9.5
+
+# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by
+# double-quotes, unless you are using Doxywizard) that should identify the
+# project for which the documentation is generated. This name is used in the
+# title of most generated pages and in a few other places.
+# The default value is: My Project.
+
+PROJECT_NAME           = "Nix"
+
+# The PROJECT_NUMBER tag can be used to enter a project or revision number. This
+# could be handy for archiving the generated documentation or if some version
+# control system is used.
+
+PROJECT_NUMBER         = @PROJECT_NUMBER@
+
+OUTPUT_DIRECTORY       = @OUTPUT_DIRECTORY@
+
+# Using the PROJECT_BRIEF tag one can provide an optional one line description
+# for a project that appears at the top of each page and should give viewer a
+# quick idea about the purpose of the project. Keep the description short.
+
+PROJECT_BRIEF          = "Nix, the purely functional package manager: C API (experimental)"
+
+# If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output.
+# The default value is: YES.
+
+GENERATE_LATEX         = NO
+
+# The INPUT tag is used to specify the files and/or directories that contain
+# documented source files. You may enter file names like myfile.cpp or
+# directories like /usr/src/myproject. Separate the files or directories with
+# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING
+# Note: If this tag is empty the current directory is searched.
+
+# FIXME Make this list more maintainable somehow. We could maybe generate this
+# in the Makefile, but we would need to change how `.in` files are preprocessed
+# so they can expand variables despite configure variables.
+
+INPUT                  = \
+  @src@/src/libutil-c \
+  @src@/src/libexpr-c \
+  @src@/src/libstore-c \
+  @src@/doc/external-api/README.md
+
+FILE_PATTERNS = nix_api_*.h *.md
+
+# The INCLUDE_PATH tag can be used to specify one or more directories that
+# contain include files that are not input files but should be processed by the
+# preprocessor. Note that the INCLUDE_PATH is not recursive, so the setting of
+# RECURSIVE has no effect here.
+# This tag requires that the tag SEARCH_INCLUDES is set to YES.
+
+EXCLUDE_PATTERNS = *_internal.h
+GENERATE_TREEVIEW = YES
+OPTIMIZE_OUTPUT_FOR_C  = YES
+
+USE_MDFILE_AS_MAINPAGE = doc/external-api/README.md

src/external-api-docs/meson.build

@@ -0,0 +1,31 @@
+project('nix-external-api-docs',
+  version : files('.version'),
+  meson_version : '>= 1.1',
+  license : 'LGPL-2.1-or-later',
+)
+
+fs = import('fs')
+
+doxygen_cfg = configure_file(
+  input : 'doxygen.cfg.in',
+  output : 'doxygen.cfg',
+  configuration : {
+    'PROJECT_NUMBER': meson.project_version(),
+    'OUTPUT_DIRECTORY' : meson.current_build_dir(),
+    'src' : fs.parent(fs.parent(meson.project_source_root())),
+  },
+)
+
+doxygen = find_program('doxygen', native : true, required : true)
+
+custom_target(
+  'external-api-docs',
+  command : [ doxygen , doxygen_cfg ],
+  input : [
+    doxygen_cfg,
+  ],
+  output : 'html',
+  install : true,
+  install_dir : get_option('datadir') / 'doc/nix/external-api',
+  build_always_stale : true,
+)

src/external-api-docs/package.nix

@@ -0,0 +1,59 @@
+{ lib
+, mkMesonDerivation
+
+, meson
+, ninja
+, doxygen
+
+# Configuration Options
+
+, version
+}:
+
+let
+  inherit (lib) fileset;
+in
+
+mkMesonDerivation (finalAttrs: {
+  pname = "nix-external-api-docs";
+  inherit version;
+
+  workDir = ./.;
+  fileset =
+    let
+      cpp = fileset.fileFilter (file: file.hasExt "cc" || file.hasExt "h");
+    in
+    fileset.unions [
+      ./.version
+      ../../.version
+      ./meson.build
+      ./doxygen.cfg.in
+      ./README.md
+      # Source is not compiled, but still must be available for Doxygen
+      # to gather comments.
+      (cpp ../libexpr-c)
+      (cpp ../libstore-c)
+      (cpp ../libutil-c)
+    ];
+
+  nativeBuildInputs = [
+    meson
+    ninja
+    doxygen
+  ];
+
+  preConfigure =
+    ''
+      chmod u+w ./.version
+      echo ${finalAttrs.version} > ./.version
+    '';
+
+  postInstall = ''
+    mkdir -p ''${!outputDoc}/nix-support
+    echo "doc external-api-docs $out/share/doc/nix/external-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products
+  '';
+
+  meta = {
+    platforms = lib.platforms.all;
+  };
+})

src/internal-api-docs/.gitignore

@@ -0,0 +1,3 @@
+/doxygen.cfg
+/html
+/latex

src/internal-api-docs/.version

@@ -0,0 +1 @@
+../../.version

src/internal-api-docs/doxygen.cfg.in

@@ -0,0 +1,99 @@
+# Doxyfile 1.9.5
+
+# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by
+# double-quotes, unless you are using Doxywizard) that should identify the
+# project for which the documentation is generated. This name is used in the
+# title of most generated pages and in a few other places.
+# The default value is: My Project.
+
+PROJECT_NAME           = "Nix"
+
+# The PROJECT_NUMBER tag can be used to enter a project or revision number. This
+# could be handy for archiving the generated documentation or if some version
+# control system is used.
+
+PROJECT_NUMBER         = @PROJECT_NUMBER@
+
+OUTPUT_DIRECTORY       = @OUTPUT_DIRECTORY@
+
+# Using the PROJECT_BRIEF tag one can provide an optional one line description
+# for a project that appears at the top of each page and should give viewer a
+# quick idea about the purpose of the project. Keep the description short.
+
+PROJECT_BRIEF          = "Nix, the purely functional package manager; unstable internal interfaces"
+
+# If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output.
+# The default value is: YES.
+
+GENERATE_LATEX         = NO
+
+# The INPUT tag is used to specify the files and/or directories that contain
+# documented source files. You may enter file names like myfile.cpp or
+# directories like /usr/src/myproject. Separate the files or directories with
+# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING
+# Note: If this tag is empty the current directory is searched.
+
+# FIXME Make this list more maintainable somehow. We could maybe generate this
+# in the Makefile, but we would need to change how `.in` files are preprocessed
+# so they can expand variables despite configure variables.
+
+INPUT                  = \
+  @src@/libcmd \
+  @src@/libexpr \
+  @src@/libexpr/flake \
+  @src@/nix-expr-tests \
+  @src@/nix-expr-tests/value \
+  @src@/nix-expr-test-support/test \
+  @src@/nix-expr-test-support/test/value \
+  @src@/libexpr/value \
+  @src@/libfetchers \
+  @src@/libmain \
+  @src@/libstore \
+  @src@/libstore/build \
+  @src@/libstore/builtins \
+  @src@/nix-store-tests \
+  @src@/nix-store-test-support/test \
+  @src@/libutil \
+  @src@/nix-util-tests \
+  @src@/nix-util-test-support/test \
+  @src@/nix \
+  @src@/nix-env \
+  @src@/nix-store
+
+# If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names
+# in the source code. If set to NO, only conditional compilation will be
+# performed. Macro expansion can be done in a controlled way by setting
+# EXPAND_ONLY_PREDEF to YES.
+# The default value is: NO.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+MACRO_EXPANSION        = YES
+
+# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then
+# the macro expansion is limited to the macros specified with the PREDEFINED and
+# EXPAND_AS_DEFINED tags.
+# The default value is: NO.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+EXPAND_ONLY_PREDEF     = YES
+
+# The INCLUDE_PATH tag can be used to specify one or more directories that
+# contain include files that are not input files but should be processed by the
+# preprocessor. Note that the INCLUDE_PATH is not recursive, so the setting of
+# RECURSIVE has no effect here.
+# This tag requires that the tag SEARCH_INCLUDES is set to YES.
+
+INCLUDE_PATH           =
+
+# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this
+# tag can be used to specify a list of macro names that should be expanded. The
+# macro definition that is found in the sources will be used. Use the PREDEFINED
+# tag if you want to use a different macro definition that overrules the
+# definition found in the source code.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+EXPAND_AS_DEFINED      = \
+  DECLARE_COMMON_SERIALISER \
+  DECLARE_WORKER_SERIALISER \
+  DECLARE_SERVE_SERIALISER \
+  LENGTH_PREFIXED_PROTO_HELPER

src/internal-api-docs/meson.build

@@ -0,0 +1,31 @@
+project('nix-internal-api-docs',
+  version : files('.version'),
+  meson_version : '>= 1.1',
+  license : 'LGPL-2.1-or-later',
+)
+
+fs = import('fs')
+
+doxygen_cfg = configure_file(
+  input : 'doxygen.cfg.in',
+  output : 'doxygen.cfg',
+  configuration : {
+    'PROJECT_NUMBER': meson.project_version(),
+    'OUTPUT_DIRECTORY' : meson.current_build_dir(),
+    'src' : fs.parent(fs.parent(meson.project_source_root())) / 'src',
+  },
+)
+
+doxygen = find_program('doxygen', native : true, required : true)
+
+custom_target(
+  'internal-api-docs',
+  command : [ doxygen , doxygen_cfg ],
+  input : [
+    doxygen_cfg,
+  ],
+  output : 'html',
+  install : true,
+  install_dir : get_option('datadir') / 'doc/nix/internal-api',
+  build_always_stale : true,
+)

src/internal-api-docs/package.nix

@@ -0,0 +1,54 @@
+{ lib
+, mkMesonDerivation
+
+, meson
+, ninja
+, doxygen
+
+# Configuration Options
+
+, version
+}:
+
+let
+  inherit (lib) fileset;
+in
+
+mkMesonDerivation (finalAttrs: {
+  pname = "nix-internal-api-docs";
+  inherit version;
+
+  workDir = ./.;
+  fileset = let
+    cpp = fileset.fileFilter (file: file.hasExt "cc" || file.hasExt "hh");
+  in fileset.unions [
+    ./.version
+    ../../.version
+    ./meson.build
+    ./doxygen.cfg.in
+    # Source is not compiled, but still must be available for Doxygen
+    # to gather comments.
+    (cpp ../.)
+  ];
+
+  nativeBuildInputs = [
+    meson
+    ninja
+    doxygen
+  ];
+
+  preConfigure =
+    ''
+      chmod u+w ./.version
+      echo ${finalAttrs.version} > ./.version
+    '';
+
+  postInstall = ''
+    mkdir -p ''${!outputDoc}/nix-support
+    echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products
+  '';
+
+  meta = {
+    platforms = lib.platforms.all;
+  };
+})

src/libcmd/.version

@@ -0,0 +1 @@
+../../.version

src/libcmd/build-utils-meson

@@ -0,0 +1 @@
+../../build-utils-meson

src/libcmd/built-path.cc

@@ -1,6 +1,7 @@
 #include "built-path.hh"
 #include "derivations.hh"
 #include "store-api.hh"
+#include "comparator.hh"
 
 #include <nlohmann/json.hpp>
 
@@ -8,30 +9,24 @@
 
 namespace nix {
 
-#define CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, COMPARATOR) \
-    bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \
-    { \
-        const MY_TYPE* me = this; \
-        auto fields1 = std::tie(*me->drvPath, me->FIELD); \
-        me = &other; \
-        auto fields2 = std::tie(*me->drvPath, me->FIELD); \
-        return fields1 COMPARATOR fields2; \
-    }
-#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \
-    CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, ==) \
-    CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \
-    CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <)
+// Custom implementation to avoid `ref` ptr equality
+GENERATE_CMP_EXT(
+    ,
+    std::strong_ordering,
+    SingleBuiltPathBuilt,
+    *me->drvPath,
+    me->output);
 
-#define FIELD_TYPE std::pair<std::string, StorePath>
-CMP(SingleBuiltPath, SingleBuiltPathBuilt, output)
-#undef FIELD_TYPE
+// Custom implementation to avoid `ref` ptr equality
 
-#define FIELD_TYPE std::map<std::string, StorePath>
-CMP(SingleBuiltPath, BuiltPathBuilt, outputs)
-#undef FIELD_TYPE
-
-#undef CMP
-#undef CMP_ONE
+// TODO no `GENERATE_CMP_EXT` because no `std::set::operator<=>` on
+// Darwin, per header.
+GENERATE_EQUAL(
+    ,
+    BuiltPathBuilt ::,
+    BuiltPathBuilt,
+    *me->drvPath,
+    me->outputs);
 
 StorePath SingleBuiltPath::outPath() const
 {

src/libcmd/built-path.hh

@@ -18,7 +18,8 @@ struct SingleBuiltPathBuilt {
     static SingleBuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view);
     nlohmann::json toJSON(const StoreDirConfig & store) const;
 
-    DECLARE_CMP(SingleBuiltPathBuilt);
+    bool operator ==(const SingleBuiltPathBuilt &) const noexcept;
+    std::strong_ordering operator <=>(const SingleBuiltPathBuilt &) const noexcept;
 };
 
 using _SingleBuiltPathRaw = std::variant<
@@ -33,6 +34,9 @@ struct SingleBuiltPath : _SingleBuiltPathRaw {
     using Opaque = DerivedPathOpaque;
     using Built = SingleBuiltPathBuilt;
 
+    bool operator == (const SingleBuiltPath &) const = default;
+    auto operator <=> (const SingleBuiltPath &) const = default;
+
     inline const Raw & raw() const {
         return static_cast<const Raw &>(*this);
     }
@@ -59,11 +63,13 @@ struct BuiltPathBuilt {
     ref<SingleBuiltPath> drvPath;
     std::map<std::string, StorePath> outputs;
 
+    bool operator == (const BuiltPathBuilt &) const noexcept;
+    // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet.
+    //std::strong_ordering operator <=> (const BuiltPathBuilt &) const noexcept;
+
     std::string to_string(const StoreDirConfig & store) const;
     static BuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view);
     nlohmann::json toJSON(const StoreDirConfig & store) const;
-
-    DECLARE_CMP(BuiltPathBuilt);
 };
 
 using _BuiltPathRaw = std::variant<
@@ -82,6 +88,10 @@ struct BuiltPath : _BuiltPathRaw {
     using Opaque = DerivedPathOpaque;
     using Built = BuiltPathBuilt;
 
+    bool operator == (const BuiltPath &) const = default;
+    // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet.
+    //auto operator <=> (const BuiltPath &) const = default;
+
     inline const Raw & raw() const {
         return static_cast<const Raw &>(*this);
     }

src/libcmd/command.cc

@@ -1,3 +1,5 @@
+#include <nlohmann/json.hpp>
+
 #include "command.hh"
 #include "markdown.hh"
 #include "store-api.hh"
@@ -6,8 +8,7 @@
 #include "nixexpr.hh"
 #include "profiles.hh"
 #include "repl.hh"
-
-#include <nlohmann/json.hpp>
+#include "strings.hh"
 
 extern char * * environ __attribute__((weak));
 
@@ -127,12 +128,12 @@ ref<EvalState> EvalCommand::getEvalState()
     if (!evalState) {
         evalState =
             #if HAVE_BOEHMGC
-            std::allocate_shared<EvalState>(traceable_allocator<EvalState>(),
-                lookupPath, getEvalStore(), getStore())
+            std::allocate_shared<EvalState>(
+                traceable_allocator<EvalState>(),
             #else
             std::make_shared<EvalState>(
-                lookupPath, getEvalStore(), getStore())
             #endif
+                lookupPath, getEvalStore(), fetchSettings, evalSettings, getStore())
             ;
 
         evalState->repair = repair;

src/libcmd/common-eval-args.cc

@@ -1,18 +1,59 @@
+#include "fetch-settings.hh"
 #include "eval-settings.hh"
 #include "common-eval-args.hh"
 #include "shared.hh"
+#include "config-global.hh"
 #include "filetransfer.hh"
 #include "eval.hh"
 #include "fetchers.hh"
 #include "registry.hh"
 #include "flake/flakeref.hh"
+#include "flake/settings.hh"
 #include "store-api.hh"
 #include "command.hh"
 #include "tarball.hh"
 #include "fetch-to-store.hh"
+#include "compatibility-settings.hh"
+#include "eval-settings.hh"
 
 namespace nix {
 
+namespace fs { using namespace std::filesystem; }
+
+fetchers::Settings fetchSettings;
+
+static GlobalConfig::Register rFetchSettings(&fetchSettings);
+
+EvalSettings evalSettings {
+    settings.readOnlyMode,
+    {
+        {
+            "flake",
+            [](ref<Store> store, std::string_view rest) {
+                experimentalFeatureSettings.require(Xp::Flakes);
+                // FIXME `parseFlakeRef` should take a `std::string_view`.
+                auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false);
+                debug("fetching flake search path element '%s''", rest);
+                auto storePath = flakeRef.resolve(store).fetchTree(store).first;
+                return store->toRealPath(storePath);
+            },
+        },
+    },
+};
+
+static GlobalConfig::Register rEvalSettings(&evalSettings);
+
+
+flake::Settings flakeSettings;
+
+static GlobalConfig::Register rFlakeSettings(&flakeSettings);
+
+
+CompatibilitySettings compatibilitySettings {};
+
+static GlobalConfig::Register rCompatibilitySettings(&compatibilitySettings);
+
+
 MixEvalArgs::MixEvalArgs()
 {
     addFlag({
@@ -20,7 +61,7 @@ MixEvalArgs::MixEvalArgs()
         .description = "Pass the value *expr* as the argument *name* to Nix functions.",
         .category = category,
         .labels = {"name", "expr"},
-        .handler = {[&](std::string name, std::string expr) { autoArgs.insert_or_assign(name, AutoArg{AutoArgExpr(expr)}); }}
+        .handler = {[&](std::string name, std::string expr) { autoArgs.insert_or_assign(name, AutoArg{AutoArgExpr{expr}}); }}
     });
 
     addFlag({
@@ -28,7 +69,7 @@ MixEvalArgs::MixEvalArgs()
         .description = "Pass the string *string* as the argument *name* to Nix functions.",
         .category = category,
         .labels = {"name", "string"},
-        .handler = {[&](std::string name, std::string s) { autoArgs.insert_or_assign(name, AutoArg{AutoArgString(s)}); }},
+        .handler = {[&](std::string name, std::string s) { autoArgs.insert_or_assign(name, AutoArg{AutoArgString{s}}); }},
     });
 
     addFlag({
@@ -36,7 +77,7 @@ MixEvalArgs::MixEvalArgs()
         .description = "Pass the contents of file *path* as the argument *name* to Nix functions.",
         .category = category,
         .labels = {"name", "path"},
-        .handler = {[&](std::string name, std::string path) { autoArgs.insert_or_assign(name, AutoArg{AutoArgFile(path)}); }},
+        .handler = {[&](std::string name, std::string path) { autoArgs.insert_or_assign(name, AutoArg{AutoArgFile{path}}); }},
         .completer = completePath
     });
 
@@ -52,75 +93,11 @@ MixEvalArgs::MixEvalArgs()
         .longName = "include",
         .shortName = 'I',
         .description = R"(
-  Add *path* to the Nix search path. The Nix search path is
-  initialized from the colon-separated [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) environment
-  variable, and is used to look up the location of Nix expressions using [paths](@docroot@/language/values.md#type-path) enclosed in angle
-  brackets (i.e., `<nixpkgs>`).
+  Add *path* to search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md)
 
-  For instance, passing
+  This option may be given multiple times.
 
-  ```
-  -I /home/eelco/Dev
-  -I /etc/nixos
-  ```
-
-  will cause Nix to look for paths relative to `/home/eelco/Dev` and
-  `/etc/nixos`, in that order. This is equivalent to setting the
-  `NIX_PATH` environment variable to
-
-  ```
-  /home/eelco/Dev:/etc/nixos
-  ```
-
-  It is also possible to match paths against a prefix. For example,
-  passing
-
-  ```
-  -I nixpkgs=/home/eelco/Dev/nixpkgs-branch
-  -I /etc/nixos
-  ```
-
-  will cause Nix to search for `<nixpkgs/path>` in
-  `/home/eelco/Dev/nixpkgs-branch/path` and `/etc/nixos/nixpkgs/path`.
-
-  If a path in the Nix search path starts with `http://` or `https://`,
-  it is interpreted as the URL of a tarball that will be downloaded and
-  unpacked to a temporary location. The tarball must consist of a single
-  top-level directory. For example, passing
-
-  ```
-  -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/master.tar.gz
-  ```
-
-  tells Nix to download and use the current contents of the `master`
-  branch in the `nixpkgs` repository.
-
-  The URLs of the tarballs from the official `nixos.org` channels
-  (see [the manual page for `nix-channel`](../nix-channel.md)) can be
-  abbreviated as `channel:<channel-name>`.  For instance, the
-  following two flags are equivalent:
-
-  ```
-  -I nixpkgs=channel:nixos-21.05
-  -I nixpkgs=https://nixos.org/channels/nixos-21.05/nixexprs.tar.xz
-  ```
-
-  You can also fetch source trees using [flake URLs](./nix3-flake.md#url-like-syntax) and add them to the
-  search path. For instance,
-
-  ```
-  -I nixpkgs=flake:nixpkgs
-  ```
-
-  specifies that the prefix `nixpkgs` shall refer to the source tree
-  downloaded from the `nixpkgs` entry in the flake registry. Similarly,
-
-  ```
-  -I nixpkgs=flake:github:NixOS/nixpkgs/nixos-22.05
-  ```
-
-  makes `<nixpkgs>` refer to a particular branch of the
-  `NixOS/nixpkgs` repository on GitHub.
+  Paths added through `-I` take precedence over the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path) and the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH).
   )",
         .category = category,
         .labels = {"path"},
@@ -144,8 +121,8 @@ MixEvalArgs::MixEvalArgs()
         .category = category,
         .labels = {"original-ref", "resolved-ref"},
         .handler = {[&](std::string _from, std::string _to) {
-            auto from = parseFlakeRef(_from, absPath("."));
-            auto to = parseFlakeRef(_to, absPath("."));
+            auto from = parseFlakeRef(fetchSettings, _from, fs::current_path().string());
+            auto to = parseFlakeRef(fetchSettings, _to, fs::current_path().string());
             fetchers::Attrs extraAttrs;
             if (to.subdir != "") extraAttrs["dir"] = to.subdir;
             fetchers::overrideRegistry(from.input, to.input, extraAttrs);
@@ -175,7 +152,7 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
         auto v = state.allocValue();
         std::visit(overloaded {
             [&](const AutoArgExpr & arg) {
-                state.mkThunk_(*v, state.parseExprFromString(arg.expr, state.rootPath(".")));
+                state.mkThunk_(*v, state.parseExprFromString(arg.expr, compatibilitySettings.nixShellShebangArgumentsRelativeToScript ? state.rootPath(absPath(getCommandBaseDir())) : state.rootPath(".")));
             },
             [&](const AutoArgString & arg) {
                 v->mkString(arg.s);
@@ -196,14 +173,16 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas
 {
     if (EvalSettings::isPseudoUrl(s)) {
         auto accessor = fetchers::downloadTarball(
-            EvalSettings::resolvePseudoUrl(s)).accessor;
+            state.store,
+            state.fetchSettings,
+            EvalSettings::resolvePseudoUrl(s));
         auto storePath = fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy);
         return state.rootPath(CanonPath(state.store->toRealPath(storePath)));
     }
 
     else if (hasPrefix(s, "flake:")) {
         experimentalFeatureSettings.require(Xp::Flakes);
-        auto flakeRef = parseFlakeRef(std::string(s.substr(6)), {}, true, false);
+        auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false);
         auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first;
         return state.rootPath(CanonPath(state.store->toRealPath(storePath)));
     }

src/libcmd/common-eval-args.hh

@@ -11,10 +11,37 @@
 namespace nix {
 
 class Store;
+
+namespace fetchers { struct Settings; }
+
 class EvalState;
+struct EvalSettings;
+struct CompatibilitySettings;
 class Bindings;
 struct SourcePath;
 
+namespace flake { struct Settings; }
+
+/**
+ * @todo Get rid of global setttings variables
+ */
+extern fetchers::Settings fetchSettings;
+
+/**
+ * @todo Get rid of global setttings variables
+ */
+extern EvalSettings evalSettings;
+
+/**
+ * @todo Get rid of global setttings variables
+ */
+extern flake::Settings flakeSettings;
+
+/**
+ * Settings that control behaviors that have changed since Nix 2.3.
+ */
+extern CompatibilitySettings compatibilitySettings;
+
 struct MixEvalArgs : virtual Args, virtual MixRepair
 {
     static constexpr auto category = "Common evaluation options";

src/libcmd/compatibility-settings.hh

@@ -0,0 +1,36 @@
+#pragma once
+#include "config.hh"
+
+namespace nix {
+struct CompatibilitySettings : public Config
+{
+
+    CompatibilitySettings() = default;
+
+    // Added in Nix 2.24, July 2024.
+    Setting<bool> nixShellAlwaysLooksForShellNix{this, true, "nix-shell-always-looks-for-shell-nix", R"(
+        Before Nix 2.24, [`nix-shell`](@docroot@/command-ref/nix-shell.md) would only look at `shell.nix` if it was in the working directory - when no file was specified.
+
+        Since Nix 2.24, `nix-shell` always looks for a `shell.nix`, whether that's in the working directory, or in a directory that was passed as an argument.
+
+        You may set this to `false` to temporarily revert to the behavior of Nix 2.23 and older.
+
+        Using this setting is not recommended.
+        It will be deprecated and removed.
+    )"};
+
+    // Added in Nix 2.24, July 2024.
+    Setting<bool> nixShellShebangArgumentsRelativeToScript{
+        this, true, "nix-shell-shebang-arguments-relative-to-script", R"(
+        Before Nix 2.24, relative file path expressions in arguments in a `nix-shell` shebang were resolved relative to the working directory.
+
+        Since Nix 2.24, `nix-shell` resolves these paths in a manner that is relative to the [base directory](@docroot@/glossary.md#gloss-base-directory), defined as the script's directory.
+
+        You may set this to `false` to temporarily revert to the behavior of Nix 2.23 and older.
+
+        Using this setting is not recommended.
+        It will be deprecated and removed.
+    )"};
+};
+
+};

src/libcmd/installable-attr-path.cc

@@ -75,6 +75,8 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths()
                 std::set<std::string> outputsToInstall;
                 for (auto & output : packageInfo.queryOutputs(false, true))
                     outputsToInstall.insert(output.first);
+                if (outputsToInstall.empty())
+                    outputsToInstall.insert("out");
                 return OutputsSpec::Names { std::move(outputsToInstall) };
             },
             [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec {

src/libcmd/installable-flake.cc

@@ -43,20 +43,6 @@ std::vector<std::string> InstallableFlake::getActualAttrPaths()
     return res;
 }
 
-Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake)
-{
-    auto vFlake = state.allocValue();
-
-    callFlake(state, lockedFlake, *vFlake);
-
-    auto aOutputs = vFlake->attrs()->get(state.symbols.create("outputs"));
-    assert(aOutputs);
-
-    state.forceValue(*aOutputs->value, aOutputs->value->determinePos(noPos));
-
-    return aOutputs->value;
-}
-
 static std::string showAttrPaths(const std::vector<std::string> & paths)
 {
     std::string s;
@@ -106,19 +92,24 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths()
             fmt("while evaluating the flake output attribute '%s'", attrPath)))
         {
             return { *derivedPathWithInfo };
+        } else {
+            throw Error(
+                "expected flake output attribute '%s' to be a derivation or path but found %s: %s",
+                attrPath,
+                showType(v),
+                ValuePrinter(*this->state, v, errorPrintOptions)
+            );
         }
-        else
-            throw Error("flake output attribute '%s' is not a derivation or path", attrPath);
     }
 
     auto drvPath = attr->forceDerivation();
 
-    std::optional<NixInt> priority;
+    std::optional<NixInt::Inner> priority;
 
     if (attr->maybeGetAttr(state->sOutputSpecified)) {
     } else if (auto aMeta = attr->maybeGetAttr(state->sMeta)) {
         if (auto aPriority = aMeta->maybeGetAttr("priority"))
-            priority = aPriority->getInt();
+            priority = aPriority->getInt().value;
     }
 
     return {{
@@ -205,7 +196,8 @@ std::shared_ptr<flake::LockedFlake> InstallableFlake::getLockedFlake() const
         flake::LockFlags lockFlagsApplyConfig = lockFlags;
         // FIXME why this side effect?
         lockFlagsApplyConfig.applyNixConfig = true;
-        _lockedFlake = std::make_shared<flake::LockedFlake>(lockFlake(*state, flakeRef, lockFlagsApplyConfig));
+        _lockedFlake = std::make_shared<flake::LockedFlake>(lockFlake(
+            flakeSettings, *state, flakeRef, lockFlagsApplyConfig));
     }
     return _lockedFlake;
 }

src/libcmd/installable-flake.hh

@@ -1,6 +1,7 @@
 #pragma once
 ///@file
 
+#include "common-eval-args.hh"
 #include "installable-value.hh"
 
 namespace nix {
@@ -52,8 +53,6 @@ struct InstallableFlake : InstallableValue
 
     std::vector<std::string> getActualAttrPaths();
 
-    Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake);
-
     DerivedPathsWithInfo toDerivedPaths() override;
 
     std::pair<Value *, PosIdx> toValue(EvalState & state) override;
@@ -80,7 +79,7 @@ struct InstallableFlake : InstallableValue
  */
 static inline FlakeRef defaultNixpkgsFlakeRef()
 {
-    return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}});
+    return FlakeRef::fromAttrs(fetchSettings, {{"type","indirect"}, {"id", "nixpkgs"}});
 }
 
 ref<eval_cache::EvalCache> openEvalCache(

src/libcmd/installable-value.hh

@@ -40,7 +40,7 @@ struct ExtraPathInfoValue : ExtraPathInfo
         /**
          * An optional priority for use with "build envs". See Package
          */
-        std::optional<NixInt> priority;
+        std::optional<NixInt::Inner> priority;
 
         /**
          * The attribute path associated with this value. The idea is
@@ -66,7 +66,7 @@ struct ExtraPathInfoValue : ExtraPathInfo
 };
 
 /**
- * An Installable which corresponds a Nix langauge value, in addition to
+ * An Installable which corresponds a Nix language value, in addition to
  * a collection of \ref DerivedPath "derived paths".
  */
 struct InstallableValue : Installable

src/libcmd/installables.cc

@@ -21,15 +21,18 @@
 #include "url.hh"
 #include "registry.hh"
 #include "build-result.hh"
-#include "fs-input-accessor.hh"
 
 #include <regex>
 #include <queue>
 
 #include <nlohmann/json.hpp>
 
+#include "strings-inline.hh"
+
 namespace nix {
 
+namespace fs { using namespace std::filesystem; }
+
 void completeFlakeInputPath(
     AddCompletions & completions,
     ref<EvalState> evalState,
@@ -130,7 +133,7 @@ MixFlakeOptions::MixFlakeOptions()
             lockFlags.writeLockFile = false;
             lockFlags.inputOverrides.insert_or_assign(
                 flake::parseInputPath(inputPath),
-                parseFlakeRef(flakeRef, absPath(getCommandBaseDir()), true));
+                parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir()), true));
         }},
         .completer = {[&](AddCompletions & completions, size_t n, std::string_view prefix) {
             if (n == 0) {
@@ -147,7 +150,7 @@ MixFlakeOptions::MixFlakeOptions()
         .category = category,
         .labels = {"flake-lock-path"},
         .handler = {[&](std::string lockFilePath) {
-            lockFlags.referenceLockFilePath = getUnfilteredRootPath(CanonPath(absPath(lockFilePath)));
+            lockFlags.referenceLockFilePath = {getFSSourceAccessor(), CanonPath(absPath(lockFilePath))};
         }},
         .completer = completePath
     });
@@ -171,14 +174,15 @@ MixFlakeOptions::MixFlakeOptions()
         .handler = {[&](std::string flakeRef) {
             auto evalState = getEvalState();
             auto flake = flake::lockFlake(
+                flakeSettings,
                 *evalState,
-                parseFlakeRef(flakeRef, absPath(getCommandBaseDir())),
+                parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir())),
                 { .writeLockFile = false });
             for (auto & [inputName, input] : flake.lockFile.root->inputs) {
                 auto input2 = flake.lockFile.findInput({inputName}); // resolve 'follows' nodes
                 if (auto input3 = std::dynamic_pointer_cast<const flake::LockedNode>(input2)) {
                     overrideRegistry(
-                        fetchers::Input::fromAttrs({{"type","indirect"}, {"id", inputName}}),
+                        fetchers::Input::fromAttrs(fetchSettings, {{"type","indirect"}, {"id", inputName}}),
                         input3->lockedRef.input,
                         {});
                 }
@@ -290,10 +294,10 @@ void SourceExprCommand::completeInstallable(AddCompletions & completions, std::s
 
             if (v2.type() == nAttrs) {
                 for (auto & i : *v2.attrs()) {
-                    std::string name = state->symbols[i.name];
+                    std::string_view name = state->symbols[i.name];
                     if (name.find(searchWord) == 0) {
                         if (prefix_ == "")
-                            completions.add(name);
+                            completions.add(std::string(name));
                         else
                             completions.add(prefix_ + "." + name);
                     }
@@ -339,10 +343,11 @@ void completeFlakeRefWithFragment(
             auto flakeRefS = std::string(prefix.substr(0, hash));
 
             // TODO: ideally this would use the command base directory instead of assuming ".".
-            auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath("."));
+            auto flakeRef = parseFlakeRef(fetchSettings, expandTilde(flakeRefS), fs::current_path().string());
 
             auto evalCache = openEvalCache(*evalState,
-                std::make_shared<flake::LockedFlake>(lockFlake(*evalState, flakeRef, lockFlags)));
+                std::make_shared<flake::LockedFlake>(lockFlake(
+                    flakeSettings, *evalState, flakeRef, lockFlags)));
 
             auto root = evalCache->getRoot();
 
@@ -373,6 +378,7 @@ void completeFlakeRefWithFragment(
                         auto attrPath2 = (*attr)->getAttrPath(attr2);
                         /* Strip the attrpath prefix. */
                         attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size());
+                        // FIXME: handle names with dots
                         completions.add(flakeRefS + "#" + prefixRoot + concatStringsSep(".", evalState->symbols.resolve(attrPath2)));
                     }
                 }
@@ -404,7 +410,7 @@ void completeFlakeRef(AddCompletions & completions, ref<Store> store, std::strin
     Args::completeDir(completions, 0, prefix);
 
     /* Look for registry entries that match the prefix. */
-    for (auto & registry : fetchers::getRegistries(store)) {
+    for (auto & registry : fetchers::getRegistries(fetchSettings, store)) {
         for (auto & entry : registry->entries) {
             auto from = entry.from.to_string();
             if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) {
@@ -443,13 +449,10 @@ ref<eval_cache::EvalCache> openEvalCache(
     EvalState & state,
     std::shared_ptr<flake::LockedFlake> lockedFlake)
 {
-    auto fingerprint = lockedFlake->getFingerprint(state.store);
-    return make_ref<nix::eval_cache::EvalCache>(
-        evalSettings.useEvalCache && evalSettings.pureEval
-            ? fingerprint
-            : std::nullopt,
-        state,
-        [&state, lockedFlake]()
+    auto fingerprint = evalSettings.useEvalCache && evalSettings.pureEval
+        ? lockedFlake->getFingerprint(state.store)
+        : std::nullopt;
+    auto rootLoader = [&state, lockedFlake]()
         {
             /* For testing whether the evaluation cache is
                complete. */
@@ -465,7 +468,17 @@ ref<eval_cache::EvalCache> openEvalCache(
             assert(aOutputs);
 
             return aOutputs->value;
-        });
+        };
+
+    if (fingerprint) {
+        auto search = state.evalCaches.find(fingerprint.value());
+        if (search == state.evalCaches.end()) {
+            search = state.evalCaches.emplace(fingerprint.value(), make_ref<nix::eval_cache::EvalCache>(fingerprint, state, rootLoader)).first;
+        }
+        return search->second;
+    } else {
+        return make_ref<nix::eval_cache::EvalCache>(std::nullopt, state, rootLoader);
+    }
 }
 
 Installables SourceExprCommand::parseInstallables(
@@ -528,7 +541,8 @@ Installables SourceExprCommand::parseInstallables(
             }
 
             try {
-                auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, absPath(getCommandBaseDir()));
+                auto [flakeRef, fragment] = parseFlakeRefWithFragment(
+                    fetchSettings, std::string { prefix }, absPath(getCommandBaseDir()));
                 result.push_back(make_ref<InstallableFlake>(
                         this,
                         getEvalState(),
@@ -595,6 +609,37 @@ std::vector<BuiltPathWithResult> Installable::build(
     return res;
 }
 
+static void throwBuildErrors(
+    std::vector<KeyedBuildResult> & buildResults,
+    const Store & store)
+{
+    std::vector<KeyedBuildResult> failed;
+    for (auto & buildResult : buildResults) {
+        if (!buildResult.success()) {
+            failed.push_back(buildResult);
+        }
+    }
+
+    auto failedResult = failed.begin();
+    if (failedResult != failed.end()) {
+        if (failed.size() == 1) {
+            failedResult->rethrow();
+        } else {
+            StringSet failedPaths;
+            for (; failedResult != failed.end(); failedResult++) {
+                if (!failedResult->errorMsg.empty()) {
+                    logError(ErrorInfo{
+                        .level = lvlError,
+                        .msg = failedResult->errorMsg,
+                    });
+                }
+                failedPaths.insert(failedResult->path.to_string(store));
+            }
+            throw Error("build of %s failed", concatStringsSep(", ", quoteStrings(failedPaths)));
+        }
+    }
+}
+
 std::vector<std::pair<ref<Installable>, BuiltPathWithResult>> Installable::build2(
     ref<Store> evalStore,
     ref<Store> store,
@@ -656,10 +701,9 @@ std::vector<std::pair<ref<Installable>, BuiltPathWithResult>> Installable::build
         if (settings.printMissing)
             printMissing(store, pathsToBuild, lvlInfo);
 
-        for (auto & buildResult : store->buildPathsWithResults(pathsToBuild, bMode, evalStore)) {
-            if (!buildResult.success())
-                buildResult.rethrow();
-
+        auto buildResults = store->buildPathsWithResults(pathsToBuild, bMode, evalStore);
+        throwBuildErrors(buildResults, *store);
+        for (auto & buildResult : buildResults) {
             for (auto & aux : backmap[buildResult.path]) {
                 std::visit(overloaded {
                     [&](const DerivedPath::Built & bfd) {
@@ -815,6 +859,7 @@ std::vector<FlakeRef> RawInstallablesCommand::getFlakeRefsForCompletion()
     std::vector<FlakeRef> res;
     for (auto i : rawInstallables)
         res.push_back(parseFlakeRefWithFragment(
+            fetchSettings,
             expandTilde(i),
             absPath(getCommandBaseDir())).first);
     return res;
@@ -837,6 +882,7 @@ std::vector<FlakeRef> InstallableCommand::getFlakeRefsForCompletion()
 {
     return {
         parseFlakeRefWithFragment(
+            fetchSettings,
             expandTilde(_installable),
             absPath(getCommandBaseDir())).first
     };

src/libcmd/local.mk

@@ -6,10 +6,10 @@ libcmd_DIR := $(d)
 
 libcmd_SOURCES := $(wildcard $(d)/*.cc)
 
-libcmd_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) $(INCLUDE_libmain)
+libcmd_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) $(INCLUDE_libflake) $(INCLUDE_libmain)
 
 libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) $(THREAD_LDFLAGS)
 
-libcmd_LIBS = libstore libutil libexpr libmain libfetchers
+libcmd_LIBS = libutil libstore libfetchers libflake libexpr libmain
 
 $(eval $(call install-file-in, $(buildprefix)$(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644))

src/libcmd/markdown.cc

@@ -1,21 +1,23 @@
 #include "markdown.hh"
-#include "util.hh"
+#include "environment-variables.hh"
+#include "error.hh"
 #include "finally.hh"
 #include "terminal.hh"
 
 #if HAVE_LOWDOWN
-# include <sys/queue.h>
-# include <lowdown.h>
+#  include <sys/queue.h>
+#  include <lowdown.h>
 #endif
 
 namespace nix {
 
-std::string renderMarkdownToTerminal(std::string_view markdown)
-{
 #if HAVE_LOWDOWN
+static std::string doRenderMarkdownToTerminal(std::string_view markdown)
+{
     int windowWidth = getWindowSize().second;
 
-    struct lowdown_opts opts {
+    struct lowdown_opts opts
+    {
         .type = LOWDOWN_TERM,
         .maxdepth = 20,
         .cols = (size_t) std::max(windowWidth - 5, 60),
@@ -51,9 +53,21 @@ std::string renderMarkdownToTerminal(std::string_view markdown)
         throw Error("allocation error while rendering Markdown");
 
     return filterANSIEscapes(std::string(buf->data, buf->size), !isTTY());
-#else
-    return std::string(markdown);
-#endif
 }
 
+std::string renderMarkdownToTerminal(std::string_view markdown)
+{
+    if (auto e = getEnv("_NIX_TEST_RAW_MARKDOWN"); e && *e == "1")
+        return std::string(markdown);
+    else
+        return doRenderMarkdownToTerminal(markdown);
 }
+
+#else
+std::string renderMarkdownToTerminal(std::string_view markdown)
+{
+    return std::string(markdown);
+}
+#endif
+
+} // namespace nix

src/libcmd/markdown.hh

@@ -1,10 +1,17 @@
 #pragma once
 ///@file
 
-#include "types.hh"
+#include <string_view>
 
 namespace nix {
 
+/**
+ * Render the given Markdown text to the terminal.
+ *
+ * If Nix is compiled without Markdown support, this function will return the input text as-is.
+ *
+ * The renderer takes into account the terminal width, and wraps text accordingly.
+ */
 std::string renderMarkdownToTerminal(std::string_view markdown);
 
 }

src/libcmd/meson.build

@@ -0,0 +1,130 @@
+project('nix-cmd', 'cpp',
+  version : files('.version'),
+  default_options : [
+    'cpp_std=c++2a',
+    # TODO(Qyriad): increase the warning level
+    'warning_level=1',
+    'debug=true',
+    'optimization=2',
+    'errorlogs=true', # Please print logs for tests that fail
+  ],
+  meson_version : '>= 1.1',
+  license : 'LGPL-2.1-or-later',
+)
+
+cxx = meson.get_compiler('cpp')
+
+subdir('build-utils-meson/deps-lists')
+
+configdata = configuration_data()
+
+deps_private_maybe_subproject = [
+]
+deps_public_maybe_subproject = [
+  dependency('nix-util'),
+  dependency('nix-store'),
+  dependency('nix-fetchers'),
+  dependency('nix-expr'),
+  dependency('nix-flake'),
+  dependency('nix-main'),
+]
+subdir('build-utils-meson/subprojects')
+
+subdir('build-utils-meson/threads')
+
+nlohmann_json = dependency('nlohmann_json', version : '>= 3.9')
+deps_public += nlohmann_json
+
+lowdown = dependency('lowdown', version : '>= 0.9.0', required : get_option('markdown'))
+deps_private += lowdown
+configdata.set('HAVE_LOWDOWN', lowdown.found().to_int())
+
+readline_flavor = get_option('readline-flavor')
+if readline_flavor == 'editline'
+  editline = dependency('libeditline', 'editline', version : '>=1.14')
+  deps_private += editline
+elif readline_flavor == 'readline'
+  readline = dependency('readline')
+  deps_private += readline
+  configdata.set(
+    'USE_READLINE',
+    1,
+    description: 'Use readline instead of editline',
+  )
+else
+  error('illegal editline flavor', readline_flavor)
+endif
+
+config_h = configure_file(
+  configuration : configdata,
+  output : 'config-cmd.hh',
+)
+
+add_project_arguments(
+  # TODO(Qyriad): Yes this is how the autoconf+Make system did it.
+  # It would be nice for our headers to be idempotent instead.
+  '-include', 'config-util.hh',
+  '-include', 'config-store.hh',
+  # '-include', 'config-fetchers.h',
+  '-include', 'config-expr.hh',
+  '-include', 'config-main.hh',
+  '-include', 'config-cmd.hh',
+  language : 'cpp',
+)
+
+subdir('build-utils-meson/diagnostics')
+
+sources = files(
+  'built-path.cc',
+  'command-installable-value.cc',
+  'command.cc',
+  'common-eval-args.cc',
+  'editor-for.cc',
+  'installable-attr-path.cc',
+  'installable-derived-path.cc',
+  'installable-flake.cc',
+  'installable-value.cc',
+  'installables.cc',
+  'legacy.cc',
+  'markdown.cc',
+  'misc-store-flags.cc',
+  'network-proxy.cc',
+  'repl-interacter.cc',
+  'repl.cc',
+)
+
+include_dirs = [include_directories('.')]
+
+headers = [config_h] + files(
+  'built-path.hh',
+  'command-installable-value.hh',
+  'command.hh',
+  'common-eval-args.hh',
+  'compatibility-settings.hh',
+  'editor-for.hh',
+  'installable-attr-path.hh',
+  'installable-derived-path.hh',
+  'installable-flake.hh',
+  'installable-value.hh',
+  'installables.hh',
+  'legacy.hh',
+  'markdown.hh',
+  'misc-store-flags.hh',
+  'network-proxy.hh',
+  'repl-interacter.hh',
+  'repl.hh',
+)
+
+this_library = library(
+  'nixcmd',
+  sources,
+  dependencies : deps_public + deps_private + deps_other,
+  prelink : true, # For C++ static initializers
+  install : true,
+)
+
+install_headers(headers, subdir : 'nix', preserve_path : true)
+
+libraries_private = []
+
+subdir('build-utils-meson/export')

src/libcmd/meson.options

@@ -0,0 +1,15 @@
+# vim: filetype=meson
+
+option(
+  'markdown',
+  type: 'feature',
+  description: 'Enable Markdown rendering in the Nix binary (requires lowdown)',
+)
+
+option(
+  'readline-flavor',
+  type : 'combo',
+  choices : ['editline', 'readline'],
+  value : 'editline',
+  description : 'Which library to use for nice line editing with the Nix language REPL',
+)

src/libcmd/misc-store-flags.cc

@@ -81,9 +81,15 @@ Args::Flag fileIngestionMethod(FileIngestionMethod * method)
     How to compute the hash of the input.
     One of:
 
-    - `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function.
+    - `nar` (the default):
+      Serialises the input as a
+      [Nix Archive](@docroot@/store/file-system-object/content-address.md#serial-nix-archive)
+      and passes that to the hash function.
 
-    - `flat`: Assumes that the input is a single file and directly passes it to the hash function;
+    - `flat`:
+      Assumes that the input is a single file and
+      [directly passes](@docroot@/store/file-system-object/content-address.md#serial-flat)
+      it to the hash function.
         )",
         .labels = {"file-ingestion-method"},
         .handler = {[method](std::string s) {
@@ -101,15 +107,23 @@ Args::Flag contentAddressMethod(ContentAddressMethod * method)
     How to compute the content-address of the store object.
     One of:
 
-    - `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function.
+    - [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive)
+      (the default):
+      Serialises the input as a
+      [Nix Archive](@docroot@/store/file-system-object/content-address.md#serial-nix-archive)
+      and passes that to the hash function.
 
-    - `flat`: Assumes that the input is a single file and directly passes it to the hash function;
+    - [`flat`](@docroot@/store/store-object/content-address.md#method-flat):
+      Assumes that the input is a single file and
+      [directly passes](@docroot@/store/file-system-object/content-address.md#serial-flat)
+      it to the hash function.
 
-    - `text`: Like `flat`, but used for
-      [derivations](@docroot@/glossary.md#store-derivation) serialized in store object and 
+    - [`text`](@docroot@/store/store-object/content-address.md#method-text):
+      Like `flat`, but used for
+      [derivations](@docroot@/glossary.md#store-derivation) serialized in store object and
       [`builtins.toFile`](@docroot@/language/builtins.html#builtins-toFile).
       For advanced use-cases only;
-      for regular usage prefer `nar` and `flat.
+      for regular usage prefer `nar` and `flat`.
         )",
         .labels = {"content-address-method"},
         .handler = {[method](std::string s) {

src/libcmd/network-proxy.cc

@@ -0,0 +1,50 @@
+#include "network-proxy.hh"
+
+#include <algorithm>
+
+#include "environment-variables.hh"
+
+namespace nix {
+
+static const StringSet lowercaseVariables{"http_proxy", "https_proxy", "ftp_proxy", "all_proxy", "no_proxy"};
+
+static StringSet getAllVariables()
+{
+    StringSet variables = lowercaseVariables;
+    for (const auto & variable : lowercaseVariables) {
+        std::string upperVariable;
+        std::transform(
+            variable.begin(), variable.end(), upperVariable.begin(), [](unsigned char c) { return std::toupper(c); });
+        variables.insert(std::move(upperVariable));
+    }
+    return variables;
+}
+
+const StringSet networkProxyVariables = getAllVariables();
+
+static StringSet getExcludingNoProxyVariables()
+{
+    static const StringSet excludeVariables{"no_proxy", "NO_PROXY"};
+    StringSet variables;
+    std::set_difference(
+        networkProxyVariables.begin(),
+        networkProxyVariables.end(),
+        excludeVariables.begin(),
+        excludeVariables.end(),
+        std::inserter(variables, variables.begin()));
+    return variables;
+}
+
+static const StringSet excludingNoProxyVariables = getExcludingNoProxyVariables();
+
+bool haveNetworkProxyConnection()
+{
+    for (const auto & variable : excludingNoProxyVariables) {
+        if (getEnv(variable).has_value()) {
+            return true;
+        }
+    }
+    return false;
+}
+
+}

src/libcmd/network-proxy.hh

@@ -0,0 +1,22 @@
+#pragma once
+///@file
+
+#include "types.hh"
+
+namespace nix {
+
+/**
+ * Environment variables relating to network proxying. These are used by
+ * a few misc commands.
+ *
+ * See the Environment section of https://curl.se/docs/manpage.html for details.
+ */
+extern const StringSet networkProxyVariables;
+
+/**
+ * Heuristically check if there is a proxy connection by checking for defined
+ * proxy variables.
+ */
+bool haveNetworkProxyConnection();
+
+}

src/libcmd/package.nix

@@ -0,0 +1,104 @@
+{ lib
+, stdenv
+, mkMesonDerivation
+, releaseTools
+
+, meson
+, ninja
+, pkg-config
+
+, nix-util
+, nix-store
+, nix-fetchers
+, nix-expr
+, nix-flake
+, nix-main
+, editline
+, readline
+, lowdown
+, nlohmann_json
+
+# Configuration Options
+
+, version
+
+# Whether to enable Markdown rendering in the Nix binary.
+, enableMarkdown ? !stdenv.hostPlatform.isWindows
+
+# Which interactive line editor library to use for Nix's repl.
+#
+# Currently supported choices are:
+#
+# - editline (default)
+# - readline
+, readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline"
+}:
+
+let
+  inherit (lib) fileset;
+in
+
+mkMesonDerivation (finalAttrs: {
+  pname = "nix-cmd";
+  inherit version;
+
+  workDir = ./.;
+  fileset = fileset.unions [
+    ../../build-utils-meson
+    ./build-utils-meson
+    ../../.version
+    ./.version
+    ./meson.build
+    ./meson.options
+    (fileset.fileFilter (file: file.hasExt "cc") ./.)
+    (fileset.fileFilter (file: file.hasExt "hh") ./.)
+  ];
+
+  outputs = [ "out" "dev" ];
+
+  nativeBuildInputs = [
+    meson
+    ninja
+    pkg-config
+  ];
+
+  buildInputs = [
+    ({ inherit editline readline; }.${readlineFlavor})
+  ] ++ lib.optional enableMarkdown lowdown;
+
+  propagatedBuildInputs = [
+    nix-util
+    nix-store
+    nix-fetchers
+    nix-expr
+    nix-flake
+    nix-main
+    nlohmann_json
+  ];
+
+  preConfigure =
+    # "Inline" .version so it's not a symlink, and includes the suffix.
+    # Do the meson utils, without modification.
+    ''
+      chmod u+w ./.version
+      echo ${version} > ../../.version
+    '';
+
+  mesonFlags = [
+    (lib.mesonEnable "markdown" enableMarkdown)
+    (lib.mesonOption "readline-flavor" readlineFlavor)
+  ];
+
+  env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) {
+    LDFLAGS = "-fuse-ld=gold";
+  };
+
+  separateDebugInfo = !stdenv.hostPlatform.isStatic;
+
+  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
+
+  meta = {
+    platforms = lib.platforms.unix ++ lib.platforms.windows;
+  };
+
+})

src/libcmd/repl-interacter.cc

@@ -18,7 +18,8 @@ extern "C" {
 #include "finally.hh"
 #include "repl-interacter.hh"
 #include "file-system.hh"
-#include "libcmd/repl.hh"
+#include "repl.hh"
+#include "environment-variables.hh"
 
 namespace nix {
 
@@ -34,6 +35,7 @@ void sigintHandler(int signo)
 
 static detail::ReplCompleterMixin * curRepl; // ugly
 
+#ifndef USE_READLINE
 static char * completionCallback(char * s, int * match)
 {
     auto possible = curRepl->completePrefix(s);
@@ -73,7 +75,7 @@ static int listPossibleCallback(char * s, char *** avp)
 {
     auto possible = curRepl->completePrefix(s);
 
-    if (possible.size() > (INT_MAX / sizeof(char *)))
+    if (possible.size() > (std::numeric_limits<int>::max() / sizeof(char *)))
         throw Error("too many completions");
 
     int ac = 0;
@@ -100,6 +102,7 @@ static int listPossibleCallback(char * s, char *** avp)
 
     return ac;
 }
+#endif
 
 ReadlineLikeInteracter::Guard ReadlineLikeInteracter::init(detail::ReplCompleterMixin * repl)
 {
@@ -175,10 +178,23 @@ bool ReadlineLikeInteracter::getLine(std::string & input, ReplPromptType promptT
         return true;
     }
 
+    // editline doesn't echo the input to the output when non-interactive, unlike readline
+    // this results in a different behavior when running tests. The echoing is
+    // quite useful for reading the test output, so we add it here.
+    if (auto e = getEnv("_NIX_TEST_REPL_ECHO"); s && e && *e == "1")
+    {
+#ifndef USE_READLINE
+        // This is probably not right for multi-line input, but we don't use that
+        // in the characterisation tests, so it's fine.
+        std::cout << promptForType(promptType) << s << std::endl;
+#endif
+    }
+
     if (!s)
         return false;
     input += s;
     input += '\n';
+
     return true;
 }
 

src/libcmd/repl.cc

@@ -1,16 +1,13 @@
 #include <iostream>
 #include <cstdlib>
 #include <cstring>
-#include <climits>
 
-#include "libcmd/repl-interacter.hh"
+#include "repl-interacter.hh"
 #include "repl.hh"
 
 #include "ansicolor.hh"
 #include "shared.hh"
 #include "eval.hh"
-#include "eval-cache.hh"
-#include "eval-inline.hh"
 #include "eval-settings.hh"
 #include "attr-path.hh"
 #include "signals.hh"
@@ -28,12 +25,16 @@
 #include "markdown.hh"
 #include "local-fs-store.hh"
 #include "print.hh"
+#include "ref.hh"
+#include "value.hh"
 
 #if HAVE_BOEHMGC
 #define GC_INCLUDE_NEW
 #include <gc/gc_cpp.h>
 #endif
 
+#include "strings.hh"
+
 namespace nix {
 
 /**
@@ -75,10 +76,14 @@ struct NixRepl
     int displ;
     StringSet varNames;
 
+    RunNix * runNixPtr;
+
+    void runNix(Path program, const Strings & args, const std::optional<std::string> & input = {});
+
     std::unique_ptr<ReplInteracter> interacter;
 
     NixRepl(const LookupPath & lookupPath, nix::ref<Store> store,ref<EvalState> state,
-            std::function<AnnotatedValues()> getValues);
+            std::function<AnnotatedValues()> getValues, RunNix * runNix);
     virtual ~NixRepl() = default;
 
     ReplExitStatus mainLoop() override;
@@ -123,31 +128,16 @@ std::string removeWhitespace(std::string s)
 
 
 NixRepl::NixRepl(const LookupPath & lookupPath, nix::ref<Store> store, ref<EvalState> state,
-            std::function<NixRepl::AnnotatedValues()> getValues)
+            std::function<NixRepl::AnnotatedValues()> getValues, RunNix * runNix = nullptr)
     : AbstractNixRepl(state)
     , debugTraceIndex(0)
     , getValues(getValues)
     , staticEnv(new StaticEnv(nullptr, state->staticBaseEnv.get()))
-    , interacter(make_unique<ReadlineLikeInteracter>(getDataDir() + "/nix/repl-history"))
+    , runNixPtr{runNix}
+    , interacter(make_unique<ReadlineLikeInteracter>(getDataDir() + "/repl-history"))
 {
 }
 
-void runNix(Path program, const Strings & args,
-    const std::optional<std::string> & input = {})
-{
-    auto subprocessEnv = getEnv();
-    subprocessEnv["NIX_CONFIG"] = globalConfig.toKeyValue();
-
-    runProgram2(RunOptions {
-        .program = settings.nixBinDir+ "/" + program,
-        .args = args,
-        .environment = subprocessEnv,
-        .input = input,
-    });
-
-    return;
-}
-
 static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positions, const DebugTrace & dt)
 {
     if (dt.isError)
@@ -214,7 +204,7 @@ ReplExitStatus NixRepl::mainLoop()
                 case ProcessLineResult::PromptAgain:
                     break;
                 default:
-                    abort();
+                    unreachable();
             }
         } catch (ParseError & e) {
             if (e.msg().find("unexpected end of file") != std::string::npos) {
@@ -259,11 +249,14 @@ StringSet NixRepl::completePrefix(const std::string & prefix)
         try {
             auto dir = std::string(cur, 0, slash);
             auto prefix2 = std::string(cur, slash + 1);
-            for (auto & entry : readDirectory(dir == "" ? "/" : dir)) {
-                if (entry.name[0] != '.' && hasPrefix(entry.name, prefix2))
-                    completions.insert(prev + dir + "/" + entry.name);
+            for (auto & entry : std::filesystem::directory_iterator{dir == "" ? "/" : dir}) {
+                checkInterrupt();
+                auto name = entry.path().filename().string();
+                if (name[0] != '.' && hasPrefix(name, prefix2))
+                    completions.insert(prev + entry.path().string());
             }
         } catch (Error &) {
+        } catch (std::filesystem::filesystem_error &) {
         }
     } else if ((dot = cur.rfind('.')) == std::string::npos) {
         /* This is a variable name; look it up in the current scope. */
@@ -302,6 +295,8 @@ StringSet NixRepl::completePrefix(const std::string & prefix)
             // Quietly ignore evaluation errors.
         } catch (BadURL & e) {
             // Quietly ignore BadURL flake-related errors.
+        } catch (FileNotFound & e) {
+            // Quietly ignore non-existent file beeing `import`-ed.
         }
     }
 
@@ -508,7 +503,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
 
         // runProgram redirects stdout to a StringSink,
         // using runProgram2 to allow editors to display their UI
-        runProgram2(RunOptions { .program = editor, .lookupPath = true, .args = args });
+        runProgram2(RunOptions { .program = editor, .lookupPath = true, .args = args , .isInteractive = true });
 
         // Reload right after exiting the editor
         state->resetFileCache();
@@ -609,6 +604,35 @@ ProcessLineResult NixRepl::processLine(std::string line)
 
     else if (command == ":doc") {
         Value v;
+
+        auto expr = parseString(arg);
+        std::string fallbackName;
+        PosIdx fallbackPos;
+        DocComment fallbackDoc;
+        if (auto select = dynamic_cast<ExprSelect *>(expr)) {
+            Value vAttrs;
+            auto name = select->evalExceptFinalSelect(*state, *env, vAttrs);
+            fallbackName = state->symbols[name];
+
+            state->forceAttrs(vAttrs, noPos, "while evaluating an attribute set to look for documentation");
+            auto attrs = vAttrs.attrs();
+            assert(attrs);
+            auto attr = attrs->get(name);
+            if (!attr) {
+                // When missing, trigger the normal exception
+                // e.g. :doc builtins.foo
+                // behaves like
+                // nix-repl> builtins.foo<tab>
+                // error: attribute 'foo' missing
+                evalString(arg, v);
+                assert(false);
+            }
+            if (attr->pos) {
+                fallbackPos = attr->pos;
+                fallbackDoc = state->getDocCommentForPos(fallbackPos);
+            }
+        }
+
         evalString(arg, v);
         if (auto doc = state->getDoc(v)) {
             std::string markdown;
@@ -626,6 +650,19 @@ ProcessLineResult NixRepl::processLine(std::string line)
             markdown += stripIndentation(doc->doc);
 
             logger->cout(trim(renderMarkdownToTerminal(markdown)));
+        } else if (fallbackPos) {
+            std::stringstream ss;
+            ss << "Attribute `" << fallbackName << "`\n\n";
+            ss << "  … defined at " << state->positions[fallbackPos] << "\n\n";
+            if (fallbackDoc) {
+                ss << fallbackDoc.getInnerText(state->positions);
+            } else {
+                ss << "No documentation found.\n\n";
+            }
+
+            auto markdown = ss.str();
+            logger->cout(trim(renderMarkdownToTerminal(markdown)));
+
         } else
             throw Error("value does not have documentation");
     }
@@ -683,14 +720,14 @@ void NixRepl::loadFlake(const std::string & flakeRefS)
     if (flakeRefS.empty())
         throw Error("cannot use ':load-flake' without a path specified. (Use '.' for the current working directory.)");
 
-    auto flakeRef = parseFlakeRef(flakeRefS, absPath("."), true);
+    auto flakeRef = parseFlakeRef(fetchSettings, flakeRefS, std::filesystem::current_path().string(), true);
     if (evalSettings.pureEval && !flakeRef.input.isLocked())
         throw Error("cannot use ':load-flake' on locked flake reference '%s' (use --impure to override)", flakeRefS);
 
     Value v;
 
     flake::callFlake(*state,
-        flake::lockFlake(*state, flakeRef,
+        flake::lockFlake(flakeSettings, *state, flakeRef,
             flake::LockFlags {
                 .updateLockFile = false,
                 .useRegistries = !evalSettings.pureEval,
@@ -783,9 +820,18 @@ void NixRepl::evalString(std::string s, Value & v)
 }
 
 
+void NixRepl::runNix(Path program, const Strings & args, const std::optional<std::string> & input)
+{
+    if (runNixPtr)
+        (*runNixPtr)(program, args, input);
+    else
+        throw Error("Cannot run '%s', no method of calling the Nix CLI provided", program);
+}
+
+
 std::unique_ptr<AbstractNixRepl> AbstractNixRepl::create(
    const LookupPath & lookupPath, nix::ref<Store> store, ref<EvalState> state,
-   std::function<AnnotatedValues()> getValues)
+   std::function<AnnotatedValues()> getValues, RunNix * runNix)
 {
     return std::make_unique<NixRepl>(
         lookupPath,

src/libcmd/repl.hh

@@ -19,9 +19,19 @@ struct AbstractNixRepl
 
     typedef std::vector<std::pair<Value*,std::string>> AnnotatedValues;
 
+    using RunNix = void(Path program, const Strings & args, const std::optional<std::string> & input);
+
+    /**
+     * @param runNix Function to run the nix CLI to support various
+     * `:<something>` commands. Optional; if not provided,
+     * everything else will still work fine, but those commands won't.
+     */
     static std::unique_ptr<AbstractNixRepl> create(
-        const LookupPath & lookupPath, nix::ref<Store> store, ref<EvalState> state,
-        std::function<AnnotatedValues()> getValues);
+        const LookupPath & lookupPath,
+        nix::ref<Store> store,
+        ref<EvalState> state,
+        std::function<AnnotatedValues()> getValues,
+        RunNix * runNix = nullptr);
 
     static ReplExitStatus runSimple(
         ref<EvalState> evalState,

src/libexpr-c/.version

@@ -0,0 +1 @@
+../../.version

src/libexpr-c/build-utils-meson

@@ -0,0 +1 @@
+../../build-utils-meson

src/libexpr-c/local.mk

@@ -15,7 +15,7 @@ libexprc_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libutilc) \
                      $(INCLUDE_libstore) $(INCLUDE_libstorec) \
                      $(INCLUDE_libexpr) $(INCLUDE_libexprc)
 
-libexprc_LIBS = libutil libutilc libstore libstorec libexpr
+libexprc_LIBS = libutil libutilc libstore libstorec libfetchers libexpr
 
 libexprc_LDFLAGS += $(THREAD_LDFLAGS)
 

src/libexpr-c/meson.build

@@ -0,0 +1,93 @@
+project('nix-expr-c', 'cpp',
+  version : files('.version'),
+  default_options : [
+    'cpp_std=c++2a',
+    # TODO(Qyriad): increase the warning level
+    'warning_level=1',
+    'debug=true',
+    'optimization=2',
+    'errorlogs=true', # Please print logs for tests that fail
+  ],
+  meson_version : '>= 1.1',
+  license : 'LGPL-2.1-or-later',
+)
+
+cxx = meson.get_compiler('cpp')
+
+subdir('build-utils-meson/deps-lists')
+
+configdata = configuration_data()
+
+deps_private_maybe_subproject = [
+  dependency('nix-util'),
+  dependency('nix-store'),
+  dependency('nix-expr'),
+]
+deps_public_maybe_subproject = [
+  dependency('nix-util-c'),
+  dependency('nix-store-c'),
+]
+subdir('build-utils-meson/subprojects')
+
+subdir('build-utils-meson/threads')
+
+# TODO rename, because it will conflict with downstream projects
+configdata.set_quoted('PACKAGE_VERSION', meson.project_version())
+
+config_h = configure_file(
+  configuration : configdata,
+  output : 'config-expr.h',
+)
+
+add_project_arguments(
+  # TODO(Qyriad): Yes this is how the autoconf+Make system did it.
+  # It would be nice for our headers to be idempotent instead.
+
+  # From C++ libraries, only for internals
+  '-include', 'config-util.hh',
+  '-include', 'config-store.hh',
+  '-include', 'config-expr.hh',
+
+  # From C libraries, for our public, installed headers too
+  '-include', 'config-util.h',
+  '-include', 'config-store.h',
+  '-include', 'config-expr.h',
+  language : 'cpp',
+)
+
+subdir('build-utils-meson/diagnostics')
+
+sources = files(
+  'nix_api_expr.cc',
+  'nix_api_external.cc',
+  'nix_api_value.cc',
+)
+
+include_dirs = [include_directories('.')]
+
+headers = [config_h] + files(
+  'nix_api_expr.h',
+  'nix_api_external.h',
+  'nix_api_value.h',
+)
+
+# TODO move this header to libexpr, maybe don't use it in tests?
+headers += files('nix_api_expr_internal.h')
+
+subdir('build-utils-meson/export-all-symbols')
+
+this_library = library(
+  'nixexprc',
+  sources,
+  dependencies : deps_public + deps_private + deps_other,
+  include_directories : include_dirs,
+  link_args: linker_export_flags,
+  prelink : true, # For C++ static initializers
+  install : true,
+)
+
+install_headers(headers, subdir : 'nix', preserve_path : true)
+
+libraries_private = []
+
+subdir('build-utils-meson/export')

src/libexpr-c/nix_api_expr.cc

@@ -1,12 +1,11 @@
 #include <cstring>
-#include <iostream>
 #include <stdexcept>
 #include <string>
 
-#include "config.hh"
 #include "eval.hh"
+#include "eval-gc.hh"
 #include "globals.hh"
-#include "util.hh"
+#include "eval-settings.hh"
 
 #include "nix_api_expr.h"
 #include "nix_api_expr_internal.h"
@@ -15,10 +14,10 @@
 #include "nix_api_util.h"
 #include "nix_api_util_internal.h"
 
-#ifdef HAVE_BOEHMGC
-#include <mutex>
-#define GC_INCLUDE_NEW 1
-#include "gc_cpp.h"
+#if HAVE_BOEHMGC
+#  include <mutex>
+#  define GC_INCLUDE_NEW 1
+#  include "gc_cpp.h"
 #endif
 
 nix_err nix_libexpr_init(nix_c_context * context)
@@ -42,45 +41,56 @@ nix_err nix_libexpr_init(nix_c_context * context)
 }
 
 nix_err nix_expr_eval_from_string(
-    nix_c_context * context, EvalState * state, const char * expr, const char * path, Value * value)
+    nix_c_context * context, EvalState * state, const char * expr, const char * path, nix_value * value)
 {
     if (context)
         context->last_err_code = NIX_OK;
     try {
         nix::Expr * parsedExpr = state->state.parseExprFromString(expr, state->state.rootPath(nix::CanonPath(path)));
-        state->state.eval(parsedExpr, *(nix::Value *) value);
-        state->state.forceValue(*(nix::Value *) value, nix::noPos);
+        state->state.eval(parsedExpr, value->value);
+        state->state.forceValue(value->value, nix::noPos);
     }
     NIXC_CATCH_ERRS
 }
 
-nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, Value * arg, Value * value)
+nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, nix_value * arg, nix_value * value)
 {
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        state->state.callFunction(*(nix::Value *) fn, *(nix::Value *) arg, *(nix::Value *) value, nix::noPos);
-        state->state.forceValue(*(nix::Value *) value, nix::noPos);
+        state->state.callFunction(fn->value, arg->value, value->value, nix::noPos);
+        state->state.forceValue(value->value, nix::noPos);
     }
     NIXC_CATCH_ERRS
 }
 
-nix_err nix_value_force(nix_c_context * context, EvalState * state, Value * value)
+nix_err nix_value_call_multi(nix_c_context * context, EvalState * state, nix_value * fn, size_t nargs, nix_value ** args, nix_value * value)
 {
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        state->state.forceValue(*(nix::Value *) value, nix::noPos);
+        state->state.callFunction(fn->value, nargs, (nix::Value * *)args, value->value, nix::noPos);
+        state->state.forceValue(value->value, nix::noPos);
     }
     NIXC_CATCH_ERRS
 }
 
-nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, Value * value)
+nix_err nix_value_force(nix_c_context * context, EvalState * state, nix_value * value)
 {
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        state->state.forceValueDeep(*(nix::Value *) value);
+        state->state.forceValue(value->value, nix::noPos);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        state->state.forceValueDeep(value->value);
     }
     NIXC_CATCH_ERRS
 }
@@ -95,7 +105,23 @@ EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c
             for (size_t i = 0; lookupPath_c[i] != nullptr; i++)
                 lookupPath.push_back(lookupPath_c[i]);
 
-        return new EvalState{nix::EvalState(nix::LookupPath::parse(lookupPath), store->ptr)};
+        void * p = ::operator new(
+            sizeof(EvalState),
+            static_cast<std::align_val_t>(alignof(EvalState)));
+        auto * p2 = static_cast<EvalState *>(p);
+        new (p) EvalState {
+            .fetchSettings = nix::fetchers::Settings{},
+            .settings = nix::EvalSettings{
+                nix::settings.readOnlyMode,
+            },
+            .state = nix::EvalState(
+                nix::LookupPath::parse(lookupPath),
+                store->ptr,
+                p2->fetchSettings,
+                p2->settings),
+        };
+        loadConfFile(p2->settings);
+        return p2;
     }
     NIXC_CATCH_ERRS_NULL
 }
@@ -105,7 +131,7 @@ void nix_state_free(EvalState * state)
     delete state;
 }
 
-#ifdef HAVE_BOEHMGC
+#if HAVE_BOEHMGC
 std::unordered_map<
     const void *,
     unsigned int,
@@ -170,9 +196,18 @@ nix_err nix_gc_decref(nix_c_context * context, const void *)
 void nix_gc_now() {}
 #endif
 
+nix_err nix_value_incref(nix_c_context * context, nix_value *x)
+{
+    return nix_gc_incref(context, (const void *) x);
+}
+nix_err nix_value_decref(nix_c_context * context, nix_value *x)
+{
+    return nix_gc_decref(context, (const void *) x);
+}
+
 void nix_gc_register_finalizer(void * obj, void * cd, void (*finalizer)(void * obj, void * cd))
 {
-#ifdef HAVE_BOEHMGC
+#if HAVE_BOEHMGC
     GC_REGISTER_FINALIZER(obj, finalizer, cd, 0, 0);
 #endif
 }

src/libexpr-c/nix_api_expr.h

@@ -3,25 +3,7 @@
 /** @defgroup libexpr libexpr
  * @brief Bindings to the Nix language evaluator
  *
- * Example (without error handling):
- * @code{.c}
- * int main() {
- *    nix_libexpr_init(NULL);
- *
- *    Store* store = nix_store_open(NULL, "dummy", NULL);
- *    EvalState* state = nix_state_create(NULL, NULL, store); // empty nix path
- *    Value *value = nix_alloc_value(NULL, state);
- *
- *    nix_expr_eval_from_string(NULL, state, "builtins.nixVersion", ".", value);
- *    nix_value_force(NULL, state, value);
- *    printf("nix version: %s\n", nix_get_string(NULL, value));
- *
- *    nix_gc_decref(NULL, value);
- *    nix_state_free(state);
- *    nix_store_free(store);
- *    return 0;
- *    }
- * @endcode
+ * See *[Embedding the Nix Evaluator](@ref nix_evaluator_example)* for an example.
  * @{
  */
 /** @file
@@ -30,6 +12,17 @@
 
 #include "nix_api_store.h"
 #include "nix_api_util.h"
+#include <stddef.h>
+
+#ifndef __has_c_attribute
+#  define __has_c_attribute(x) 0
+#endif
+
+#if __has_c_attribute(deprecated)
+#  define NIX_DEPRECATED(msg) [[deprecated(msg)]]
+#else
+#  define NIX_DEPRECATED(msg)
+#endif
 
 #ifdef __cplusplus
 extern "C" {
@@ -46,14 +39,23 @@ extern "C" {
  * @see nix_state_create
  */
 typedef struct EvalState EvalState; // nix::EvalState
-/**
- * @brief Represents a value in the Nix language.
+
+/** @brief A Nix language value, or thunk that may evaluate to a value.
+ *
+ * Values are the primary objects manipulated in the Nix language.
+ * They are considered to be immutable from a user's perspective, but the process of evaluating a value changes its
+ * ValueType if it was a thunk. After a value has been evaluated, its ValueType does not change.
+ *
+ * Evaluation in this context refers to the process of evaluating a single value object, also called "forcing" the
+ * value; see `nix_value_force`.
+ *
+ * The evaluator manages its own memory, but your use of the C API must follow the reference counting rules.
  *
- * Owned by the garbage collector.
- * @struct Value
  * @see value_manip
+ * @see nix_value_incref, nix_value_decref
  */
-typedef void Value; // nix::Value
+typedef struct nix_value nix_value;
+NIX_DEPRECATED("use nix_value instead") typedef nix_value Value;
 
 // Function prototypes
 /**
@@ -82,7 +84,7 @@ nix_err nix_libexpr_init(nix_c_context * context);
  * @return NIX_OK if the evaluation was successful, an error code otherwise.
  */
 nix_err nix_expr_eval_from_string(
-    nix_c_context * context, EvalState * state, const char * expr, const char * path, Value * value);
+    nix_c_context * context, EvalState * state, const char * expr, const char * path, nix_value * value);
 
 /**
  * @brief Calls a Nix function with an argument.
@@ -96,20 +98,55 @@ nix_err nix_expr_eval_from_string(
  * @see nix_init_apply() for a similar function that does not performs the call immediately, but stores it as a thunk.
  *      Note the different argument order.
  */
-nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, Value * arg, Value * value);
+nix_err nix_value_call(nix_c_context * context, EvalState * state, nix_value * fn, nix_value * arg, nix_value * value);
+
+/**
+ * @brief Calls a Nix function with multiple arguments.
+ *
+ * Technically these are functions that return functions. It is common for Nix
+ * functions to be curried, so this function is useful for calling them.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] state The state of the evaluation.
+ * @param[in] fn The Nix function to call.
+ * @param[in] nargs The number of arguments.
+ * @param[in] args The arguments to pass to the function.
+ * @param[out] value The result of the function call.
+ *
+ * @see nix_value_call     For the single argument primitive.
+ * @see NIX_VALUE_CALL           For a macro that wraps this function for convenience.
+ */
+nix_err nix_value_call_multi(
+    nix_c_context * context, EvalState * state, nix_value * fn, size_t nargs, nix_value ** args, nix_value * value);
+
+/**
+ * @brief Calls a Nix function with multiple arguments.
+ *
+ * Technically these are functions that return functions. It is common for Nix
+ * functions to be curried, so this function is useful for calling them.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] state The state of the evaluation.
+ * @param[out] value The result of the function call.
+ * @param[in] fn The Nix function to call.
+ * @param[in] args The arguments to pass to the function.
+ *
+ * @see nix_value_call_multi
+ */
+#define NIX_VALUE_CALL(context, state, value, fn, ...)                      \
+    do {                                                                    \
+        nix_value * args_array[] = {__VA_ARGS__};                           \
+        size_t nargs = sizeof(args_array) / sizeof(args_array[0]);          \
+        nix_value_call_multi(context, state, fn, nargs, args_array, value); \
+    } while (0)
 
 /**
  * @brief Forces the evaluation of a Nix value.
  *
- * The Nix interpreter is lazy, and not-yet-evaluated Values can be
+ * The Nix interpreter is lazy, and not-yet-evaluated values can be
  * of type NIX_TYPE_THUNK instead of their actual value.
  *
- * This function converts these Values into their final type.
- *
- * @note You don't need this function for basic API usage, since all functions
- * that return a value call it for you. The only place you will see a
- * NIX_TYPE_THUNK is in the arguments that are passed to a PrimOp function
- * you supplied to nix_alloc_primop.
+ * This function mutates such a `nix_value`, so that, if successful, it has its final type.
  *
  * @param[out] context Optional, stores error information
  * @param[in] state The state of the evaluation.
@@ -118,7 +155,7 @@ nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, V
  * @return NIX_OK if the force operation was successful, an error code
  * otherwise.
  */
-nix_err nix_value_force(nix_c_context * context, EvalState * state, Value * value);
+nix_err nix_value_force(nix_c_context * context, EvalState * state, nix_value * value);
 
 /**
  * @brief Forces the deep evaluation of a Nix value.
@@ -134,13 +171,13 @@ nix_err nix_value_force(nix_c_context * context, EvalState * state, Value * valu
  * @return NIX_OK if the deep force operation was successful, an error code
  * otherwise.
  */
-nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, Value * value);
+nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_value * value);
 
 /**
  * @brief Create a new Nix language evaluator state.
  *
  * @param[out] context Optional, stores error information
- * @param[in] lookupPath Array of strings corresponding to entries in NIX_PATH.
+ * @param[in] lookupPath Null-terminated array of strings corresponding to entries in NIX_PATH.
  * @param[in] store The Nix store to use.
  * @return A new Nix state or NULL on failure.
  */
@@ -168,6 +205,11 @@ void nix_state_free(EvalState * state);
  * you're done with a value returned by the evaluator.
  * @{
  */
+
+// TODO: Deprecate nix_gc_incref in favor of the type-specific reference counting functions?
+//       e.g. nix_value_incref.
+//       It gives implementors more flexibility, and adds safety, so that generated
+//       bindings can be used without fighting the host type system (where applicable).
 /**
  * @brief Increment the garbage collector reference counter for the given object.
  *

src/libexpr-c/nix_api_expr_internal.h

@@ -1,12 +1,16 @@
 #ifndef NIX_API_EXPR_INTERNAL_H
 #define NIX_API_EXPR_INTERNAL_H
 
+#include "fetch-settings.hh"
 #include "eval.hh"
+#include "eval-settings.hh"
 #include "attr-set.hh"
 #include "nix_api_value.h"
 
 struct EvalState
 {
+    nix::fetchers::Settings fetchSettings;
+    nix::EvalSettings settings;
     nix::EvalState state;
 };
 
@@ -20,6 +24,11 @@ struct ListBuilder
     nix::ListBuilder builder;
 };
 
+struct nix_value
+{
+    nix::Value value;
+};
+
 struct nix_string_return
 {
     std::string str;

src/libexpr-c/nix_api_external.cc

@@ -14,7 +14,7 @@
 
 #include <nlohmann/json.hpp>
 
-#ifdef HAVE_BOEHMGC
+#if HAVE_BOEHMGC
 # include "gc/gc.h"
 # define GC_INCLUDE_NEW 1
 # include "gc_cpp.h"
@@ -115,7 +115,7 @@ public:
     /**
      * Compare to another value of the same type.
      */
-    virtual bool operator==(const ExternalValueBase & b) const override
+    virtual bool operator==(const ExternalValueBase & b) const noexcept override
     {
         if (!desc.equal) {
             return false;
@@ -174,7 +174,7 @@ ExternalValue * nix_create_external_value(nix_c_context * context, NixCExternalV
         context->last_err_code = NIX_OK;
     try {
         auto ret = new
-#ifdef HAVE_BOEHMGC
+#if HAVE_BOEHMGC
             (GC)
 #endif
                 NixCExternalValue(*desc, v);

src/libexpr-c/nix_api_external.h

@@ -48,7 +48,7 @@ void nix_set_string_return(nix_string_return * str, const char * c);
  * Print to the nix_printer
  *
  * @param[out] context Optional, stores error information
- * @param printer The nix_printer to print to
+ * @param[out] printer The nix_printer to print to
  * @param[in] str The string to print
  * @returns NIX_OK if everything worked
  */
@@ -136,7 +136,7 @@ typedef struct NixCExternalValueDesc
      * or setting it to the empty string, will make the conversion throw an error.
      */
     void (*printValueAsJSON)(
-        void * self, EvalState *, bool strict, nix_string_context * c, bool copyToStore, nix_string_return * res);
+        void * self, EvalState * state, bool strict, nix_string_context * c, bool copyToStore, nix_string_return * res);
     /**
      * @brief Convert the external value to XML
      *
@@ -155,7 +155,7 @@ typedef struct NixCExternalValueDesc
      */
     void (*printValueAsXML)(
         void * self,
-        EvalState *,
+        EvalState * state,
         int strict,
         int location,
         void * doc,

src/libexpr-c/nix_api_value.cc

@@ -14,27 +14,59 @@
 #include "nix_api_value.h"
 #include "value/context.hh"
 
-#ifdef HAVE_BOEHMGC
+#if HAVE_BOEHMGC
 #  include "gc/gc.h"
 #  define GC_INCLUDE_NEW 1
 #  include "gc_cpp.h"
 #endif
 
-// Helper function to throw an exception if value is null
-static const nix::Value & check_value_not_null(const Value * value)
+// Internal helper functions to check [in] and [out] `Value *` parameters
+static const nix::Value & check_value_not_null(const nix_value * value)
 {
     if (!value) {
-        throw std::runtime_error("Value is null");
+        throw std::runtime_error("nix_value is null");
     }
     return *((const nix::Value *) value);
 }
 
-static nix::Value & check_value_not_null(Value * value)
+static nix::Value & check_value_not_null(nix_value * value)
 {
     if (!value) {
-        throw std::runtime_error("Value is null");
+        throw std::runtime_error("nix_value is null");
     }
-    return *((nix::Value *) value);
+    return value->value;
+}
+
+static const nix::Value & check_value_in(const nix_value * value)
+{
+    auto & v = check_value_not_null(value);
+    if (!v.isValid()) {
+        throw std::runtime_error("Uninitialized nix_value");
+    }
+    return v;
+}
+
+static nix::Value & check_value_in(nix_value * value)
+{
+    auto & v = check_value_not_null(value);
+    if (!v.isValid()) {
+        throw std::runtime_error("Uninitialized nix_value");
+    }
+    return v;
+}
+
+static nix::Value & check_value_out(nix_value * value)
+{
+    auto & v = check_value_not_null(value);
+    if (v.isValid()) {
+        throw std::runtime_error("nix_value already initialized. Variables are immutable");
+    }
+    return v;
+}
+
+static inline nix_value * as_nix_value_ptr(nix::Value * v)
+{
+    return reinterpret_cast<nix_value *>(v);
 }
 
 /**
@@ -46,10 +78,43 @@ static void nix_c_primop_wrapper(
     PrimOpFun f, void * userdata, nix::EvalState & state, const nix::PosIdx pos, nix::Value ** args, nix::Value & v)
 {
     nix_c_context ctx;
-    f(userdata, &ctx, (EvalState *) &state, (Value **) args, (Value *) &v);
-    /* TODO: In the future, this should throw different errors depending on the error code */
-    if (ctx.last_err_code != NIX_OK)
-        state.error<nix::EvalError>("Error from builtin function: %s", *ctx.last_err).atPos(pos).debugThrow();
+
+    // v currently has a thunk, but the C API initializers require an uninitialized value.
+    //
+    // We can't destroy the thunk, because that makes it impossible to retry,
+    // which is needed for tryEval and for evaluation drivers that evaluate more
+    // than one value (e.g. an attrset with two derivations, both of which
+    // reference v).
+    //
+    // Instead we create a temporary value, and then assign the result to v.
+    // This does not give the primop definition access to the thunk, but that's
+    // ok because we don't see a need for this yet (e.g. inspecting thunks,
+    // or maybe something to make blackholes work better; we don't know).
+    nix::Value vTmp;
+
+    f(userdata, &ctx, (EvalState *) &state, (nix_value **) args, (nix_value *) &vTmp);
+
+    if (ctx.last_err_code != NIX_OK) {
+        /* TODO: Throw different errors depending on the error code */
+        state.error<nix::EvalError>("Error from custom function: %s", *ctx.last_err).atPos(pos).debugThrow();
+    }
+
+    if (!vTmp.isValid()) {
+        state.error<nix::EvalError>("Implementation error in custom function: return value was not initialized")
+            .atPos(pos)
+            .debugThrow();
+    }
+
+    if (vTmp.type() == nix::nThunk) {
+        // We might allow this in the future if it makes sense for the evaluator
+        // e.g. implementing tail recursion by returning a thunk to the next
+        // "iteration". Until then, this is most likely a mistake or misunderstanding.
+        state.error<nix::EvalError>("Implementation error in custom function: return value must not be a thunk")
+            .atPos(pos)
+            .debugThrow();
+    }
+
+    v = vTmp;
 }
 
 PrimOp * nix_alloc_primop(
@@ -66,7 +131,7 @@ PrimOp * nix_alloc_primop(
     try {
         using namespace std::placeholders;
         auto p = new
-#ifdef HAVE_BOEHMGC
+#if HAVE_BOEHMGC
             (GC)
 #endif
                 nix::PrimOp{
@@ -94,24 +159,24 @@ nix_err nix_register_primop(nix_c_context * context, PrimOp * primOp)
     NIXC_CATCH_ERRS
 }
 
-Value * nix_alloc_value(nix_c_context * context, EvalState * state)
+nix_value * nix_alloc_value(nix_c_context * context, EvalState * state)
 {
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        Value * res = state->state.allocValue();
+        nix_value * res = as_nix_value_ptr(state->state.allocValue());
         nix_gc_incref(nullptr, res);
         return res;
     }
     NIXC_CATCH_ERRS_NULL
 }
 
-ValueType nix_get_type(nix_c_context * context, const Value * value)
+ValueType nix_get_type(nix_c_context * context, const nix_value * value)
 {
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        auto & v = check_value_not_null(value);
+        auto & v = check_value_in(value);
         using namespace nix;
         switch (v.type()) {
         case nThunk:
@@ -142,48 +207,49 @@ ValueType nix_get_type(nix_c_context * context, const Value * value)
     NIXC_CATCH_ERRS_RES(NIX_TYPE_NULL);
 }
 
-const char * nix_get_typename(nix_c_context * context, const Value * value)
+const char * nix_get_typename(nix_c_context * context, const nix_value * value)
 {
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        auto & v = check_value_not_null(value);
+        auto & v = check_value_in(value);
         auto s = nix::showType(v);
         return strdup(s.c_str());
     }
     NIXC_CATCH_ERRS_NULL
 }
 
-bool nix_get_bool(nix_c_context * context, const Value * value)
+bool nix_get_bool(nix_c_context * context, const nix_value * value)
 {
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        auto & v = check_value_not_null(value);
+        auto & v = check_value_in(value);
         assert(v.type() == nix::nBool);
         return v.boolean();
     }
     NIXC_CATCH_ERRS_RES(false);
 }
 
-nix_err nix_get_string(nix_c_context * context, const Value * value, nix_get_string_callback callback, void * user_data)
+nix_err
+nix_get_string(nix_c_context * context, const nix_value * value, nix_get_string_callback callback, void * user_data)
 {
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        auto & v = check_value_not_null(value);
+        auto & v = check_value_in(value);
         assert(v.type() == nix::nString);
         call_nix_get_string_callback(v.c_str(), callback, user_data);
     }
     NIXC_CATCH_ERRS
 }
 
-const char * nix_get_path_string(nix_c_context * context, const Value * value)
+const char * nix_get_path_string(nix_c_context * context, const nix_value * value)
 {
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        auto & v = check_value_not_null(value);
+        auto & v = check_value_in(value);
         assert(v.type() == nix::nPath);
         // NOTE (from @yorickvP)
         // v._path.path should work but may not be how Eelco intended it.
@@ -197,95 +263,95 @@ const char * nix_get_path_string(nix_c_context * context, const Value * value)
     NIXC_CATCH_ERRS_NULL
 }
 
-unsigned int nix_get_list_size(nix_c_context * context, const Value * value)
+unsigned int nix_get_list_size(nix_c_context * context, const nix_value * value)
 {
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        auto & v = check_value_not_null(value);
+        auto & v = check_value_in(value);
         assert(v.type() == nix::nList);
         return v.listSize();
     }
     NIXC_CATCH_ERRS_RES(0);
 }
 
-unsigned int nix_get_attrs_size(nix_c_context * context, const Value * value)
+unsigned int nix_get_attrs_size(nix_c_context * context, const nix_value * value)
 {
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        auto & v = check_value_not_null(value);
+        auto & v = check_value_in(value);
         assert(v.type() == nix::nAttrs);
         return v.attrs()->size();
     }
     NIXC_CATCH_ERRS_RES(0);
 }
 
-double nix_get_float(nix_c_context * context, const Value * value)
+double nix_get_float(nix_c_context * context, const nix_value * value)
 {
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        auto & v = check_value_not_null(value);
+        auto & v = check_value_in(value);
         assert(v.type() == nix::nFloat);
         return v.fpoint();
     }
     NIXC_CATCH_ERRS_RES(0.0);
 }
 
-int64_t nix_get_int(nix_c_context * context, const Value * value)
+int64_t nix_get_int(nix_c_context * context, const nix_value * value)
 {
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        auto & v = check_value_not_null(value);
+        auto & v = check_value_in(value);
         assert(v.type() == nix::nInt);
-        return v.integer();
+        return v.integer().value;
     }
     NIXC_CATCH_ERRS_RES(0);
 }
 
-ExternalValue * nix_get_external(nix_c_context * context, Value * value)
+ExternalValue * nix_get_external(nix_c_context * context, nix_value * value)
 {
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        auto & v = check_value_not_null(value);
+        auto & v = check_value_out(value);
         assert(v.type() == nix::nExternal);
         return (ExternalValue *) v.external();
     }
     NIXC_CATCH_ERRS_NULL;
 }
 
-Value * nix_get_list_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int ix)
+nix_value * nix_get_list_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int ix)
 {
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        auto & v = check_value_not_null(value);
+        auto & v = check_value_in(value);
         assert(v.type() == nix::nList);
         auto * p = v.listElems()[ix];
         nix_gc_incref(nullptr, p);
         if (p != nullptr)
             state->state.forceValue(*p, nix::noPos);
-        return (Value *) p;
+        return as_nix_value_ptr(p);
     }
     NIXC_CATCH_ERRS_NULL
 }
 
-Value * nix_get_attr_byname(nix_c_context * context, const Value * value, EvalState * state, const char * name)
+nix_value * nix_get_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name)
 {
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        auto & v = check_value_not_null(value);
+        auto & v = check_value_in(value);
         assert(v.type() == nix::nAttrs);
         nix::Symbol s = state->state.symbols.create(name);
         auto attr = v.attrs()->get(s);
         if (attr) {
             nix_gc_incref(nullptr, attr->value);
             state->state.forceValue(*attr->value, nix::noPos);
-            return attr->value;
+            return as_nix_value_ptr(attr->value);
         }
         nix_set_err_msg(context, NIX_ERR_KEY, "missing attribute");
         return nullptr;
@@ -293,12 +359,12 @@ Value * nix_get_attr_byname(nix_c_context * context, const Value * value, EvalSt
     NIXC_CATCH_ERRS_NULL
 }
 
-bool nix_has_attr_byname(nix_c_context * context, const Value * value, EvalState * state, const char * name)
+bool nix_has_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name)
 {
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        auto & v = check_value_not_null(value);
+        auto & v = check_value_in(value);
         assert(v.type() == nix::nAttrs);
         nix::Symbol s = state->state.symbols.create(name);
         auto attr = v.attrs()->get(s);
@@ -309,102 +375,103 @@ bool nix_has_attr_byname(nix_c_context * context, const Value * value, EvalState
     NIXC_CATCH_ERRS_RES(false);
 }
 
-Value *
-nix_get_attr_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int i, const char ** name)
+nix_value * nix_get_attr_byidx(
+    nix_c_context * context, const nix_value * value, EvalState * state, unsigned int i, const char ** name)
 {
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        auto & v = check_value_not_null(value);
+        auto & v = check_value_in(value);
         const nix::Attr & a = (*v.attrs())[i];
-        *name = ((const std::string &) (state->state.symbols[a.name])).c_str();
+        *name = state->state.symbols[a.name].c_str();
         nix_gc_incref(nullptr, a.value);
         state->state.forceValue(*a.value, nix::noPos);
-        return a.value;
+        return as_nix_value_ptr(a.value);
     }
     NIXC_CATCH_ERRS_NULL
 }
 
-const char * nix_get_attr_name_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int i)
+const char *
+nix_get_attr_name_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int i)
 {
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        auto & v = check_value_not_null(value);
+        auto & v = check_value_in(value);
         const nix::Attr & a = (*v.attrs())[i];
-        return ((const std::string &) (state->state.symbols[a.name])).c_str();
+        return state->state.symbols[a.name].c_str();
     }
     NIXC_CATCH_ERRS_NULL
 }
 
-nix_err nix_init_bool(nix_c_context * context, Value * value, bool b)
+nix_err nix_init_bool(nix_c_context * context, nix_value * value, bool b)
 {
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        auto & v = check_value_not_null(value);
+        auto & v = check_value_out(value);
         v.mkBool(b);
     }
     NIXC_CATCH_ERRS
 }
 
 // todo string context
-nix_err nix_init_string(nix_c_context * context, Value * value, const char * str)
+nix_err nix_init_string(nix_c_context * context, nix_value * value, const char * str)
 {
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        auto & v = check_value_not_null(value);
+        auto & v = check_value_out(value);
         v.mkString(std::string_view(str));
     }
     NIXC_CATCH_ERRS
 }
 
-nix_err nix_init_path_string(nix_c_context * context, EvalState * s, Value * value, const char * str)
+nix_err nix_init_path_string(nix_c_context * context, EvalState * s, nix_value * value, const char * str)
 {
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        auto & v = check_value_not_null(value);
+        auto & v = check_value_out(value);
         v.mkPath(s->state.rootPath(nix::CanonPath(str)));
     }
     NIXC_CATCH_ERRS
 }
 
-nix_err nix_init_float(nix_c_context * context, Value * value, double d)
+nix_err nix_init_float(nix_c_context * context, nix_value * value, double d)
 {
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        auto & v = check_value_not_null(value);
+        auto & v = check_value_out(value);
         v.mkFloat(d);
     }
     NIXC_CATCH_ERRS
 }
 
-nix_err nix_init_int(nix_c_context * context, Value * value, int64_t i)
+nix_err nix_init_int(nix_c_context * context, nix_value * value, int64_t i)
 {
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        auto & v = check_value_not_null(value);
+        auto & v = check_value_out(value);
         v.mkInt(i);
     }
     NIXC_CATCH_ERRS
 }
 
-nix_err nix_init_null(nix_c_context * context, Value * value)
+nix_err nix_init_null(nix_c_context * context, nix_value * value)
 {
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        auto & v = check_value_not_null(value);
+        auto & v = check_value_out(value);
         v.mkNull();
     }
     NIXC_CATCH_ERRS
 }
 
-nix_err nix_init_apply(nix_c_context * context, Value * value, Value * fn, Value * arg)
+nix_err nix_init_apply(nix_c_context * context, nix_value * value, nix_value * fn, nix_value * arg)
 {
     if (context)
         context->last_err_code = NIX_OK;
@@ -417,12 +484,12 @@ nix_err nix_init_apply(nix_c_context * context, Value * value, Value * fn, Value
     NIXC_CATCH_ERRS
 }
 
-nix_err nix_init_external(nix_c_context * context, Value * value, ExternalValue * val)
+nix_err nix_init_external(nix_c_context * context, nix_value * value, ExternalValue * val)
 {
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        auto & v = check_value_not_null(value);
+        auto & v = check_value_out(value);
         auto r = (nix::ExternalValueBase *) val;
         v.mkExternal(r);
     }
@@ -444,7 +511,8 @@ ListBuilder * nix_make_list_builder(nix_c_context * context, EvalState * state,
     NIXC_CATCH_ERRS_NULL
 }
 
-nix_err nix_list_builder_insert(nix_c_context * context, ListBuilder * list_builder, unsigned int index, Value * value)
+nix_err
+nix_list_builder_insert(nix_c_context * context, ListBuilder * list_builder, unsigned int index, nix_value * value)
 {
     if (context)
         context->last_err_code = NIX_OK;
@@ -464,46 +532,46 @@ void nix_list_builder_free(ListBuilder * list_builder)
 #endif
 }
 
-nix_err nix_make_list(nix_c_context * context, ListBuilder * list_builder, Value * value)
+nix_err nix_make_list(nix_c_context * context, ListBuilder * list_builder, nix_value * value)
 {
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        auto & v = check_value_not_null(value);
+        auto & v = check_value_out(value);
         v.mkList(list_builder->builder);
     }
     NIXC_CATCH_ERRS
 }
 
-nix_err nix_init_primop(nix_c_context * context, Value * value, PrimOp * p)
+nix_err nix_init_primop(nix_c_context * context, nix_value * value, PrimOp * p)
 {
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        auto & v = check_value_not_null(value);
+        auto & v = check_value_out(value);
         v.mkPrimOp((nix::PrimOp *) p);
     }
     NIXC_CATCH_ERRS
 }
 
-nix_err nix_copy_value(nix_c_context * context, Value * value, Value * source)
+nix_err nix_copy_value(nix_c_context * context, nix_value * value, const nix_value * source)
 {
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        auto & v = check_value_not_null(value);
-        auto & s = check_value_not_null(source);
+        auto & v = check_value_out(value);
+        auto & s = check_value_in(source);
         v = s;
     }
     NIXC_CATCH_ERRS
 }
 
-nix_err nix_make_attrs(nix_c_context * context, Value * value, BindingsBuilder * b)
+nix_err nix_make_attrs(nix_c_context * context, nix_value * value, BindingsBuilder * b)
 {
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        auto & v = check_value_not_null(value);
+        auto & v = check_value_out(value);
         v.mkAttrs(b->builder);
     }
     NIXC_CATCH_ERRS
@@ -524,7 +592,7 @@ BindingsBuilder * nix_make_bindings_builder(nix_c_context * context, EvalState *
     NIXC_CATCH_ERRS_NULL
 }
 
-nix_err nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * bb, const char * name, Value * value)
+nix_err nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * bb, const char * name, nix_value * value)
 {
     if (context)
         context->last_err_code = NIX_OK;
@@ -545,12 +613,12 @@ void nix_bindings_builder_free(BindingsBuilder * bb)
 #endif
 }
 
-nix_realised_string * nix_string_realise(nix_c_context * context, EvalState * state, Value * value, bool isIFD)
+nix_realised_string * nix_string_realise(nix_c_context * context, EvalState * state, nix_value * value, bool isIFD)
 {
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        auto & v = check_value_not_null(value);
+        auto & v = check_value_in(value);
         nix::NixStringContext stringContext;
         auto rawStr = state->state.coerceToString(nix::noPos, v, stringContext, "while realising a string").toOwned();
         nix::StorePathSet storePaths;

src/libexpr-c/nix_api_value.h

@@ -35,8 +35,11 @@ typedef enum {
 } ValueType;
 
 // forward declarations
-typedef void Value;
+typedef struct nix_value nix_value;
 typedef struct EvalState EvalState;
+
+[[deprecated("use nix_value instead")]] typedef nix_value Value;
+
 // type defs
 /** @brief Stores an under-construction set of bindings
  * @ingroup value_manip
@@ -79,6 +82,7 @@ typedef struct nix_realised_string nix_realised_string;
  * @{
  */
 /** @brief Function pointer for primops
+ *
  * When you want to return an error, call nix_set_err_msg(context, NIX_ERR_UNKNOWN, "your error message here").
  *
  * @param[in] user_data Arbitrary data that was initially supplied to nix_alloc_primop
@@ -89,7 +93,8 @@ typedef struct nix_realised_string nix_realised_string;
  * @param[out] ret return value
  * @see nix_alloc_primop, nix_init_primop
  */
-typedef void (*PrimOpFun)(void * user_data, nix_c_context * context, EvalState * state, Value ** args, Value * ret);
+typedef void (*PrimOpFun)(
+    void * user_data, nix_c_context * context, EvalState * state, nix_value ** args, nix_value * ret);
 
 /** @brief Allocate a PrimOp
  *
@@ -141,13 +146,33 @@ nix_err nix_register_primop(nix_c_context * context, PrimOp * primOp);
  * @return value, or null in case of errors
  *
  */
-Value * nix_alloc_value(nix_c_context * context, EvalState * state);
+nix_value * nix_alloc_value(nix_c_context * context, EvalState * state);
+
+/**
+ * @brief Increment the garbage collector reference counter for the given `nix_value`.
+ *
+ * The Nix language evaluator C API keeps track of alive objects by reference counting.
+ * When you're done with a refcounted pointer, call nix_value_decref().
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] value The object to keep alive
+ */
+nix_err nix_value_incref(nix_c_context * context, nix_value * value);
+
+/**
+ * @brief Decrement the garbage collector reference counter for the given object
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] value The object to stop referencing
+ */
+nix_err nix_value_decref(nix_c_context * context, nix_value * value);
 
 /** @addtogroup value_manip Manipulating values
- * @brief Functions to inspect and change Nix language values, represented by Value.
+ * @brief Functions to inspect and change Nix language values, represented by nix_value.
  * @{
  */
-/** @name Getters
+/** @anchor getters
+ * @name Getters
  */
 /**@{*/
 /** @brief Get value type
@@ -155,7 +180,7 @@ Value * nix_alloc_value(nix_c_context * context, EvalState * state);
  * @param[in] value Nix value to inspect
  * @return type of nix value
  */
-ValueType nix_get_type(nix_c_context * context, const Value * value);
+ValueType nix_get_type(nix_c_context * context, const nix_value * value);
 
 /** @brief Get type name of value as defined in the evaluator
  * @param[out] context Optional, stores error information
@@ -163,14 +188,14 @@ ValueType nix_get_type(nix_c_context * context, const Value * value);
  * @return type name, owned string
  * @todo way to free the result
  */
-const char * nix_get_typename(nix_c_context * context, const Value * value);
+const char * nix_get_typename(nix_c_context * context, const nix_value * value);
 
 /** @brief Get boolean value
  * @param[out] context Optional, stores error information
  * @param[in] value Nix value to inspect
  * @return true or false, error info via context
  */
-bool nix_get_bool(nix_c_context * context, const Value * value);
+bool nix_get_bool(nix_c_context * context, const nix_value * value);
 
 /** @brief Get the raw string
  *
@@ -184,7 +209,7 @@ bool nix_get_bool(nix_c_context * context, const Value * value);
  * @return error code, NIX_OK on success.
  */
 nix_err
-nix_get_string(nix_c_context * context, const Value * value, nix_get_string_callback callback, void * user_data);
+nix_get_string(nix_c_context * context, const nix_value * value, nix_get_string_callback callback, void * user_data);
 
 /** @brief Get path as string
  * @param[out] context Optional, stores error information
@@ -192,42 +217,42 @@ nix_get_string(nix_c_context * context, const Value * value, nix_get_string_call
  * @return string
  * @return NULL in case of error.
  */
-const char * nix_get_path_string(nix_c_context * context, const Value * value);
+const char * nix_get_path_string(nix_c_context * context, const nix_value * value);
 
 /** @brief Get the length of a list
  * @param[out] context Optional, stores error information
  * @param[in] value Nix value to inspect
  * @return length of list, error info via context
  */
-unsigned int nix_get_list_size(nix_c_context * context, const Value * value);
+unsigned int nix_get_list_size(nix_c_context * context, const nix_value * value);
 
 /** @brief Get the element count of an attrset
  * @param[out] context Optional, stores error information
  * @param[in] value Nix value to inspect
  * @return attrset element count, error info via context
  */
-unsigned int nix_get_attrs_size(nix_c_context * context, const Value * value);
+unsigned int nix_get_attrs_size(nix_c_context * context, const nix_value * value);
 
 /** @brief Get float value in 64 bits
  * @param[out] context Optional, stores error information
  * @param[in] value Nix value to inspect
  * @return float contents, error info via context
  */
-double nix_get_float(nix_c_context * context, const Value * value);
+double nix_get_float(nix_c_context * context, const nix_value * value);
 
 /** @brief Get int value
  * @param[out] context Optional, stores error information
  * @param[in] value Nix value to inspect
  * @return int contents, error info via context
  */
-int64_t nix_get_int(nix_c_context * context, const Value * value);
+int64_t nix_get_int(nix_c_context * context, const nix_value * value);
 
 /** @brief Get external reference
  * @param[out] context Optional, stores error information
  * @param[in] value Nix value to inspect
  * @return reference to external, NULL in case of error
  */
-ExternalValue * nix_get_external(nix_c_context * context, Value *);
+ExternalValue * nix_get_external(nix_c_context * context, nix_value *);
 
 /** @brief Get the ix'th element of a list
  *
@@ -238,7 +263,7 @@ ExternalValue * nix_get_external(nix_c_context * context, Value *);
  * @param[in] ix list element to get
  * @return value, NULL in case of errors
  */
-Value * nix_get_list_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int ix);
+nix_value * nix_get_list_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int ix);
 
 /** @brief Get an attr by name
  *
@@ -249,7 +274,7 @@ Value * nix_get_list_byidx(nix_c_context * context, const Value * value, EvalSta
  * @param[in] name attribute name
  * @return value, NULL in case of errors
  */
-Value * nix_get_attr_byname(nix_c_context * context, const Value * value, EvalState * state, const char * name);
+nix_value * nix_get_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name);
 
 /** @brief Check if an attribute name exists on a value
  * @param[out] context Optional, stores error information
@@ -258,7 +283,7 @@ Value * nix_get_attr_byname(nix_c_context * context, const Value * value, EvalSt
  * @param[in] name attribute name
  * @return value, error info via context
  */
-bool nix_has_attr_byname(nix_c_context * context, const Value * value, EvalState * state, const char * name);
+bool nix_has_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name);
 
 /** @brief Get an attribute by index in the sorted bindings
  *
@@ -272,8 +297,8 @@ bool nix_has_attr_byname(nix_c_context * context, const Value * value, EvalState
  * @param[out] name will store a pointer to the attribute name
  * @return value, NULL in case of errors
  */
-Value *
-nix_get_attr_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int i, const char ** name);
+nix_value * nix_get_attr_byidx(
+    nix_c_context * context, const nix_value * value, EvalState * state, unsigned int i, const char ** name);
 
 /** @brief Get an attribute name by index in the sorted bindings
  *
@@ -286,7 +311,8 @@ nix_get_attr_byidx(nix_c_context * context, const Value * value, EvalState * sta
  * @param[in] i attribute index
  * @return name, NULL in case of errors
  */
-const char * nix_get_attr_name_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int i);
+const char *
+nix_get_attr_name_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int i);
 
 /**@}*/
 /** @name Initializers
@@ -303,7 +329,7 @@ const char * nix_get_attr_name_byidx(nix_c_context * context, const Value * valu
  * @param[in] b the boolean value
  * @return error code, NIX_OK on success.
  */
-nix_err nix_init_bool(nix_c_context * context, Value * value, bool b);
+nix_err nix_init_bool(nix_c_context * context, nix_value * value, bool b);
 
 /** @brief Set a string
  * @param[out] context Optional, stores error information
@@ -311,7 +337,7 @@ nix_err nix_init_bool(nix_c_context * context, Value * value, bool b);
  * @param[in] str the string, copied
  * @return error code, NIX_OK on success.
  */
-nix_err nix_init_string(nix_c_context * context, Value * value, const char * str);
+nix_err nix_init_string(nix_c_context * context, nix_value * value, const char * str);
 
 /** @brief Set a path
  * @param[out] context Optional, stores error information
@@ -319,7 +345,7 @@ nix_err nix_init_string(nix_c_context * context, Value * value, const char * str
  * @param[in] str the path string, copied
  * @return error code, NIX_OK on success.
  */
-nix_err nix_init_path_string(nix_c_context * context, EvalState * s, Value * value, const char * str);
+nix_err nix_init_path_string(nix_c_context * context, EvalState * s, nix_value * value, const char * str);
 
 /** @brief Set a float
  * @param[out] context Optional, stores error information
@@ -327,7 +353,7 @@ nix_err nix_init_path_string(nix_c_context * context, EvalState * s, Value * val
  * @param[in] d the float, 64-bits
  * @return error code, NIX_OK on success.
  */
-nix_err nix_init_float(nix_c_context * context, Value * value, double d);
+nix_err nix_init_float(nix_c_context * context, nix_value * value, double d);
 
 /** @brief Set an int
  * @param[out] context Optional, stores error information
@@ -336,13 +362,13 @@ nix_err nix_init_float(nix_c_context * context, Value * value, double d);
  * @return error code, NIX_OK on success.
  */
 
-nix_err nix_init_int(nix_c_context * context, Value * value, int64_t i);
+nix_err nix_init_int(nix_c_context * context, nix_value * value, int64_t i);
 /** @brief Set null
  * @param[out] context Optional, stores error information
  * @param[out] value Nix value to modify
  * @return error code, NIX_OK on success.
  */
-nix_err nix_init_null(nix_c_context * context, Value * value);
+nix_err nix_init_null(nix_c_context * context, nix_value * value);
 
 /** @brief Set the value to a thunk that will perform a function application when needed.
  *
@@ -358,7 +384,7 @@ nix_err nix_init_null(nix_c_context * context, Value * value);
  * @see nix_value_call() for a similar function that performs the call immediately and only stores the return value.
  *      Note the different argument order.
  */
-nix_err nix_init_apply(nix_c_context * context, Value * value, Value * fn, Value * arg);
+nix_err nix_init_apply(nix_c_context * context, nix_value * value, nix_value * fn, nix_value * arg);
 
 /** @brief Set an external value
  * @param[out] context Optional, stores error information
@@ -366,7 +392,7 @@ nix_err nix_init_apply(nix_c_context * context, Value * value, Value * fn, Value
  * @param[in] val the external value to set. Will be GC-referenced by the value.
  * @return error code, NIX_OK on success.
  */
-nix_err nix_init_external(nix_c_context * context, Value * value, ExternalValue * val);
+nix_err nix_init_external(nix_c_context * context, nix_value * value, ExternalValue * val);
 
 /** @brief Create a list from a list builder
  * @param[out] context Optional, stores error information
@@ -374,7 +400,7 @@ nix_err nix_init_external(nix_c_context * context, Value * value, ExternalValue
  * @param[out] value Nix value to modify
  * @return error code, NIX_OK on success.
  */
-nix_err nix_make_list(nix_c_context * context, ListBuilder * list_builder, Value * value);
+nix_err nix_make_list(nix_c_context * context, ListBuilder * list_builder, nix_value * value);
 
 /** @brief Create a list builder
  * @param[out] context Optional, stores error information
@@ -391,7 +417,8 @@ ListBuilder * nix_make_list_builder(nix_c_context * context, EvalState * state,
  * @param[in] value value to insert
  * @return error code, NIX_OK on success.
  */
-nix_err nix_list_builder_insert(nix_c_context * context, ListBuilder * list_builder, unsigned int index, Value * value);
+nix_err
+nix_list_builder_insert(nix_c_context * context, ListBuilder * list_builder, unsigned int index, nix_value * value);
 
 /** @brief Free a list builder
  *
@@ -406,7 +433,7 @@ void nix_list_builder_free(ListBuilder * list_builder);
  * @param[in] b bindings builder to use. Make sure to unref this afterwards.
  * @return error code, NIX_OK on success.
  */
-nix_err nix_make_attrs(nix_c_context * context, Value * value, BindingsBuilder * b);
+nix_err nix_make_attrs(nix_c_context * context, nix_value * value, BindingsBuilder * b);
 
 /** @brief Set primop
  * @param[out] context Optional, stores error information
@@ -415,14 +442,14 @@ nix_err nix_make_attrs(nix_c_context * context, Value * value, BindingsBuilder *
  * @see nix_alloc_primop
  * @return error code, NIX_OK on success.
  */
-nix_err nix_init_primop(nix_c_context * context, Value * value, PrimOp * op);
+nix_err nix_init_primop(nix_c_context * context, nix_value * value, PrimOp * op);
 /** @brief Copy from another value
  * @param[out] context Optional, stores error information
  * @param[out] value Nix value to modify
  * @param[in] source value to copy from
  * @return error code, NIX_OK on success.
  */
-nix_err nix_copy_value(nix_c_context * context, Value * value, Value * source);
+nix_err nix_copy_value(nix_c_context * context, nix_value * value, const nix_value * source);
 /**@}*/
 
 /** @brief Create a bindings builder
@@ -442,7 +469,7 @@ BindingsBuilder * nix_make_bindings_builder(nix_c_context * context, EvalState *
  * @return error code, NIX_OK on success.
  */
 nix_err
-nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * builder, const char * name, Value * value);
+nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * builder, const char * name, nix_value * value);
 
 /** @brief Free a bindings builder
  *
@@ -469,7 +496,7 @@ void nix_bindings_builder_free(BindingsBuilder * builder);
                     You should set this to false when building for your application's purpose.
  * @return NULL if failed, are a new nix_realised_string, which must be freed with nix_realised_string_free
  */
-nix_realised_string * nix_string_realise(nix_c_context * context, EvalState * state, Value * value, bool isIFD);
+nix_realised_string * nix_string_realise(nix_c_context * context, EvalState * state, nix_value * value, bool isIFD);
 
 /** @brief Start of the string
  * @param[in] realised_string

src/libexpr-c/package.nix

@@ -0,0 +1,74 @@
+{ lib
+, stdenv
+, mkMesonDerivation
+
+, meson
+, ninja
+, pkg-config
+
+, nix-store-c
+, nix-expr
+
+# Configuration Options
+
+, version
+}:
+
+let
+  inherit (lib) fileset;
+in
+
+mkMesonDerivation (finalAttrs: {
+  pname = "nix-expr-c";
+  inherit version;
+
+  workDir = ./.;
+  fileset = fileset.unions [
+    ../../build-utils-meson
+    ./build-utils-meson
+    ../../.version
+    ./.version
+    ./meson.build
+    # ./meson.options
+    (fileset.fileFilter (file: file.hasExt "cc") ./.)
+    (fileset.fileFilter (file: file.hasExt "hh") ./.)
+    (fileset.fileFilter (file: file.hasExt "h") ./.)
+  ];
+
+  outputs = [ "out" "dev" ];
+
+  nativeBuildInputs = [
+    meson
+    ninja
+    pkg-config
+  ];
+
+  propagatedBuildInputs = [
+    nix-store-c
+    nix-expr
+  ];
+
+  preConfigure =
+    # "Inline" .version so it's not a symlink, and includes the suffix.
+    # Do the meson utils, without modification.
+    ''
+      chmod u+w ./.version
+      echo ${version} > ../../.version
+    '';
+
+  mesonFlags = [
+  ];
+
+  env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) {
+    LDFLAGS = "-fuse-ld=gold";
+  };
+
+  separateDebugInfo = !stdenv.hostPlatform.isStatic;
+
+  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
+
+  meta = {
+    platforms = lib.platforms.unix ++ lib.platforms.windows;
+  };
+
+})

src/libexpr/.version

@@ -0,0 +1 @@
+../../.version

src/libexpr/attr-path.cc

@@ -76,7 +76,7 @@ std::pair<Value *, PosIdx> findAlongAttrPath(EvalState & state, const std::strin
             if (!a) {
                 std::set<std::string> attrNames;
                 for (auto & attr : *v->attrs())
-                    attrNames.insert(state.symbols[attr.name]);
+                    attrNames.insert(std::string(state.symbols[attr.name]));
 
                 auto suggestions = Suggestions::bestMatches(attrNames, attr);
                 throw AttrPathNotFound(suggestions, "attribute '%1%' in selection path '%2%' not found", attr, attrPath);
@@ -134,7 +134,7 @@ std::pair<SourcePath, uint32_t> findPackageFilename(EvalState & state, Value & v
         return {SourcePath{path.accessor, CanonPath(fn.substr(0, colon))}, lineno};
     } catch (std::invalid_argument & e) {
         fail();
-        abort();
+        unreachable();
     }
 }
 

src/libexpr/attr-set.hh

@@ -5,7 +5,6 @@
 #include "symbol-table.hh"
 
 #include <algorithm>
-#include <optional>
 
 namespace nix {
 
@@ -28,9 +27,9 @@ struct Attr
     Attr(Symbol name, Value * value, PosIdx pos = noPos)
         : name(name), pos(pos), value(value) { };
     Attr() { };
-    bool operator < (const Attr & a) const
+    auto operator <=> (const Attr & a) const
     {
-        return name < a.name;
+        return name <=> a.name;
     }
 };
 

src/libexpr/build-utils-meson

@@ -0,0 +1 @@
+../../build-utils-meson

src/libexpr/call-flake.nix

@@ -4,7 +4,7 @@
 lockFileStr:
 
 # A mapping of lock file node IDs to { sourceInfo, subdir } attrsets,
-# with sourceInfo.outPath providing an InputAccessor to a previously
+# with sourceInfo.outPath providing an SourceAccessor to a previously
 # fetched tree. This is necessary for possibly unlocked inputs, in
 # particular the root input, but also --override-inputs pointing to
 # unlocked trees.

src/libexpr/eval-cache.cc

@@ -4,9 +4,30 @@
 #include "eval.hh"
 #include "eval-inline.hh"
 #include "store-api.hh"
+// Need specialization involving `SymbolStr` just in this one module.
+#include "strings-inline.hh"
 
 namespace nix::eval_cache {
 
+CachedEvalError::CachedEvalError(ref<AttrCursor> cursor, Symbol attr)
+    : EvalError(cursor->root->state, "cached failure of attribute '%s'", cursor->getAttrPathStr(attr))
+    , cursor(cursor), attr(attr)
+{ }
+
+void CachedEvalError::force()
+{
+    auto & v = cursor->forceValue();
+
+    if (v.type() == nAttrs) {
+        auto a = v.attrs()->get(this->attr);
+
+        state.forceValue(*a->value, a->pos);
+    }
+
+    // Shouldn't happen.
+    throw EvalError(state, "evaluation of cached failed attribute '%s' unexpectedly succeeded", cursor->getAttrPathStr(attr));
+}
+
 static const char * schema = R"sql(
 create table if not exists Attributes (
     parent      integer not null,
@@ -48,7 +69,7 @@ struct AttrDb
     {
         auto state(_state->lock());
 
-        Path cacheDir = getCacheDir() + "/nix/eval-cache-v5";
+        Path cacheDir = getCacheDir() + "/eval-cache-v5";
         createDirs(cacheDir);
 
         Path dbPath = cacheDir + "/" + fingerprint.to_string(HashFormat::Base16, false) + ".sqlite";
@@ -76,7 +97,7 @@ struct AttrDb
     {
         try {
             auto state(_state->lock());
-            if (!failed)
+            if (!failed && state->txn->active)
                 state->txn->commit();
             state->txn.reset();
         } catch (...) {
@@ -206,7 +227,7 @@ struct AttrDb
                 (key.first)
                 (symbols[key.second])
                 (AttrType::ListOfStrings)
-                (concatStringsSep("\t", l)).exec();
+                (dropEmptyInitThenConcatStringsSep("\t", l)).exec();
 
             return state->db.getLastInsertedRowId();
         });
@@ -307,7 +328,7 @@ struct AttrDb
             case AttrType::Bool:
                 return {{rowId, queryAttribute.getInt(2) != 0}};
             case AttrType::Int:
-                return {{rowId, int_t{queryAttribute.getInt(2)}}};
+                return {{rowId, int_t{NixInt{queryAttribute.getInt(2)}}}};
             case AttrType::ListOfStrings:
                 return {{rowId, tokenizeString<std::vector<std::string>>(queryAttribute.getStr(2), "\t")}};
             case AttrType::Missing:
@@ -416,12 +437,12 @@ std::vector<Symbol> AttrCursor::getAttrPath(Symbol name) const
 
 std::string AttrCursor::getAttrPathStr() const
 {
-    return concatStringsSep(".", root->state.symbols.resolve(getAttrPath()));
+    return dropEmptyInitThenConcatStringsSep(".", root->state.symbols.resolve(getAttrPath()));
 }
 
 std::string AttrCursor::getAttrPathStr(Symbol name) const
 {
-    return concatStringsSep(".", root->state.symbols.resolve(getAttrPath(name)));
+    return dropEmptyInitThenConcatStringsSep(".", root->state.symbols.resolve(getAttrPath(name)));
 }
 
 Value & AttrCursor::forceValue()
@@ -450,7 +471,7 @@ Value & AttrCursor::forceValue()
         else if (v.type() == nBool)
             cachedValue = {root->db->setBool(getKey(), v.boolean()), v.boolean()};
         else if (v.type() == nInt)
-            cachedValue = {root->db->setInt(getKey(), v.integer()), int_t{v.integer()}};
+            cachedValue = {root->db->setInt(getKey(), v.integer().value), int_t{v.integer()}};
         else if (v.type() == nAttrs)
             ; // FIXME: do something?
         else
@@ -465,12 +486,12 @@ Suggestions AttrCursor::getSuggestionsForAttr(Symbol name)
     auto attrNames = getAttrs();
     std::set<std::string> strAttrNames;
     for (auto & name : attrNames)
-        strAttrNames.insert(root->state.symbols[name]);
+        strAttrNames.insert(std::string(root->state.symbols[name]));
 
     return Suggestions::bestMatches(strAttrNames, root->state.symbols[name]);
 }
 
-std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErrors)
+std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name)
 {
     if (root->db) {
         if (!cachedValue)
@@ -487,12 +508,9 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErro
                 if (attr) {
                     if (std::get_if<missing_t>(&attr->second))
                         return nullptr;
-                    else if (std::get_if<failed_t>(&attr->second)) {
-                        if (forceErrors)
-                            debug("reevaluating failed cached attribute '%s'", getAttrPathStr(name));
-                        else
-                            throw CachedEvalError(root->state, "cached failure of attribute '%s'", getAttrPathStr(name));
-                    } else
+                    else if (std::get_if<failed_t>(&attr->second))
+                        throw CachedEvalError(ref(shared_from_this()), name);
+                    else
                         return std::make_shared<AttrCursor>(root,
                             std::make_pair(shared_from_this(), name), nullptr, std::move(attr));
                 }
@@ -537,9 +555,9 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(std::string_view name)
     return maybeGetAttr(root->state.symbols.create(name));
 }
 
-ref<AttrCursor> AttrCursor::getAttr(Symbol name, bool forceErrors)
+ref<AttrCursor> AttrCursor::getAttr(Symbol name)
 {
-    auto p = maybeGetAttr(name, forceErrors);
+    auto p = maybeGetAttr(name);
     if (!p)
         throw Error("attribute '%s' does not exist", getAttrPathStr(name));
     return ref(p);
@@ -550,11 +568,11 @@ ref<AttrCursor> AttrCursor::getAttr(std::string_view name)
     return getAttr(root->state.symbols.create(name));
 }
 
-OrSuggestions<ref<AttrCursor>> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force)
+OrSuggestions<ref<AttrCursor>> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath)
 {
     auto res = shared_from_this();
     for (auto & attr : attrPath) {
-        auto child = res->maybeGetAttr(attr, force);
+        auto child = res->maybeGetAttr(attr);
         if (!child) {
             auto suggestions = res->getSuggestionsForAttr(attr);
             return OrSuggestions<ref<AttrCursor>>::failed(suggestions);
@@ -751,8 +769,9 @@ bool AttrCursor::isDerivation()
 
 StorePath AttrCursor::forceDerivation()
 {
-    auto aDrvPath = getAttr(root->state.sDrvPath, true);
+    auto aDrvPath = getAttr(root->state.sDrvPath);
     auto drvPath = root->state.store->parseStorePath(aDrvPath->getString());
+    drvPath.requireDerivation();
     if (!root->state.store->isValidPath(drvPath) && !settings.readOnlyMode) {
         /* The eval cache contains 'drvPath', but the actual path has
            been garbage-collected. So force it to be regenerated. */

src/libexpr/eval-cache.hh

@@ -10,14 +10,28 @@
 
 namespace nix::eval_cache {
 
-MakeError(CachedEvalError, EvalError);
-
 struct AttrDb;
 class AttrCursor;
 
+struct CachedEvalError : EvalError
+{
+    const ref<AttrCursor> cursor;
+    const Symbol attr;
+
+    CachedEvalError(ref<AttrCursor> cursor, Symbol attr);
+
+    /**
+     * Evaluate this attribute, which should result in a regular
+     * `EvalError` exception being thrown.
+     */
+    [[noreturn]]
+    void force();
+};
+
 class EvalCache : public std::enable_shared_from_this<EvalCache>
 {
     friend class AttrCursor;
+    friend struct CachedEvalError;
 
     std::shared_ptr<AttrDb> db;
     EvalState & state;
@@ -73,6 +87,7 @@ typedef std::variant<
 class AttrCursor : public std::enable_shared_from_this<AttrCursor>
 {
     friend class EvalCache;
+    friend struct CachedEvalError;
 
     ref<EvalCache> root;
     typedef std::optional<std::pair<std::shared_ptr<AttrCursor>, Symbol>> Parent;
@@ -102,11 +117,11 @@ public:
 
     Suggestions getSuggestionsForAttr(Symbol name);
 
-    std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name, bool forceErrors = false);
+    std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name);
 
     std::shared_ptr<AttrCursor> maybeGetAttr(std::string_view name);
 
-    ref<AttrCursor> getAttr(Symbol name, bool forceErrors = false);
+    ref<AttrCursor> getAttr(Symbol name);
 
     ref<AttrCursor> getAttr(std::string_view name);
 
@@ -114,7 +129,7 @@ public:
      * Get an attribute along a chain of attrsets. Note that this does
      * not auto-call functors or functions.
      */
-    OrSuggestions<ref<AttrCursor>> findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force = false);
+    OrSuggestions<ref<AttrCursor>> findAlongAttrPath(const std::vector<Symbol> & attrPath);
 
     std::string getString();
 

src/libexpr/eval-error.cc

@@ -27,8 +27,7 @@ EvalErrorBuilder<T> & EvalErrorBuilder<T>::atPos(Value & value, PosIdx fallback)
 template<class T>
 EvalErrorBuilder<T> & EvalErrorBuilder<T>::withTrace(PosIdx pos, const std::string_view text)
 {
-    error.err.traces.push_front(
-        Trace{.pos = error.state.positions[pos], .hint = HintFmt(std::string(text))});
+    error.addTrace(error.state.positions[pos], text);
     return *this;
 }
 
@@ -71,15 +70,17 @@ EvalErrorBuilder<T>::addTrace(PosIdx pos, std::string_view formatString, const A
     return *this;
 }
 
+template<class T>
+EvalErrorBuilder<T> & EvalErrorBuilder<T>::setIsFromExpr()
+{
+    error.err.isFromExpr = true;
+    return *this;
+}
+
 template<class T>
 void EvalErrorBuilder<T>::debugThrow()
 {
-    if (error.state.debugRepl && !error.state.debugTraces.empty()) {
-        const DebugTrace & last = error.state.debugTraces.front();
-        const Env * env = &last.env;
-        const Expr * expr = &last.expr;
-        error.state.runDebugRepl(&error, *env, *expr);
-    }
+    error.state.runDebugRepl(&error);
 
     // `EvalState` is the only class that can construct an `EvalErrorBuilder`,
     // and it does so in dynamic storage. This is the final method called on
@@ -91,6 +92,15 @@ void EvalErrorBuilder<T>::debugThrow()
     throw error;
 }
 
+template<class T>
+void EvalErrorBuilder<T>::panic()
+{
+    logError(error.info());
+    printError("This is a bug! An unexpected condition occurred, causing the Nix evaluator to have to stop. If you could share a reproducible example or a core dump, please open an issue at https://github.com/NixOS/nix/issues");
+    abort();
+}
+
+template class EvalErrorBuilder<EvalBaseError>;
 template class EvalErrorBuilder<EvalError>;
 template class EvalErrorBuilder<AssertionError>;
 template class EvalErrorBuilder<ThrownError>;
@@ -99,7 +109,6 @@ template class EvalErrorBuilder<TypeError>;
 template class EvalErrorBuilder<UndefinedVarError>;
 template class EvalErrorBuilder<MissingArgumentError>;
 template class EvalErrorBuilder<InfiniteRecursionError>;
-template class EvalErrorBuilder<CachedEvalError>;
 template class EvalErrorBuilder<InvalidPathError>;
 
 }

src/libexpr/eval-error.hh

@@ -1,7 +1,5 @@
 #pragma once
 
-#include <algorithm>
-
 #include "error.hh"
 #include "pos-idx.hh"
 
@@ -15,27 +13,39 @@ class EvalState;
 template<class T>
 class EvalErrorBuilder;
 
-class EvalError : public Error
+/**
+ * Base class for all errors that occur during evaluation.
+ *
+ * Most subclasses should inherit from `EvalError` instead of this class.
+ */
+class EvalBaseError : public Error
 {
     template<class T>
     friend class EvalErrorBuilder;
 public:
     EvalState & state;
 
-    EvalError(EvalState & state, ErrorInfo && errorInfo)
+    EvalBaseError(EvalState & state, ErrorInfo && errorInfo)
         : Error(errorInfo)
         , state(state)
     {
     }
 
     template<typename... Args>
-    explicit EvalError(EvalState & state, const std::string & formatString, const Args &... formatArgs)
+    explicit EvalBaseError(EvalState & state, const std::string & formatString, const Args &... formatArgs)
         : Error(formatString, formatArgs...)
         , state(state)
     {
     }
 };
 
+/**
+ * `EvalError` is the base class for almost all errors that occur during evaluation.
+ *
+ * All instances of `EvalError` should show a degree of purity that allows them to be
+ * cached in pure mode. This means that they should not depend on the configuration or the overall environment.
+ */
+MakeError(EvalError, EvalBaseError);
 MakeError(ParseError, Error);
 MakeError(AssertionError, EvalError);
 MakeError(ThrownError, AssertionError);
@@ -43,7 +53,6 @@ MakeError(Abort, EvalError);
 MakeError(TypeError, EvalError);
 MakeError(UndefinedVarError, EvalError);
 MakeError(MissingArgumentError, EvalError);
-MakeError(CachedEvalError, EvalError);
 MakeError(InfiniteRecursionError, EvalError);
 
 struct InvalidPathError : public EvalError
@@ -91,6 +100,8 @@ public:
 
     [[nodiscard, gnu::noinline]] EvalErrorBuilder<T> & addTrace(PosIdx pos, HintFmt hint);
 
+    [[nodiscard, gnu::noinline]] EvalErrorBuilder<T> & setIsFromExpr();
+
     template<typename... Args>
     [[nodiscard, gnu::noinline]] EvalErrorBuilder<T> &
     addTrace(PosIdx pos, std::string_view formatString, const Args &... formatArgs);
@@ -99,6 +110,12 @@ public:
      * Delete the `EvalErrorBuilder` and throw the underlying exception.
      */
     [[gnu::noinline, gnu::noreturn]] void debugThrow();
+
+    /**
+     * A programming error or fatal condition occurred. Abort the process for core dump and debugging.
+     * This does not print a proper backtrace, because unwinding the stack is destructive.
+     */
+    [[gnu::noinline, gnu::noreturn]] void panic();
 };
 
 }

src/libexpr/eval-gc.cc

@@ -0,0 +1,117 @@
+#include "error.hh"
+#include "environment-variables.hh"
+#include "eval-settings.hh"
+#include "config-global.hh"
+#include "serialise.hh"
+#include "eval-gc.hh"
+
+#if HAVE_BOEHMGC
+
+#  include <pthread.h>
+#  if __FreeBSD__
+#    include <pthread_np.h>
+#  endif
+
+#  include <gc/gc.h>
+#  include <gc/gc_cpp.h>
+#  include <gc/gc_allocator.h>
+
+#  include <boost/coroutine2/coroutine.hpp>
+#  include <boost/coroutine2/protected_fixedsize_stack.hpp>
+#  include <boost/context/stack_context.hpp>
+
+#endif
+
+namespace nix {
+
+#if HAVE_BOEHMGC
+/* Called when the Boehm GC runs out of memory. */
+static void * oomHandler(size_t requested)
+{
+    /* Convert this to a proper C++ exception. */
+    throw std::bad_alloc();
+}
+
+static inline void initGCReal()
+{
+    /* Initialise the Boehm garbage collector. */
+
+    /* Don't look for interior pointers. This reduces the odds of
+       misdetection a bit. */
+    GC_set_all_interior_pointers(0);
+
+    /* We don't have any roots in data segments, so don't scan from
+       there. */
+    GC_set_no_dls(1);
+
+    /* Enable perf measurements. This is just a setting; not much of a
+       start of something. */
+    GC_start_performance_measurement();
+
+    GC_INIT();
+
+    GC_set_oom_fn(oomHandler);
+
+    /* Set the initial heap size to something fairly big (25% of
+       physical RAM, up to a maximum of 384 MiB) so that in most cases
+       we don't need to garbage collect at all.  (Collection has a
+       fairly significant overhead.)  The heap size can be overridden
+       through libgc's GC_INITIAL_HEAP_SIZE environment variable.  We
+       should probably also provide a nix.conf setting for this.  Note
+       that GC_expand_hp() causes a lot of virtual, but not physical
+       (resident) memory to be allocated.  This might be a problem on
+       systems that don't overcommit. */
+    if (!getEnv("GC_INITIAL_HEAP_SIZE")) {
+        size_t size = 32 * 1024 * 1024;
+#  if HAVE_SYSCONF && defined(_SC_PAGESIZE) && defined(_SC_PHYS_PAGES)
+        size_t maxSize = 384 * 1024 * 1024;
+        long pageSize = sysconf(_SC_PAGESIZE);
+        long pages = sysconf(_SC_PHYS_PAGES);
+        if (pageSize != -1)
+            size = (pageSize * pages) / 4; // 25% of RAM
+        if (size > maxSize)
+            size = maxSize;
+#  endif
+        debug("setting initial heap size to %1% bytes", size);
+        GC_expand_hp(size);
+    }
+}
+
+static size_t gcCyclesAfterInit = 0;
+
+size_t getGCCycles()
+{
+    assertGCInitialized();
+    return static_cast<size_t>(GC_get_gc_no()) - gcCyclesAfterInit;
+}
+
+#endif
+
+static bool gcInitialised = false;
+
+void initGC()
+{
+    if (gcInitialised)
+        return;
+
+#if HAVE_BOEHMGC
+    initGCReal();
+
+    gcCyclesAfterInit = GC_get_gc_no();
+#endif
+
+    // NIX_PATH must override the regular setting
+    // See the comment in applyConfig
+    if (auto nixPathEnv = getEnv("NIX_PATH")) {
+        globalConfig.set("nix-path", concatStringsSep(" ", EvalSettings::parseNixPath(nixPathEnv.value())));
+    }
+
+    gcInitialised = true;
+}
+
+void assertGCInitialized()
+{
+    assert(gcInitialised);
+}
+
+} // namespace nix

src/libexpr/eval-gc.hh

@@ -0,0 +1,25 @@
+#pragma once
+///@file
+
+#include <cstddef>
+
+namespace nix {
+
+/**
+ * Initialise the Boehm GC, if applicable.
+ */
+void initGC();
+
+/**
+ * Make sure `initGC` has already been called.
+ */
+void assertGCInitialized();
+
+#ifdef HAVE_BOEHMGC
+/**
+ * The number of GC cycles since initGC().
+ */
+size_t getGCCycles();
+#endif
+
+} // namespace nix

src/libexpr/eval-inline.hh

@@ -4,6 +4,7 @@
 #include "print.hh"
 #include "eval.hh"
 #include "eval-error.hh"
+#include "eval-settings.hh"
 
 namespace nix {
 
@@ -138,5 +139,12 @@ inline void EvalState::forceList(Value & v, const PosIdx pos, std::string_view e
     }
 }
 
+[[gnu::always_inline]]
+inline CallDepth EvalState::addCallDepth(const PosIdx pos) {
+    if (callDepth > settings.maxCallDepth)
+        error<EvalError>("stack overflow; max-call-depth exceeded").atPos(pos).debugThrow();
+
+    return CallDepth(callDepth);
+};
 
 }

src/libexpr/eval-settings.cc

@@ -8,7 +8,7 @@ namespace nix {
 
 /* Very hacky way to parse $NIX_PATH, which is colon-separated, but
    can contain URLs (e.g. "nixpkgs=https://bla...:foo=https://"). */
-static Strings parseNixPath(const std::string & s)
+Strings EvalSettings::parseNixPath(const std::string & s)
 {
     Strings res;
 
@@ -44,10 +44,13 @@ static Strings parseNixPath(const std::string & s)
     return res;
 }
 
-EvalSettings::EvalSettings()
+EvalSettings::EvalSettings(bool & readOnlyMode, EvalSettings::LookupPathHooks lookupPathHooks)
+    : readOnlyMode{readOnlyMode}
+    , lookupPathHooks{lookupPathHooks}
 {
-    auto var = getEnv("NIX_PATH");
-    if (var) nixPath = parseNixPath(*var);
+    auto var = getEnv("NIX_ABORT_ON_WARN");
+    if (var && (var == "1" || var == "yes" || var == "true"))
+        builtinsAbortOnWarn = true;
 }
 
 Strings EvalSettings::getDefaultNixPath()
@@ -63,11 +66,9 @@ Strings EvalSettings::getDefaultNixPath()
         }
     };
 
-    if (!evalSettings.restrictEval && !evalSettings.pureEval) {
-        add(getNixDefExpr() + "/channels");
-        add(rootChannelsDir() + "/nixpkgs", "nixpkgs");
-        add(rootChannelsDir());
-    }
+    add(getNixDefExpr() + "/channels");
+    add(rootChannelsDir() + "/nixpkgs", "nixpkgs");
+    add(rootChannelsDir());
 
     return res;
 }
@@ -89,20 +90,16 @@ std::string EvalSettings::resolvePseudoUrl(std::string_view url)
         return std::string(url);
 }
 
-const std::string & EvalSettings::getCurrentSystem()
+const std::string & EvalSettings::getCurrentSystem() const
 {
     const auto & evalSystem = currentSystem.get();
     return evalSystem != "" ? evalSystem : settings.thisSystem.get();
 }
 
-EvalSettings evalSettings;
-
-static GlobalConfig::Register rEvalSettings(&evalSettings);
-
 Path getNixDefExpr()
 {
     return settings.useXDGBaseDirectories
-        ? getStateDir() + "/nix/defexpr"
+        ? getStateDir() + "/defexpr"
         : getHome() + "/.nix-defexpr";
 }
 

src/libexpr/eval-settings.hh

@@ -2,49 +2,117 @@
 ///@file
 
 #include "config.hh"
+#include "ref.hh"
 
 namespace nix {
 
+class Store;
+
 struct EvalSettings : Config
 {
-    EvalSettings();
+    /**
+     * Function used to interpet look path entries of a given scheme.
+     *
+     * The argument is the non-scheme part of the lookup path entry (see
+     * `LookupPathHooks` below).
+     *
+     * The return value is (a) whether the entry was valid, and, if so,
+     * what does it map to.
+     *
+     * @todo Return (`std::optional` of) `SourceAccssor` or something
+     * more structured instead of mere `std::string`?
+     */
+    using LookupPathHook = std::optional<std::string>(ref<Store> store, std::string_view);
+
+    /**
+     * Map from "scheme" to a `LookupPathHook`.
+     *
+     * Given a lookup path value (i.e. either the whole thing, or after
+     * the `<key>=`) in the form of:
+     *
+     * ```
+     * <scheme>:<arbitrary string>
+     * ```
+     *
+     * if `<scheme>` is a key in this map, then `<arbitrary string>` is
+     * passed to the hook that is the value in this map.
+     */
+    using LookupPathHooks = std::map<std::string, std::function<LookupPathHook>>;
+
+    EvalSettings(bool & readOnlyMode, LookupPathHooks lookupPathHooks = {});
+
+    bool & readOnlyMode;
 
     static Strings getDefaultNixPath();
 
     static bool isPseudoUrl(std::string_view s);
 
+    static Strings parseNixPath(const std::string & s);
+
     static std::string resolvePseudoUrl(std::string_view url);
 
-    Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation",
-        "Whether builtin functions that allow executing native code should be enabled."};
+    LookupPathHooks lookupPathHooks;
+
+    Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation", R"(
+        Enable built-in functions that allow executing native code.
+
+        In particular, this adds:
+        - `builtins.importNative` *path* *symbol*
+
+          Opens dynamic shared object (DSO) at *path*, loads the function with the symbol name *symbol* from it and runs it.
+          The loaded function must have the following signature:
+          ```cpp
+          extern "C" typedef void (*ValueInitialiser) (EvalState & state, Value & v);
+          ```
+
+          The [Nix C++ API documentation](@docroot@/development/documentation.md#api-documentation) has more details on evaluator internals.
+
+        - `builtins.exec` *arguments*
+
+          Execute a program, where *arguments* are specified as a list of strings, and parse its output as a Nix expression.
+    )"};
 
     Setting<Strings> nixPath{
-        this, getDefaultNixPath(), "nix-path",
+        this, {}, "nix-path",
         R"(
           List of search paths to use for [lookup path](@docroot@/language/constructs/lookup-path.md) resolution.
           This setting determines the value of
-          [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath) and can be used with [`builtins.findFile`](@docroot@/language/builtin-constants.md#builtins-findFile).
+          [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath) and can be used with [`builtins.findFile`](@docroot@/language/builtins.md#builtins-findFile).
 
-          The default value is
+          - The configuration setting is overridden by the [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH)
+          environment variable.
+          - `NIX_PATH` is overridden by [specifying the setting as the command line flag](@docroot@/command-ref/conf-file.md#command-line-flags) `--nix-path`.
+          - Any current value is extended by the [`-I` option](@docroot@/command-ref/opt-common.md#opt-I) or `--extra-nix-path`.
 
-          ```
-          $HOME/.nix-defexpr/channels
-          nixpkgs=$NIX_STATE_DIR/profiles/per-user/root/channels/nixpkgs
-          $NIX_STATE_DIR/profiles/per-user/root/channels
-          ```
+          If the respective paths are accessible, the default values are:
 
-          It can be overridden with the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH) or the [`-I` command line option](@docroot@/command-ref/opt-common.md#opt-I).
+          - `$HOME/.nix-defexpr/channels`
+
+            The [user channel link](@docroot@/command-ref/files/default-nix-expression.md#user-channel-link), pointing to the current state of [channels](@docroot@/command-ref/files/channels.md) for the current user.
+
+          - `nixpkgs=$NIX_STATE_DIR/profiles/per-user/root/channels/nixpkgs`
+
+            The current state of the `nixpkgs` channel for the `root` user.
+
+          - `$NIX_STATE_DIR/profiles/per-user/root/channels`
+
+            The current state of all channels for the `root` user.
+
+          These files are set up by the [Nix installer](@docroot@/installation/installing-binary.md).
+          See [`NIX_STATE_DIR`](@docroot@/command-ref/env-common.md#env-NIX_STATE_DIR) for details on the environment variable.
 
           > **Note**
           >
-          > If [pure evaluation](#conf-pure-eval) is enabled, `nixPath` evaluates to the empty list `[ ]`.
+          > If [restricted evaluation](@docroot@/command-ref/conf-file.md#conf-restrict-eval) is enabled, the default value is empty.
+          >
+          > If [pure evaluation](#conf-pure-eval) is enabled, `builtins.nixPath` *always* evaluates to the empty list `[ ]`.
         )", {}, false};
 
     Setting<std::string> currentSystem{
         this, "", "eval-system",
         R"(
           This option defines
-          [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem)
+          [`builtins.currentSystem`](@docroot@/language/builtins.md#builtins-currentSystem)
           in the Nix language if it is set as a non-empty string.
           Otherwise, if it is defined as the empty string (the default), the value of the
           [`system` ](#conf-system)
@@ -58,14 +126,14 @@ struct EvalSettings : Config
      * Implements the `eval-system` vs `system` defaulting logic
      * described for `eval-system`.
      */
-    const std::string & getCurrentSystem();
+    const std::string & getCurrentSystem() const;
 
     Setting<bool> restrictEval{
         this, false, "restrict-eval",
         R"(
           If set to `true`, the Nix evaluator will not allow access to any
           files outside of
-          [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath),
+          [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath),
           or to URIs outside of
           [`allowed-uris`](@docroot@/command-ref/conf-file.md#conf-allowed-uris).
         )"};
@@ -76,10 +144,10 @@ struct EvalSettings : Config
 
           - Restrict file system and network access to files specified by cryptographic hash
           - Disable impure constants:
-            - [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem)
-            - [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime)
-            - [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath)
-            - [`builtins.storePath`](@docroot@/language/builtin-constants.md#builtins-storePath)
+            - [`builtins.currentSystem`](@docroot@/language/builtins.md#builtins-currentSystem)
+            - [`builtins.currentTime`](@docroot@/language/builtins.md#builtins-currentTime)
+            - [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath)
+            - [`builtins.storePath`](@docroot@/language/builtins.md#builtins-storePath)
         )"
         };
 
@@ -126,7 +194,11 @@ struct EvalSettings : Config
         )"};
 
     Setting<bool> useEvalCache{this, true, "eval-cache",
-        "Whether to use the flake evaluation cache."};
+        R"(
+            Whether to use the flake evaluation cache.
+            Certain commands won't have to evaluate when invoked for the second time with a particular version of a flake.
+            Intermediate results are not cached.
+        )"};
 
     Setting<bool> ignoreExceptionsDuringTry{this, false, "ignore-try",
         R"(
@@ -142,16 +214,40 @@ struct EvalSettings : Config
 
     Setting<bool> builtinsTraceDebugger{this, false, "debugger-on-trace",
         R"(
-          If set to true and the `--debugger` flag is given,
-          [`builtins.trace`](@docroot@/language/builtins.md#builtins-trace) will
-          enter the debugger like
-          [`builtins.break`](@docroot@/language/builtins.md#builtins-break).
+          If set to true and the `--debugger` flag is given, the following functions
+          will enter the debugger like [`builtins.break`](@docroot@/language/builtins.md#builtins-break).
+
+          * [`builtins.trace`](@docroot@/language/builtins.md#builtins-trace)
+          * [`builtins.traceVerbose`](@docroot@/language/builtins.md#builtins-traceVerbose)
+            if [`trace-verbose`](#conf-trace-verbose) is set to true.
+          * [`builtins.warn`](@docroot@/language/builtins.md#builtins-warn)
 
           This is useful for debugging warnings in third-party Nix code.
         )"};
-};
 
-extern EvalSettings evalSettings;
+    Setting<bool> builtinsDebuggerOnWarn{this, false, "debugger-on-warn",
+        R"(
+          If set to true and the `--debugger` flag is given, [`builtins.warn`](@docroot@/language/builtins.md#builtins-warn)
+          will enter the debugger like [`builtins.break`](@docroot@/language/builtins.md#builtins-break).
+
+          This is useful for debugging warnings in third-party Nix code.
+
+          Use [`debugger-on-trace`](#conf-debugger-on-trace) to also enter the debugger on legacy warnings that are logged with [`builtins.trace`](@docroot@/language/builtins.md#builtins-trace).
+        )"};
+
+    Setting<bool> builtinsAbortOnWarn{this, false, "abort-on-warn",
+        R"(
+          If set to true, [`builtins.warn`](@docroot@/language/builtins.md#builtins-warn) will throw an error when logging a warning.
+
+          This will give you a stack trace that leads to the location of the warning.
+
+          This is useful for finding information about warnings in third-party Nix code when you can not start the interactive debugger, such as when Nix is called from a non-interactive script. See [`debugger-on-warn`](#conf-debugger-on-warn).
+
+          Currently, a stack trace can only be produced when the debugger is enabled, or when evaluation is aborted.
+
+          This option can be enabled by setting `NIX_ABORT_ON_WARN=1` in the environment.
+        )"};
+};
 
 /**
  * Conventionally part of the default nix path in impure mode.

src/libexpr/eval.hh

@@ -9,14 +9,15 @@
 #include "symbol-table.hh"
 #include "config.hh"
 #include "experimental-features.hh"
-#include "input-accessor.hh"
+#include "position.hh"
+#include "pos-table.hh"
+#include "source-accessor.hh"
 #include "search-path.hh"
 #include "repl-exit-status.hh"
+#include "ref.hh"
 
 #include <map>
 #include <optional>
-#include <unordered_map>
-#include <mutex>
 #include <functional>
 
 namespace nix {
@@ -29,17 +30,36 @@ namespace nix {
 constexpr size_t maxPrimOpArity = 8;
 
 class Store;
+namespace fetchers { struct Settings; }
+struct EvalSettings;
 class EvalState;
 class StorePath;
 struct SingleDerivedPath;
 enum RepairFlag : bool;
-struct MemoryInputAccessor;
+struct MemorySourceAccessor;
+namespace eval_cache {
+    class EvalCache;
+}
 
+/**
+ * Increments a count on construction and decrements on destruction.
+ */
+class CallDepth {
+  size_t & count;
+
+public:
+  CallDepth(size_t & count) : count(count) {
+    ++count;
+  }
+  ~CallDepth() {
+    --count;
+  }
+};
 
 /**
  * Function that implements a primop.
  */
-typedef void (* PrimOpFun) (EvalState & state, const PosIdx pos, Value * * args, Value & v);
+using PrimOpFun = void(EvalState & state, const PosIdx pos, Value * * args, Value & v);
 
 /**
  * Info about a primitive operation, and its implementation
@@ -80,7 +100,7 @@ struct PrimOp
     /**
      * Implementation of the primop.
      */
-    std::function<std::remove_pointer<PrimOpFun>::type> fun;
+    std::function<PrimOpFun> fun;
 
     /**
      * Optional experimental for this to be gated on.
@@ -125,6 +145,8 @@ struct Constant
     typedef std::map<std::string, Value *> ValMap;
 #endif
 
+typedef std::unordered_map<PosIdx, DocComment> DocCommentMap;
+
 struct Env
 {
     Env * up;
@@ -143,12 +165,6 @@ std::string printValue(EvalState & state, Value & v);
 std::ostream & operator << (std::ostream & os, const ValueType t);
 
 
-/**
- * Initialise the Boehm GC, if applicable.
- */
-void initGC();
-
-
 struct RegexCache;
 
 std::shared_ptr<RegexCache> makeRegexCache();
@@ -164,13 +180,18 @@ struct DebugTrace {
 class EvalState : public std::enable_shared_from_this<EvalState>
 {
 public:
+    const fetchers::Settings & fetchSettings;
+    const EvalSettings & settings;
     SymbolTable symbols;
     PosTable positions;
 
     const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue,
         sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls,
         sFile, sLine, sColumn, sFunctor, sToString,
-        sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
+        sRight, sWrong, sStructuredAttrs,
+        sAllowedReferences, sAllowedRequisites, sDisallowedReferences, sDisallowedRequisites,
+        sMaxSize, sMaxClosureSize,
+        sBuilder, sArgs,
         sContentAddressed, sImpure,
         sOutputHash, sOutputHashAlgo, sOutputHashMode,
         sRecurseForDerivations,
@@ -226,18 +247,18 @@ public:
     /**
      * The accessor for the root filesystem.
      */
-    const ref<InputAccessor> rootFS;
+    const ref<SourceAccessor> rootFS;
 
     /**
      * The in-memory filesystem for <nix/...> paths.
      */
-    const ref<MemoryInputAccessor> corepkgsFS;
+    const ref<MemorySourceAccessor> corepkgsFS;
 
     /**
      * In-memory filesystem for internal, non-user-callable Nix
      * expressions like call-flake.nix.
      */
-    const ref<MemoryInputAccessor> internalFS;
+    const ref<MemorySourceAccessor> internalFS;
 
     const SourcePath derivationInternal;
 
@@ -273,6 +294,18 @@ public:
             return std::shared_ptr<const StaticEnv>();;
     }
 
+    /** Whether a debug repl can be started. If `false`, `runDebugRepl(error)` will return without starting a repl. */
+    bool canDebug();
+
+    /** Use front of `debugTraces`; see `runDebugRepl(error,env,expr)` */
+    void runDebugRepl(const Error * error);
+
+    /**
+     * Run a debug repl with the given error, environment and expression.
+     * @param error The error to debug, may be nullptr.
+     * @param env The environment to debug, matching the expression.
+     * @param expr The expression to debug, matching the environment.
+     */
     void runDebugRepl(const Error * error, const Env & env, const Expr & expr);
 
     template<class T, typename... Args>
@@ -282,19 +315,24 @@ public:
         return *new EvalErrorBuilder<T>(*this, args...);
     }
 
+    /**
+     * A cache for evaluation caches, so as to reuse the same root value if possible
+     */
+    std::map<const Hash, ref<eval_cache::EvalCache>> evalCaches;
+
 private:
 
     /* Cache for calls to addToStore(); maps source paths to the store
        paths. */
-    std::map<SourcePath, StorePath> srcToStore;
+    Sync<std::unordered_map<SourcePath, StorePath>> srcToStore;
 
     /**
      * A cache from path names to parse trees.
      */
 #if HAVE_BOEHMGC
-    typedef std::map<SourcePath, Expr *, std::less<SourcePath>, traceable_allocator<std::pair<const SourcePath, Expr *>>> FileParseCache;
+    typedef std::unordered_map<SourcePath, Expr *, std::hash<SourcePath>, std::equal_to<SourcePath>, traceable_allocator<std::pair<const SourcePath, Expr *>>> FileParseCache;
 #else
-    typedef std::map<SourcePath, Expr *> FileParseCache;
+    typedef std::unordered_map<SourcePath, Expr *> FileParseCache;
 #endif
     FileParseCache fileParseCache;
 
@@ -302,12 +340,18 @@ private:
      * A cache from path names to values.
      */
 #if HAVE_BOEHMGC
-    typedef std::map<SourcePath, Value, std::less<SourcePath>, traceable_allocator<std::pair<const SourcePath, Value>>> FileEvalCache;
+    typedef std::unordered_map<SourcePath, Value, std::hash<SourcePath>, std::equal_to<SourcePath>, traceable_allocator<std::pair<const SourcePath, Value>>> FileEvalCache;
 #else
-    typedef std::map<SourcePath, Value> FileEvalCache;
+    typedef std::unordered_map<SourcePath, Value> FileEvalCache;
 #endif
     FileEvalCache fileEvalCache;
 
+    /**
+     * Associate source positions of certain AST nodes with their preceding doc comment, if they have one.
+     * Grouped by file.
+     */
+    std::unordered_map<SourcePath, DocCommentMap> positionToDocComment;
+
     LookupPath lookupPath;
 
     std::map<std::string, std::optional<std::string>> lookupPathResolved;
@@ -334,6 +378,8 @@ public:
     EvalState(
         const LookupPath & _lookupPath,
         ref<Store> store,
+        const fetchers::Settings & fetchSettings,
+        const EvalSettings & settings,
         std::shared_ptr<Store> buildStore = nullptr);
     ~EvalState();
 
@@ -539,6 +585,11 @@ public:
      */
     SingleDerivedPath coerceToSingleDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx);
 
+#if HAVE_BOEHMGC
+    /** A GC root for the baseEnv reference. */
+    std::shared_ptr<Env *> baseEnvP;
+#endif
+
 public:
 
     /**
@@ -589,6 +640,12 @@ public:
         const char * doc;
     };
 
+    /**
+     * Retrieve the documentation for a value. This will evaluate the value if
+     * it is a thunk, and it will partially apply __functor if applicable.
+     *
+     * @param v The value to get the documentation for.
+     */
     std::optional<Doc> getDoc(Value & v);
 
 private:
@@ -613,12 +670,26 @@ private:
 
 public:
 
+    /**
+     * Check that the call depth is within limits, and increment it, until the returned object is destroyed.
+     */
+    inline CallDepth addCallDepth(const PosIdx pos);
+
     /**
      * Do a deep equality test between two values.  That is, list
      * elements and attributes are compared recursively.
      */
     bool eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx);
 
+    /**
+     * Like `eqValues`, but throws an `AssertionError` if not equal.
+     *
+     * WARNING:
+     * Callers should call `eqValues` first and report if `assertEqValues` behaves
+     * incorrectly. (e.g. if it doesn't throw if eqValues returns false or vice versa)
+     */
+    void assertEqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx);
+
     bool isFunctor(Value & fun);
 
     // FIXME: use std::span
@@ -743,6 +814,8 @@ public:
         std::string_view pathArg,
         PosIdx pos);
 
+    DocComment getDocCommentForPos(PosIdx pos);
+
 private:
 
     /**
@@ -825,8 +898,10 @@ std::string showType(const Value & v);
 
 /**
  * If `path` refers to a directory, then append "/default.nix".
+ *
+ * @param addDefaultNix Whether to append "/default.nix" after resolving symlinks.
  */
-SourcePath resolveExprPath(SourcePath path);
+SourcePath resolveExprPath(SourcePath path, bool addDefaultNix = true);
 
 /**
  * Whether a URI is allowed, assuming restrictEval is enabled

src/libexpr/fetchurl.nix

@@ -28,11 +28,8 @@ derivation ({
   # No need to double the amount of network traffic
   preferLocalBuild = true;
 
+  # This attribute does nothing; it's here to avoid changing evaluation results.
   impureEnvVars = [
-    # We borrow these environment variables from the caller to allow
-    # easy proxy configuration.  This is impure, but a fixed-output
-    # derivation like fetchurl is allowed to do so since its result is
-    # by definition pure.
     "http_proxy" "https_proxy" "ftp_proxy" "all_proxy" "no_proxy"
   ];
 

src/libexpr/get-drvs.cc

@@ -69,10 +69,17 @@ std::string PackageInfo::querySystem() const
 std::optional<StorePath> PackageInfo::queryDrvPath() const
 {
     if (!drvPath && attrs) {
-        NixStringContext context;
-        if (auto i = attrs->get(state->sDrvPath))
-            drvPath = {state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the 'drvPath' attribute of a derivation")};
-        else
+        if (auto i = attrs->get(state->sDrvPath)) {
+            NixStringContext context;
+            auto found = state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the 'drvPath' attribute of a derivation");
+            try {
+                found.requireDerivation();
+            } catch (Error & e) {
+                e.addTrace(state->positions[i->pos], "while evaluating the 'drvPath' attribute of a derivation");
+                throw;
+            }
+            drvPath = {std::move(found)};
+        } else
             drvPath = {std::nullopt};
     }
     return drvPath.value_or(std::nullopt);
@@ -239,8 +246,8 @@ NixInt PackageInfo::queryMetaInt(const std::string & name, NixInt def)
     if (v->type() == nString) {
         /* Backwards compatibility with before we had support for
            integer meta fields. */
-        if (auto n = string2Int<NixInt>(v->c_str()))
-            return *n;
+        if (auto n = string2Int<NixInt::Inner>(v->c_str()))
+            return NixInt{*n};
     }
     return def;
 }
@@ -335,9 +342,9 @@ std::optional<PackageInfo> getDerivation(EvalState & state, Value & v,
 }
 
 
-static std::string addToPath(const std::string & s1, const std::string & s2)
+static std::string addToPath(const std::string & s1, std::string_view s2)
 {
-    return s1.empty() ? s2 : s1 + "." + s2;
+    return s1.empty() ? std::string(s2) : s1 + "." + s2;
 }
 
 
@@ -367,21 +374,26 @@ static void getDerivations(EvalState & state, Value & vIn,
            bound to the attribute with the "lower" name should take
            precedence). */
         for (auto & i : v.attrs()->lexicographicOrder(state.symbols)) {
-            debug("evaluating attribute '%1%'", state.symbols[i->name]);
-            if (!std::regex_match(std::string(state.symbols[i->name]), attrRegex))
-                continue;
-            std::string pathPrefix2 = addToPath(pathPrefix, state.symbols[i->name]);
-            if (combineChannels)
-                getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
-            else if (getDerivation(state, *i->value, pathPrefix2, drvs, done, ignoreAssertionFailures)) {
-                /* If the value of this attribute is itself a set,
-                   should we recurse into it?  => Only if it has a
-                   `recurseForDerivations = true' attribute. */
-                if (i->value->type() == nAttrs) {
-                    auto j = i->value->attrs()->get(state.sRecurseForDerivations);
-                    if (j && state.forceBool(*j->value, j->pos, "while evaluating the attribute `recurseForDerivations`"))
-                        getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
+            try {
+                debug("evaluating attribute '%1%'", state.symbols[i->name]);
+                if (!std::regex_match(std::string(state.symbols[i->name]), attrRegex))
+                    continue;
+                std::string pathPrefix2 = addToPath(pathPrefix, state.symbols[i->name]);
+                if (combineChannels)
+                    getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
+                else if (getDerivation(state, *i->value, pathPrefix2, drvs, done, ignoreAssertionFailures)) {
+                    /* If the value of this attribute is itself a set,
+                    should we recurse into it?  => Only if it has a
+                    `recurseForDerivations = true' attribute. */
+                    if (i->value->type() == nAttrs) {
+                        auto j = i->value->attrs()->get(state.sRecurseForDerivations);
+                        if (j && state.forceBool(*j->value, j->pos, "while evaluating the attribute `recurseForDerivations`"))
+                            getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
+                    }
                 }
+            } catch (Error & e) {
+                e.addTrace(state.positions[i->pos], "while evaluating the attribute '%s'", state.symbols[i->name]);
+                throw;
             }
         }
     }

src/libexpr/json-to-value.cc

@@ -2,6 +2,7 @@
 #include "value.hh"
 #include "eval.hh"
 
+#include <limits>
 #include <variant>
 #include <nlohmann/json.hpp>
 
@@ -42,7 +43,7 @@ class JSONSax : nlohmann::json_sax<json> {
             auto attrs2 = state.buildBindings(attrs.size());
             for (auto & i : attrs)
                 attrs2.insert(i.first, i.second);
-            parent->value(state).mkAttrs(attrs2.alreadySorted());
+            parent->value(state).mkAttrs(attrs2);
             return std::move(parent);
         }
         void add() override { v = nullptr; }
@@ -80,42 +81,46 @@ class JSONSax : nlohmann::json_sax<json> {
 public:
     JSONSax(EvalState & state, Value & v) : state(state), rs(new JSONState(&v)) {};
 
-    bool null()
+    bool null() override
     {
         rs->value(state).mkNull();
         rs->add();
         return true;
     }
 
-    bool boolean(bool val)
+    bool boolean(bool val) override
     {
         rs->value(state).mkBool(val);
         rs->add();
         return true;
     }
 
-    bool number_integer(number_integer_t val)
+    bool number_integer(number_integer_t val) override
     {
         rs->value(state).mkInt(val);
         rs->add();
         return true;
     }
 
-    bool number_unsigned(number_unsigned_t val)
+    bool number_unsigned(number_unsigned_t val_) override
     {
+        if (val_ > std::numeric_limits<NixInt::Inner>::max()) {
+            throw Error("unsigned json number %1% outside of Nix integer range", val_);
+        }
+        NixInt::Inner val = val_;
         rs->value(state).mkInt(val);
         rs->add();
         return true;
     }
 
-    bool number_float(number_float_t val, const string_t & s)
+    bool number_float(number_float_t val, const string_t & s) override
     {
         rs->value(state).mkFloat(val);
         rs->add();
         return true;
     }
 
-    bool string(string_t & val)
+    bool string(string_t & val) override
     {
         rs->value(state).mkString(val);
         rs->add();
@@ -123,7 +128,7 @@ public:
     }
 
 #if NLOHMANN_JSON_VERSION_MAJOR >= 3 && NLOHMANN_JSON_VERSION_MINOR >= 8
-    bool binary(binary_t&)
+    bool binary(binary_t&) override
     {
         // This function ought to be unreachable
         assert(false);
@@ -131,35 +136,35 @@ public:
     }
 #endif
 
-    bool start_object(std::size_t len)
+    bool start_object(std::size_t len) override
     {
         rs = std::make_unique<JSONObjectState>(std::move(rs));
         return true;
     }
 
-    bool key(string_t & name)
+    bool key(string_t & name) override
     {
         dynamic_cast<JSONObjectState*>(rs.get())->key(name, state);
         return true;
     }
 
-    bool end_object() {
+    bool end_object() override {
         rs = rs->resolve(state);
         rs->add();
         return true;
     }
 
-    bool end_array() {
+    bool end_array() override {
         return end_object();
     }
 
-    bool start_array(size_t len) {
+    bool start_array(size_t len) override {
         rs = std::make_unique<JSONListState>(std::move(rs),
             len != std::numeric_limits<size_t>::max() ? len : 128);
         return true;
     }
 
-    bool parse_error(std::size_t, const std::string&, const nlohmann::detail::exception& ex) {
+    bool parse_error(std::size_t, const std::string&, const nlohmann::detail::exception& ex) override {
         throw JSONParseError("%s", ex.what());
     }
 };

src/libexpr/lexer-helpers.cc

@@ -0,0 +1,30 @@
+#include "lexer-tab.hh"
+#include "lexer-helpers.hh"
+#include "parser-tab.hh"
+
+void nix::lexer::internal::initLoc(YYLTYPE * loc)
+{
+    loc->beginOffset = loc->endOffset = 0;
+}
+
+void nix::lexer::internal::adjustLoc(yyscan_t yyscanner, YYLTYPE * loc, const char * s, size_t len)
+{
+    loc->stash();
+
+    LexerState & lexerState = *yyget_extra(yyscanner);
+
+    if (lexerState.docCommentDistance == 1) {
+        // Preceding token was a doc comment.
+        ParserLocation doc;
+        doc.beginOffset = lexerState.lastDocCommentLoc.beginOffset;
+        ParserLocation docEnd;
+        docEnd.beginOffset = lexerState.lastDocCommentLoc.endOffset;
+        DocComment docComment{lexerState.at(doc), lexerState.at(docEnd)};
+        PosIdx locPos = lexerState.at(*loc);
+        lexerState.positionToDocComment.emplace(locPos, docComment);
+    }
+    lexerState.docCommentDistance++;
+
+    loc->beginOffset = loc->endOffset;
+    loc->endOffset += len;
+}

src/libexpr/lexer-helpers.hh

@@ -0,0 +1,9 @@
+#pragma once
+
+namespace nix::lexer::internal {
+
+void initLoc(YYLTYPE * loc);
+
+void adjustLoc(yyscan_t yyscanner, YYLTYPE * loc, const char * s, size_t len);
+
+} // namespace nix::lexer

src/libexpr/lexer.l

@@ -5,7 +5,7 @@
 %option stack
 %option nodefault
 %option nounput noyy_top_state
-
+%option extra-type="::nix::LexerState *"
 
 %s DEFAULT
 %x STRING
@@ -14,38 +14,31 @@
 %x INPATH_SLASH
 %x PATH_START
 
+%top {
+#include "parser-tab.hh" // YYSTYPE
+#include "parser-state.hh"
+}
 
 %{
 #ifdef __clang__
 #pragma clang diagnostic ignored "-Wunneeded-internal-declaration"
 #endif
 
-#include <boost/lexical_cast.hpp>
-
 #include "nixexpr.hh"
 #include "parser-tab.hh"
+#include "lexer-helpers.hh"
+
+namespace nix {
+  struct LexerState;
+}
 
 using namespace nix;
+using namespace nix::lexer::internal;
 
 namespace nix {
 
 #define CUR_POS state->at(*yylloc)
 
-static void initLoc(YYLTYPE * loc)
-{
-    loc->first_line = loc->last_line = 0;
-    loc->first_column = loc->last_column = 0;
-}
-
-static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
-{
-    loc->stash();
-
-    loc->first_column = loc->last_column;
-    loc->last_column += len;
-}
-
-
 // we make use of the fact that the parser receives a private copy of the input
 // string and can munge around in it.
 static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length)
@@ -74,6 +67,14 @@ static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length)
     return {result, size_t(t - result)};
 }
 
+static void requireExperimentalFeature(const ExperimentalFeature & feature, const Pos & pos)
+{
+    if (!experimentalFeatureSettings.isEnabled(feature))
+        throw ParseError(ErrorInfo{
+            .msg = HintFmt("experimental Nix feature '%1%' is disabled; add '--extra-experimental-features %1%' to enable it", showExperimentalFeature(feature)),
+            .pos = pos,
+        });
+}
 
 }
 
@@ -81,7 +82,7 @@ static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length)
 #pragma GCC diagnostic ignored "-Wimplicit-fallthrough"
 
 #define YY_USER_INIT initLoc(yylloc)
-#define YY_USER_ACTION adjustLoc(yylloc, yytext, yyleng);
+#define YY_USER_ACTION adjustLoc(yyscanner, yylloc, yytext, yyleng);
 
 #define PUSH_STATE(state) yy_push_state(state, yyscanner)
 #define POP_STATE() yy_pop_state(yyscanner)
@@ -126,12 +127,19 @@ or          { return OR_KW; }
 \-\>        { return IMPL; }
 \/\/        { return UPDATE; }
 \+\+        { return CONCAT; }
+\<\|        { requireExperimentalFeature(Xp::PipeOperators, state->positions[CUR_POS]);
+              return PIPE_FROM;
+            }
+\|\>        { requireExperimentalFeature(Xp::PipeOperators, state->positions[CUR_POS]);
+              return PIPE_INTO;
+            }
 
 {ID}        { yylval->id = {yytext, (size_t) yyleng}; return ID; }
 {INT}       { errno = 0;
-              try {
-                  yylval->n = boost::lexical_cast<int64_t>(yytext);
-              } catch (const boost::bad_lexical_cast &) {
+              std::optional<int64_t> numMay = string2Int<int64_t>(yytext);
+              if (numMay.has_value()) {
+                  yylval->n = NixInt{*numMay};
+              } else {
                   throw ParseError(ErrorInfo{
                       .msg = HintFmt("invalid integer '%1%'", yytext),
                       .pos = state->positions[CUR_POS],
@@ -280,9 +288,32 @@ or          { return OR_KW; }
 {SPATH}     { yylval->path = {yytext, (size_t) yyleng}; return SPATH; }
 {URI}       { yylval->uri = {yytext, (size_t) yyleng}; return URI; }
 
-[ \t\r\n]+    /* eat up whitespace */
-\#[^\r\n]*    /* single-line comments */
-\/\*([^*]|\*+[^*/])*\*+\/  /* long comments */
+%{
+// Doc comment rule
+//
+//   \/\*\*                            /**
+//         [^/*]                       reject /**/ (empty comment) and /***
+//              ([^*]|\*+[^*/])*\*+\/  same as the long comment rule
+//              (             )*       zero or more non-ending sequences
+//                              \*     end(1)
+//                                 \/  end(2)
+%}
+\/\*\*[^/*]([^*]|\*+[^*/])*\*+\/  /* doc comments */ {
+    LexerState & lexerState = *yyget_extra(yyscanner);
+    lexerState.docCommentDistance = 0;
+    lexerState.lastDocCommentLoc.beginOffset = yylloc->beginOffset;
+    lexerState.lastDocCommentLoc.endOffset = yylloc->endOffset;
+}
+
+
+%{
+// The following rules have docCommentDistance--
+// This compensates for the docCommentDistance++ which happens by default to
+// make all the other rules invalidate the doc comment.
+%}
+[ \t\r\n]+    /* eat up whitespace */ { yyget_extra(yyscanner)->docCommentDistance--; }
+\#[^\r\n]*    /* single-line comments */ { yyget_extra(yyscanner)->docCommentDistance--; }
+\/\*([^*]|\*+[^*/])*\*+\/  /* long comments */ { yyget_extra(yyscanner)->docCommentDistance--; }
 
 {ANY}       {
               /* Don't return a negative number, as this will cause

src/libexpr/local.mk

@@ -8,14 +8,15 @@ libexpr_SOURCES := \
   $(wildcard $(d)/*.cc) \
   $(wildcard $(d)/value/*.cc) \
   $(wildcard $(d)/primops/*.cc) \
-  $(wildcard $(d)/flake/*.cc) \
   $(d)/lexer-tab.cc \
   $(d)/parser-tab.cc
 # Not just for this library itself, but also for downstream libraries using this library
 
 INCLUDE_libexpr := -I $(d)
 
-libexpr_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libmain) $(INCLUDE_libexpr)
+libexpr_CXXFLAGS += \
+	$(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) \
+	-DGC_THREADS
 
 libexpr_LIBS = libutil libstore libfetchers
 
@@ -43,11 +44,7 @@ $(eval $(call install-file-in, $(buildprefix)$(d)/nix-expr.pc, $(libdir)/pkgconf
 
 $(foreach i, $(wildcard src/libexpr/value/*.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix/value, 0644)))
-$(foreach i, $(wildcard src/libexpr/flake/*.hh), \
-  $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644)))
 
 $(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh
 
-$(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh $(d)/flake/call-flake.nix.gen.hh
-
-$(buildprefix)src/libexpr/primops/fromTOML.o:	ERROR_SWITCH_ENUM =
+$(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh $(d)/call-flake.nix.gen.hh

src/libexpr/meson.build

@@ -0,0 +1,210 @@
+project('nix-expr', 'cpp',
+  version : files('.version'),
+  default_options : [
+    'cpp_std=c++2a',
+    # TODO(Qyriad): increase the warning level
+    'warning_level=1',
+    'debug=true',
+    'optimization=2',
+    'errorlogs=true', # Please print logs for tests that fail
+  ],
+  meson_version : '>= 1.1',
+  license : 'LGPL-2.1-or-later',
+)
+
+cxx = meson.get_compiler('cpp')
+
+subdir('build-utils-meson/deps-lists')
+
+configdata = configuration_data()
+
+deps_private_maybe_subproject = [
+]
+deps_public_maybe_subproject = [
+  dependency('nix-util'),
+  dependency('nix-store'),
+  dependency('nix-fetchers'),
+]
+subdir('build-utils-meson/subprojects')
+
+subdir('build-utils-meson/threads')
+
+boost = dependency(
+  'boost',
+  modules : ['container', 'context'],
+  include_type: 'system',
+)
+# boost is a public dependency, but not a pkg-config dependency unfortunately, so we
+# put in `deps_other`.
+deps_other += boost
+
+nlohmann_json = dependency('nlohmann_json', version : '>= 3.9')
+deps_public += nlohmann_json
+
+bdw_gc = dependency('bdw-gc', required : get_option('gc'))
+if bdw_gc.found()
+  deps_public += bdw_gc
+  foreach funcspec : [
+    'pthread_attr_get_np',
+    'pthread_getattr_np',
+  ]
+    define_name = 'HAVE_' + funcspec.underscorify().to_upper()
+    define_value = cxx.has_function(funcspec).to_int()
+    configdata.set(define_name, define_value)
+  endforeach
+  configdata.set('GC_THREADS', 1)
+endif
+configdata.set('HAVE_BOEHMGC', bdw_gc.found().to_int())
+
+toml11 = dependency(
+  'toml11',
+  version : '>=3.7.0',
+  method : 'cmake',
+  include_type: 'system',
+)
+deps_other += toml11
+
+config_h = configure_file(
+  configuration : configdata,
+  output : 'config-expr.hh',
+)
+
+add_project_arguments(
+  # TODO(Qyriad): Yes this is how the autoconf+Make system did it.
+  # It would be nice for our headers to be idempotent instead.
+  '-include', 'config-util.hh',
+  '-include', 'config-store.hh',
+  # '-include', 'config-fetchers.h',
+  '-include', 'config-expr.hh',
+  language : 'cpp',
+)
+
+subdir('build-utils-meson/diagnostics')
+
+parser_tab = custom_target(
+  input : 'parser.y',
+  output : [
+    'parser-tab.cc',
+    'parser-tab.hh',
+  ],
+  command : [
+    'bison',
+    '-v',
+    '-o',
+    '@OUTPUT0@',
+    '@INPUT@',
+    '-d',
+  ],
+  # NOTE(Qyriad): Meson doesn't support installing only part of a custom target, so we add
+  # an install script below which removes parser-tab.cc.
+  install : true,
+  install_dir : get_option('includedir') / 'nix',
+)
+
+lexer_tab = custom_target(
+  input : [
+    'lexer.l',
+    parser_tab,
+  ],
+  output : [
+    'lexer-tab.cc',
+    'lexer-tab.hh',
+  ],
+  command : [
+    'flex',
+    '--outfile',
+    '@OUTPUT0@',
+    '--header-file=' + '@OUTPUT1@',
+    '@INPUT0@',
+  ],
+  # NOTE(Qyriad): Meson doesn't support installing only part of a custom target, so we add
+  # an install script below which removes lexer-tab.cc.
+  install : true,
+  install_dir : get_option('includedir') / 'nix',
+)
+
+subdir('build-utils-meson/generate-header')
+
+generated_headers = []
+foreach header : [
+  'imported-drv-to-derivation.nix',
+  'fetchurl.nix',
+  'call-flake.nix',
+]
+  generated_headers += gen_header.process(header)
+endforeach
+
+sources = files(
+  'attr-path.cc',
+  'attr-set.cc',
+  'eval-cache.cc',
+  'eval-error.cc',
+  'eval-gc.cc',
+  'eval-settings.cc',
+  'eval.cc',
+  'function-trace.cc',
+  'get-drvs.cc',
+  'json-to-value.cc',
+  'lexer-helpers.cc',
+  'nixexpr.cc',
+  'paths.cc',
+  'primops.cc',
+  'print-ambiguous.cc',
+  'print.cc',
+  'search-path.cc',
+  'value-to-json.cc',
+  'value-to-xml.cc',
+  'value/context.cc',
+)
+
+include_dirs = [include_directories('.')]
+
+headers = [config_h] + files(
+  'attr-path.hh',
+  'attr-set.hh',
+  'eval-cache.hh',
+  'eval-error.hh',
+  'eval-gc.hh',
+  'eval-inline.hh',
+  'eval-settings.hh',
+  'eval.hh',
+  'function-trace.hh',
+  'gc-small-vector.hh',
+  'get-drvs.hh',
+  'json-to-value.hh',
+  # internal: 'lexer-helpers.hh',
+  'nixexpr.hh',
+  'parser-state.hh',
+  'pos-idx.hh',
+  'pos-table.hh',
+  'primops.hh',
+  'print-ambiguous.hh',
+  'print-options.hh',
+  'print.hh',
+  'repl-exit-status.hh',
+  'search-path.hh',
+  'symbol-table.hh',
+  'value-to-json.hh',
+  'value-to-xml.hh',
+  'value.hh',
+  'value/context.hh',
+)
+
+subdir('primops')
+
+this_library = library(
+  'nixexpr',
+  sources,
+  parser_tab,
+  lexer_tab,
+  generated_headers,
+  dependencies : deps_public + deps_private + deps_other,
+  prelink : true, # For C++ static initializers
+  install : true,
+)
+
+install_headers(headers, subdir : 'nix', preserve_path : true)
+
+libraries_private = []
+
+subdir('build-utils-meson/export')

src/libexpr/meson.options

@@ -0,0 +1,3 @@
+option('gc', type : 'feature',
+  description : 'enable garbage collection in the Nix expression evaluator (requires Boehm GC)',
+)

src/libexpr/nixexpr.cc

@@ -1,11 +1,13 @@
 #include "nixexpr.hh"
-#include "derivations.hh"
 #include "eval.hh"
 #include "symbol-table.hh"
 #include "util.hh"
 #include "print.hh"
 
 #include <cstdlib>
+#include <sstream>
+
+#include "strings-inline.hh"
 
 namespace nix {
 
@@ -23,7 +25,7 @@ std::ostream & operator <<(std::ostream & str, const SymbolStr & symbol)
 
 void Expr::show(const SymbolTable & symbols, std::ostream & str) const
 {
-    abort();
+    unreachable();
 }
 
 void ExprInt::show(const SymbolTable & symbols, std::ostream & str) const
@@ -80,7 +82,9 @@ void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) co
         return sa < sb;
     });
     std::vector<Symbol> inherits;
-    std::map<ExprInheritFrom *, std::vector<Symbol>> inheritsFrom;
+    // We can use the displacement as a proxy for the order in which the symbols were parsed.
+    // The assignment of displacements should be deterministic, so that showBindings is deterministic.
+    std::map<Displacement, std::vector<Symbol>> inheritsFrom;
     for (auto & i : sorted) {
         switch (i->second.kind) {
         case AttrDef::Kind::Plain:
@@ -91,7 +95,7 @@ void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) co
         case AttrDef::Kind::InheritedFrom: {
             auto & select = dynamic_cast<ExprSelect &>(*i->second.e);
             auto & from = dynamic_cast<ExprInheritFrom &>(*select.e);
-            inheritsFrom[&from].push_back(i->first);
+            inheritsFrom[from.displ].push_back(i->first);
             break;
         }
         }
@@ -103,7 +107,7 @@ void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) co
     }
     for (const auto & [from, syms] : inheritsFrom) {
         str << "inherit (";
-        (*inheritFromExprs)[from->displ]->show(symbols, str);
+        (*inheritFromExprs)[from]->show(symbols, str);
         str << ")";
         for (auto sym : syms) str << " " << symbols[sym];
         str << "; ";
@@ -267,7 +271,7 @@ std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath)
 
 void Expr::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
 {
-    abort();
+    unreachable();
 }
 
 void ExprInt::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
@@ -581,6 +585,22 @@ std::string ExprLambda::showNamePos(const EvalState & state) const
     return fmt("%1% at %2%", id, state.positions[pos]);
 }
 
+void ExprLambda::setDocComment(DocComment docComment) {
+    // RFC 145 specifies that the innermost doc comment wins.
+    // See https://github.com/NixOS/rfcs/blob/master/rfcs/0145-doc-strings.md#ambiguous-placement
+    if (!this->docComment) {
+        this->docComment = docComment;
+
+        // Curried functions are defined by putting a function directly
+        // in the body of another function. To render docs for those, we
+        // need to propagate the doc comment to the innermost function.
+        //
+        // If we have our own comment, we've already propagated it, so this
+        // belongs in the same conditional.
+        body->setDocComment(docComment);
+    }
+};
+
 
 
 /* Position table. */
@@ -625,4 +645,22 @@ size_t SymbolTable::totalSize() const
     return n;
 }
 
+std::string DocComment::getInnerText(const PosTable & positions) const {
+    auto beginPos = positions[begin];
+    auto endPos = positions[end];
+    auto docCommentStr = beginPos.getSnippetUpTo(endPos).value_or("");
+
+    // Strip "/**" and "*/"
+    constexpr size_t prefixLen = 3;
+    constexpr size_t suffixLen = 2;
+    std::string docStr = docCommentStr.substr(prefixLen, docCommentStr.size() - prefixLen - suffixLen);
+    if (docStr.empty())
+        return {};
+    // Turn the now missing "/**" into indentation
+    docStr = "   " + docStr;
+    // Strip indentation (for the whole, potentially multi-line string)
+    docStr = stripIndentation(docStr);
+    return docStr;
+}
+
 }

src/libexpr/nixexpr.hh

@@ -6,21 +6,58 @@
 
 #include "value.hh"
 #include "symbol-table.hh"
-#include "error.hh"
-#include "position.hh"
 #include "eval-error.hh"
 #include "pos-idx.hh"
-#include "pos-table.hh"
 
 namespace nix {
 
-
-struct Env;
-struct Value;
 class EvalState;
+class PosTable;
+struct Env;
 struct ExprWith;
 struct StaticEnv;
+struct Value;
 
+/**
+ * A documentation comment, in the sense of [RFC 145](https://github.com/NixOS/rfcs/blob/master/rfcs/0145-doc-strings.md)
+ *
+ * Note that this does not implement the following:
+ *  - argument attribute names ("formals"): TBD
+ *  - argument names: these are internal to the function and their names may not be optimal for documentation
+ *  - function arity (degree of currying or number of ':'s):
+ *      - Functions returning partially applied functions have a higher arity
+ *        than can be determined locally and without evaluation.
+ *        We do not want to present false data.
+ *      - Some functions should be thought of as transformations of other
+ *        functions. For instance `overlay -> overlay -> overlay` is the simplest
+ *        way to understand `composeExtensions`, but its implementation looks like
+ *        `f: g: final: prev: <...>`. The parameters `final` and `prev` are part
+ *        of the overlay concept, while distracting from the function's purpose.
+ */
+struct DocComment {
+
+    /**
+     * Start of the comment, including the opening, ie `/` and `**`.
+     */
+    PosIdx begin;
+
+    /**
+     * Position right after the final asterisk and `/` that terminate the comment.
+     */
+    PosIdx end;
+
+    /**
+     * Whether the comment is set.
+     *
+     * A `DocComment` is small enough that it makes sense to pass by value, and
+     * therefore baking optionality into it is also useful, to avoiding the memory
+     * overhead of `std::optional`.
+     */
+    operator bool() const { return static_cast<bool>(begin); }
+
+    std::string getInnerText(const PosTable & positions) const;
+
+};
 
 /**
  * An attribute path is a sequence of attribute names.
@@ -57,6 +94,7 @@ struct Expr
     virtual void eval(EvalState & state, Env & env, Value & v);
     virtual Value * maybeThunk(EvalState & state, Env & env);
     virtual void setName(Symbol name);
+    virtual void setDocComment(DocComment docComment) { };
     virtual PosIdx getPos() const { return noPos; }
 };
 
@@ -69,6 +107,7 @@ struct ExprInt : Expr
 {
     Value v;
     ExprInt(NixInt n) { v.mkInt(n); };
+    ExprInt(NixInt::Inner n) { v.mkInt(n); };
     Value * maybeThunk(EvalState & state, Env & env) override;
     COMMON_METHODS
 };
@@ -92,10 +131,10 @@ struct ExprString : Expr
 
 struct ExprPath : Expr
 {
-    ref<InputAccessor> accessor;
+    ref<SourceAccessor> accessor;
     std::string s;
     Value v;
-    ExprPath(ref<InputAccessor> accessor, std::string s) : accessor(accessor), s(std::move(s))
+    ExprPath(ref<SourceAccessor> accessor, std::string s) : accessor(accessor), s(std::move(s))
     {
         v.mkPath(&*accessor, this->s.c_str());
     }
@@ -148,7 +187,7 @@ struct ExprInheritFrom : ExprVar
         this->fromWith = nullptr;
     }
 
-    void bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env);
+    void bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env) override;
 };
 
 struct ExprSelect : Expr
@@ -159,6 +198,17 @@ struct ExprSelect : Expr
     ExprSelect(const PosIdx & pos, Expr * e, AttrPath attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(std::move(attrPath)) { };
     ExprSelect(const PosIdx & pos, Expr * e, Symbol name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); };
     PosIdx getPos() const override { return pos; }
+
+    /**
+     * Evaluate the `a.b.c` part of `a.b.c.d`. This exists mostly for the purpose of :doc in the repl.
+     *
+     * @param[out] v The attribute set that should contain the last attribute name (if it exists).
+     * @return The last attribute name in `attrPath`
+     *
+     * @note This does *not* evaluate the final attribute, and does not fail if that's the only attribute that does not exist.
+     */
+    Symbol evalExceptFinalSelect(EvalState & state, Env & env, Value & attrs);
+
     COMMON_METHODS
 };
 
@@ -281,6 +331,8 @@ struct ExprLambda : Expr
     Symbol arg;
     Formals * formals;
     Expr * body;
+    DocComment docComment;
+
     ExprLambda(PosIdx pos, Symbol arg, Formals * formals, Expr * body)
         : pos(pos), arg(arg), formals(formals), body(body)
     {
@@ -293,6 +345,7 @@ struct ExprLambda : Expr
     std::string showNamePos(const EvalState & state) const;
     inline bool hasFormals() const { return formals != nullptr; }
     PosIdx getPos() const override { return pos; }
+    virtual void setDocComment(DocComment docComment) override;
     COMMON_METHODS
 };
 

src/libexpr/package.nix

@@ -0,0 +1,113 @@
+{ lib
+, stdenv
+, mkMesonDerivation
+, releaseTools
+
+, meson
+, ninja
+, pkg-config
+, bison
+, flex
+, cmake # for resolving toml11 dep
+
+, nix-util
+, nix-store
+, nix-fetchers
+, boost
+, boehmgc
+, nlohmann_json
+, toml11
+
+# Configuration Options
+
+, version
+
+# Whether to use garbage collection for the Nix language evaluator.
+#
+# If it is disabled, we just leak memory, but this is not as bad as it
+# sounds so long as evaluation just takes places within short-lived
+# processes. (When the process exits, the memory is reclaimed; it is
+# only leaked *within* the process.)
+#
+# Temporarily disabled on Windows because the `GC_throw_bad_alloc`
+# symbol is missing during linking.
+, enableGC ? !stdenv.hostPlatform.isWindows
+}:
+
+let
+  inherit (lib) fileset;
+in
+
+mkMesonDerivation (finalAttrs: {
+  pname = "nix-expr";
+  inherit version;
+
+  workDir = ./.;
+  fileset = fileset.unions [
+    ../../build-utils-meson
+    ./build-utils-meson
+    ../../.version
+    ./.version
+    ./meson.build
+    ./meson.options
+    ./primops/meson.build
+    (fileset.fileFilter (file: file.hasExt "cc") ./.)
+    (fileset.fileFilter (file: file.hasExt "hh") ./.)
+    ./lexer.l
+    ./parser.y
+    (fileset.fileFilter (file: file.hasExt "nix") ./.)
+  ];
+
+  outputs = [ "out" "dev" ];
+
+  nativeBuildInputs = [
+    meson
+    ninja
+    pkg-config
+    bison
+    flex
+    cmake
+  ];
+
+  buildInputs = [
+    toml11
+  ];
+
+  propagatedBuildInputs = [
+    nix-util
+    nix-store
+    nix-fetchers
+    boost
+    nlohmann_json
+  ] ++ lib.optional enableGC boehmgc;
+
+  preConfigure =
+    # "Inline" .version so it's not a symlink, and includes the suffix.
+    # Do the meson utils, without modification.
+    ''
+      chmod u+w ./.version
+      echo ${version} > ../../.version
+    '';
+
+  mesonFlags = [
+    (lib.mesonEnable "gc" enableGC)
+  ];
+
+  env = {
+    # Needed for Meson to find Boost.
+    # https://github.com/NixOS/nixpkgs/issues/86131.
+    BOOST_INCLUDEDIR = "${lib.getDev boost}/include";
+    BOOST_LIBRARYDIR = "${lib.getLib boost}/lib";
+  } // lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) {
+    LDFLAGS = "-fuse-ld=gold";
+  };
+
+  separateDebugInfo = !stdenv.hostPlatform.isStatic;
+
+  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
+
+  meta = {
+    platforms = lib.platforms.unix ++ lib.platforms.windows;
+  };
+
+})

src/libexpr/parser-state.hh

@@ -1,6 +1,8 @@
 #pragma once
 ///@file
 
+#include <limits>
+
 #include "eval.hh"
 
 namespace nix {
@@ -18,38 +20,74 @@ struct StringToken
     operator std::string_view() const { return {p, l}; }
 };
 
+// This type must be trivially copyable; see YYLTYPE_IS_TRIVIAL in parser.y.
 struct ParserLocation
 {
-    int first_line, first_column;
-    int last_line, last_column;
+    int beginOffset;
+    int endOffset;
 
     // backup to recover from yyless(0)
-    int stashed_first_column, stashed_last_column;
+    int stashedBeginOffset, stashedEndOffset;
 
     void stash() {
-        stashed_first_column = first_column;
-        stashed_last_column = last_column;
+        stashedBeginOffset = beginOffset;
+        stashedEndOffset = endOffset;
     }
 
     void unstash() {
-        first_column = stashed_first_column;
-        last_column = stashed_last_column;
+        beginOffset = stashedBeginOffset;
+        endOffset = stashedEndOffset;
     }
+
+    /** Latest doc comment position, or 0. */
+    int doc_comment_first_column, doc_comment_last_column;
+};
+
+struct LexerState
+{
+    /**
+     * Tracks the distance to the last doc comment, in terms of lexer tokens.
+     *
+     * The lexer sets this to 0 when reading a doc comment, and increments it
+     * for every matched rule; see `lexer-helpers.cc`.
+     * Whitespace and comment rules decrement the distance, so that they result
+     * in a net 0 change in distance.
+     */
+    int docCommentDistance = std::numeric_limits<int>::max();
+
+    /**
+     * The location of the last doc comment.
+     *
+     * (stashing fields are not used)
+     */
+    ParserLocation lastDocCommentLoc;
+
+    /**
+     * @brief Maps some positions to a DocComment, where the comment is relevant to the location.
+     */
+    std::unordered_map<PosIdx, DocComment> & positionToDocComment;
+
+    PosTable & positions;
+    PosTable::Origin origin;
+
+    PosIdx at(const ParserLocation & loc);
 };
 
 struct ParserState
 {
+    const LexerState & lexerState;
     SymbolTable & symbols;
     PosTable & positions;
     Expr * result;
     SourcePath basePath;
     PosTable::Origin origin;
-    const ref<InputAccessor> rootFS;
+    const ref<SourceAccessor> rootFS;
     const Expr::AstSymbols & s;
+    const EvalSettings & settings;
 
     void dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos);
     void dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos);
-    void addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * e, const PosIdx pos);
+    void addAttr(ExprAttrs * attrs, AttrPath && attrPath, const ParserLocation & loc, Expr * e, const ParserLocation & exprLoc);
     Formals * validateFormals(Formals * formals, PosIdx pos = noPos, Symbol arg = {});
     Expr * stripIndentation(const PosIdx pos,
         std::vector<std::pair<PosIdx, std::variant<Expr *, StringToken>>> && es);
@@ -73,11 +111,12 @@ inline void ParserState::dupAttr(Symbol attr, const PosIdx pos, const PosIdx pre
     });
 }
 
-inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * e, const PosIdx pos)
+inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, const ParserLocation & loc, Expr * e, const ParserLocation & exprLoc)
 {
     AttrPath::iterator i;
     // All attrpaths have at least one attr
     assert(!attrPath.empty());
+    auto pos = at(loc);
     // Checking attrPath validity.
     // ===========================
     for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) {
@@ -142,6 +181,12 @@ inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr *
     } else {
         attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, e, pos));
     }
+
+    auto it = lexerState.positionToDocComment.find(pos);
+    if (it != lexerState.positionToDocComment.end()) {
+        e->setDocComment(it->second);
+        lexerState.positionToDocComment.emplace(at(exprLoc), it->second);
+    }
 }
 
 inline Formals * ParserState::validateFormals(Formals * formals, PosIdx pos, Symbol arg)
@@ -254,12 +299,23 @@ inline Expr * ParserState::stripIndentation(const PosIdx pos,
                 s2 = std::string(s2, 0, p + 1);
         }
 
-        es2->emplace_back(i->first, new ExprString(std::move(s2)));
+        // Ignore empty strings for a minor optimisation and AST simplification
+        if (s2 != "") {
+            es2->emplace_back(i->first, new ExprString(std::move(s2)));
+        }
     };
     for (; i != es.end(); ++i, --n) {
         std::visit(overloaded { trimExpr, trimString }, i->second);
     }
 
+    // If there is nothing at all, return the empty string directly.
+    // This also ensures that equivalent empty strings result in the same ast, which is helpful when testing formatters.
+    if (es2->size() == 0) {
+        auto *const result = new ExprString("");
+        delete es2;
+        return result;
+    }
+
     /* If this is a single string, then don't do a concatenation. */
     if (es2->size() == 1 && dynamic_cast<ExprString *>((*es2)[0].second)) {
         auto *const result = (*es2)[0].second;
@@ -269,9 +325,14 @@ inline Expr * ParserState::stripIndentation(const PosIdx pos,
     return new ExprConcatStrings(pos, true, es2);
 }
 
+inline PosIdx LexerState::at(const ParserLocation & loc)
+{
+    return positions.add(origin, loc.beginOffset);
+}
+
 inline PosIdx ParserState::at(const ParserLocation & loc)
 {
-    return positions.add(origin, loc.first_column);
+    return positions.add(origin, loc.beginOffset);
 }
 
 }

src/libexpr/parser.y

@@ -1,4 +1,4 @@
-%glr-parser
+%define api.location.type { ::nix::ParserLocation }
 %define api.pure
 %locations
 %define parse.error verbose
@@ -8,8 +8,7 @@
 %parse-param { nix::ParserState * state }
 %lex-param { void * scanner }
 %lex-param { nix::ParserState * state }
-%expect 1
-%expect-rr 1
+%expect 0
 
 %code requires {
 
@@ -25,23 +24,51 @@
 #include "nixexpr.hh"
 #include "eval.hh"
 #include "eval-settings.hh"
-#include "globals.hh"
 #include "parser-state.hh"
 
-#define YYLTYPE ::nix::ParserLocation
+// Bison seems to have difficulty growing the parser stack when using C++ with
+// a custom location type. This undocumented macro tells Bison that our
+// location type is "trivially copyable" in C++-ese, so it is safe to use the
+// same memcpy macro it uses to grow the stack that it uses with its own
+// default location type. Without this, we get "error: memory exhausted" when
+// parsing some large Nix files. Our other options are to increase the initial
+// stack size (200 by default) to be as large as we ever want to support (so
+// that growing the stack is unnecessary), or redefine the stack-relocation
+// macro ourselves (which is also undocumented).
+#define YYLTYPE_IS_TRIVIAL 1
+
 #define YY_DECL int yylex \
     (YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParserState * state)
 
+// For efficiency, we only track offsets; not line,column coordinates
+# define YYLLOC_DEFAULT(Current, Rhs, N)                                \
+    do                                                                  \
+      if (N)                                                            \
+        {                                                               \
+          (Current).beginOffset = YYRHSLOC (Rhs, 1).beginOffset;        \
+          (Current).endOffset  = YYRHSLOC (Rhs, N).endOffset;           \
+        }                                                               \
+      else                                                              \
+        {                                                               \
+          (Current).beginOffset = (Current).endOffset =                 \
+            YYRHSLOC (Rhs, 0).endOffset;                                \
+        }                                                               \
+    while (0)
+
 namespace nix {
 
+typedef std::unordered_map<PosIdx, DocComment> DocCommentMap;
+
 Expr * parseExprFromBuf(
     char * text,
     size_t length,
     Pos::Origin origin,
     const SourcePath & basePath,
     SymbolTable & symbols,
+    const EvalSettings & settings,
     PosTable & positions,
-    const ref<InputAccessor> rootFS,
+    DocCommentMap & docComments,
+    const ref<SourceAccessor> rootFS,
     const Expr::AstSymbols & astSymbols);
 
 }
@@ -59,14 +86,13 @@ YY_DECL;
 
 using namespace nix;
 
-#define CUR_POS state->at(*yylocp)
+#define CUR_POS state->at(yyloc)
 
 
 void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * error)
 {
     if (std::string_view(error).starts_with("syntax error, unexpected end of file")) {
-        loc->first_column = loc->last_column;
-        loc->first_line = loc->last_line;
+        loc->beginOffset = loc->endOffset;
     }
     throw ParseError({
         .msg = HintFmt(error),
@@ -74,6 +100,22 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char *
     });
 }
 
+#define SET_DOC_POS(lambda, pos) setDocPosition(state->lexerState, lambda, state->at(pos))
+static void setDocPosition(const LexerState & lexerState, ExprLambda * lambda, PosIdx start) {
+    auto it = lexerState.positionToDocComment.find(start);
+    if (it != lexerState.positionToDocComment.end()) {
+        lambda->setDocComment(it->second);
+    }
+}
+
+static Expr * makeCall(PosIdx pos, Expr * fn, Expr * arg) {
+    if (auto e2 = dynamic_cast<ExprCall *>(fn)) {
+        e2->args.push_back(arg);
+        return fn;
+    }
+    return new ExprCall(pos, fn, {arg});
+}
+
 
 %}
 
@@ -98,9 +140,10 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char *
 
 %type <e> start expr expr_function expr_if expr_op
 %type <e> expr_select expr_simple expr_app
+%type <e> expr_pipe_from expr_pipe_into
 %type <list> expr_list
-%type <attrs> binds
-%type <formals> formals
+%type <attrs> binds binds1
+%type <formals> formals formal_set
 %type <formal> formal
 %type <attrNames> attrpath
 %type <inheritAttrs> attrs
@@ -115,10 +158,12 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char *
 %token <path> PATH HPATH SPATH PATH_END
 %token <uri> URI
 %token IF THEN ELSE ASSERT WITH LET IN_KW REC INHERIT EQ NEQ AND OR IMPL OR_KW
+%token PIPE_FROM PIPE_INTO /* <| and |> */
 %token DOLLAR_CURLY /* == ${ */
 %token IND_STRING_OPEN IND_STRING_CLOSE
 %token ELLIPSIS
 
+
 %right IMPL
 %left OR
 %left AND
@@ -140,18 +185,28 @@ expr: expr_function;
 
 expr_function
   : ID ':' expr_function
-    { $$ = new ExprLambda(CUR_POS, state->symbols.create($1), 0, $3); }
-  | '{' formals '}' ':' expr_function
-    { $$ = new ExprLambda(CUR_POS, state->validateFormals($2), $5); }
-  | '{' formals '}' '@' ID ':' expr_function
-    {
-      auto arg = state->symbols.create($5);
-      $$ = new ExprLambda(CUR_POS, arg, state->validateFormals($2, CUR_POS, arg), $7);
+    { auto me = new ExprLambda(CUR_POS, state->symbols.create($1), 0, $3);
+      $$ = me;
+      SET_DOC_POS(me, @1);
     }
-  | ID '@' '{' formals '}' ':' expr_function
+  | formal_set ':' expr_function[body]
+    { auto me = new ExprLambda(CUR_POS, state->validateFormals($formal_set), $body);
+      $$ = me;
+      SET_DOC_POS(me, @1);
+    }
+  | formal_set '@' ID ':' expr_function[body]
     {
-      auto arg = state->symbols.create($1);
-      $$ = new ExprLambda(CUR_POS, arg, state->validateFormals($4, CUR_POS, arg), $7);
+      auto arg = state->symbols.create($ID);
+      auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($formal_set, CUR_POS, arg), $body);
+      $$ = me;
+      SET_DOC_POS(me, @1);
+    }
+  | ID '@' formal_set ':' expr_function[body]
+    {
+      auto arg = state->symbols.create($ID);
+      auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($formal_set, CUR_POS, arg), $body);
+      $$ = me;
+      SET_DOC_POS(me, @1);
     }
   | ASSERT expr ';' expr_function
     { $$ = new ExprAssert(CUR_POS, $2, $4); }
@@ -170,9 +225,21 @@ expr_function
 
 expr_if
   : IF expr THEN expr ELSE expr { $$ = new ExprIf(CUR_POS, $2, $4, $6); }
+  | expr_pipe_from
+  | expr_pipe_into
   | expr_op
   ;
 
+expr_pipe_from
+  : expr_op PIPE_FROM expr_pipe_from { $$ = makeCall(state->at(@2), $1, $3); }
+  | expr_op PIPE_FROM expr_op        { $$ = makeCall(state->at(@2), $1, $3); }
+  ;
+
+expr_pipe_into
+  : expr_pipe_into PIPE_INTO expr_op { $$ = makeCall(state->at(@2), $3, $1); }
+  | expr_op        PIPE_INTO expr_op { $$ = makeCall(state->at(@2), $3, $1); }
+  ;
+
 expr_op
   : '!' expr_op %prec NOT { $$ = new ExprOpNot($2); }
   | '-' expr_op %prec NEGATE { $$ = new ExprCall(CUR_POS, new ExprVar(state->s.sub), {new ExprInt(0), $2}); }
@@ -197,13 +264,7 @@ expr_op
   ;
 
 expr_app
-  : expr_app expr_select {
-      if (auto e2 = dynamic_cast<ExprCall *>($1)) {
-          e2->args.push_back($2);
-          $$ = $1;
-      } else
-          $$ = new ExprCall(CUR_POS, $1, {$2});
-  }
+  : expr_app expr_select { $$ = makeCall(CUR_POS, $1, $2); }
   | expr_select
   ;
 
@@ -259,11 +320,13 @@ expr_simple
   /* Let expressions `let {..., body = ...}' are just desugared
      into `(rec {..., body = ...}).body'. */
   | LET '{' binds '}'
-    { $3->recursive = true; $$ = new ExprSelect(noPos, $3, state->s.body); }
+    { $3->recursive = true; $3->pos = CUR_POS; $$ = new ExprSelect(noPos, $3, state->s.body); }
   | REC '{' binds '}'
-    { $3->recursive = true; $$ = $3; }
-  | '{' binds '}'
-    { $$ = $2; }
+    { $3->recursive = true; $3->pos = CUR_POS; $$ = $3; }
+  | '{' binds1 '}'
+    { $2->pos = CUR_POS; $$ = $2; }
+  | '{' '}'
+    { $$ = new ExprAttrs(CUR_POS); }
   | '[' expr_list ']' { $$ = $2; }
   ;
 
@@ -287,21 +350,21 @@ string_parts_interpolated
 
 path_start
   : PATH {
-    Path path(absPath({$1.p, $1.l}, state->basePath.path.abs()));
+    Path path(absPath(std::string_view{$1.p, $1.l}, state->basePath.path.abs()));
     /* add back in the trailing '/' to the first segment */
     if ($1.p[$1.l-1] == '/' && $1.l > 1)
       path += "/";
-    $$ = new ExprPath(ref<InputAccessor>(state->rootFS), std::move(path));
+    $$ = new ExprPath(ref<SourceAccessor>(state->rootFS), std::move(path));
   }
   | HPATH {
-    if (evalSettings.pureEval) {
+    if (state->settings.pureEval) {
         throw Error(
             "the path '%s' can not be resolved in pure mode",
             std::string_view($1.p, $1.l)
         );
     }
     Path path(getHome() + std::string($1.p + 1, $1.l - 1));
-    $$ = new ExprPath(ref<InputAccessor>(state->rootFS), std::move(path));
+    $$ = new ExprPath(ref<SourceAccessor>(state->rootFS), std::move(path));
   }
   ;
 
@@ -312,37 +375,50 @@ ind_string_parts
   ;
 
 binds
-  : binds attrpath '=' expr ';' { $$ = $1; state->addAttr($$, std::move(*$2), $4, state->at(@2)); delete $2; }
-  | binds INHERIT attrs ';'
-    { $$ = $1;
-      for (auto & [i, iPos] : *$3) {
-          if ($$->attrs.find(i.symbol) != $$->attrs.end())
-              state->dupAttr(i.symbol, iPos, $$->attrs[i.symbol].pos);
-          $$->attrs.emplace(
+  : binds1
+  | { $$ = new ExprAttrs; }
+  ;
+
+binds1
+  : binds1[accum] attrpath '=' expr ';'
+    { $$ = $accum;
+      state->addAttr($$, std::move(*$attrpath), @attrpath, $expr, @expr);
+      delete $attrpath;
+    }
+  | binds[accum] INHERIT attrs ';'
+    { $$ = $accum;
+      for (auto & [i, iPos] : *$attrs) {
+          if ($accum->attrs.find(i.symbol) != $accum->attrs.end())
+              state->dupAttr(i.symbol, iPos, $accum->attrs[i.symbol].pos);
+          $accum->attrs.emplace(
               i.symbol,
               ExprAttrs::AttrDef(new ExprVar(iPos, i.symbol), iPos, ExprAttrs::AttrDef::Kind::Inherited));
       }
-      delete $3;
+      delete $attrs;
     }
-  | binds INHERIT '(' expr ')' attrs ';'
-    { $$ = $1;
-      if (!$$->inheritFromExprs)
-          $$->inheritFromExprs = std::make_unique<std::vector<Expr *>>();
-      $$->inheritFromExprs->push_back($4);
-      auto from = new nix::ExprInheritFrom(state->at(@4), $$->inheritFromExprs->size() - 1);
-      for (auto & [i, iPos] : *$6) {
-          if ($$->attrs.find(i.symbol) != $$->attrs.end())
-              state->dupAttr(i.symbol, iPos, $$->attrs[i.symbol].pos);
-          $$->attrs.emplace(
+  | binds[accum] INHERIT '(' expr ')' attrs ';'
+    { $$ = $accum;
+      if (!$accum->inheritFromExprs)
+          $accum->inheritFromExprs = std::make_unique<std::vector<Expr *>>();
+      $accum->inheritFromExprs->push_back($expr);
+      auto from = new nix::ExprInheritFrom(state->at(@expr), $accum->inheritFromExprs->size() - 1);
+      for (auto & [i, iPos] : *$attrs) {
+          if ($accum->attrs.find(i.symbol) != $accum->attrs.end())
+              state->dupAttr(i.symbol, iPos, $accum->attrs[i.symbol].pos);
+          $accum->attrs.emplace(
               i.symbol,
               ExprAttrs::AttrDef(
                   new ExprSelect(iPos, from, i.symbol),
                   iPos,
                   ExprAttrs::AttrDef::Kind::InheritedFrom));
       }
-      delete $6;
+      delete $attrs;
+    }
+  | attrpath '=' expr ';'
+    { $$ = new ExprAttrs;
+      state->addAttr($$, std::move(*$attrpath), @attrpath, $expr, @expr);
+      delete $attrpath;
     }
-  | { $$ = new ExprAttrs(state->at(@0)); }
   ;
 
 attrs
@@ -400,15 +476,19 @@ expr_list
   | { $$ = new ExprList; }
   ;
 
+formal_set
+  : '{' formals ',' ELLIPSIS '}' { $$ = $formals;    $$->ellipsis = true; }
+  | '{' ELLIPSIS '}'             { $$ = new Formals; $$->ellipsis = true; }
+  | '{' formals ',' '}'          { $$ = $formals;    $$->ellipsis = false; }
+  | '{' formals '}'              { $$ = $formals;    $$->ellipsis = false; }
+  | '{' '}'                      { $$ = new Formals; $$->ellipsis = false; }
+  ;
+
 formals
-  : formal ',' formals
-    { $$ = $3; $$->formals.emplace_back(*$1); delete $1; }
+  : formals[accum] ',' formal
+    { $$ = $accum; $$->formals.emplace_back(*$formal); delete $formal; }
   | formal
-    { $$ = new Formals; $$->formals.emplace_back(*$1); $$->ellipsis = false; delete $1; }
-  |
-    { $$ = new Formals; $$->ellipsis = false; }
-  | ELLIPSIS
-    { $$ = new Formals; $$->ellipsis = true; }
+    { $$ = new Formals; $$->formals.emplace_back(*$formal); delete $formal; }
   ;
 
 formal
@@ -429,21 +509,30 @@ Expr * parseExprFromBuf(
     Pos::Origin origin,
     const SourcePath & basePath,
     SymbolTable & symbols,
+    const EvalSettings & settings,
     PosTable & positions,
-    const ref<InputAccessor> rootFS,
+    DocCommentMap & docComments,
+    const ref<SourceAccessor> rootFS,
     const Expr::AstSymbols & astSymbols)
 {
     yyscan_t scanner;
+    LexerState lexerState {
+        .positionToDocComment = docComments,
+        .positions = positions,
+        .origin = positions.addOrigin(origin, length),
+    };
     ParserState state {
+        .lexerState = lexerState,
         .symbols = symbols,
         .positions = positions,
         .basePath = basePath,
-        .origin = positions.addOrigin(origin, length),
+        .origin = lexerState.origin,
         .rootFS = rootFS,
         .s = astSymbols,
+        .settings = settings,
     };
 
-    yylex_init(&scanner);
+    yylex_init_extra(&lexerState, &scanner);
     Finally _destroy([&] { yylex_destroy(scanner); });
 
     yy_scan_buffer(text, length, scanner);

src/libexpr/pos-idx.hh

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <cinttypes>
+#include <functional>
 
 namespace nix {
 
@@ -8,6 +9,7 @@ class PosIdx
 {
     friend struct LazyPosAcessors;
     friend class PosTable;
+    friend class std::hash<PosIdx>;
 
 private:
     uint32_t id;
@@ -28,9 +30,9 @@ public:
         return id > 0;
     }
 
-    bool operator<(const PosIdx other) const
+    auto operator<=>(const PosIdx other) const
     {
-        return id < other.id;
+        return id <=> other.id;
     }
 
     bool operator==(const PosIdx other) const
@@ -38,12 +40,25 @@ public:
         return id == other.id;
     }
 
-    bool operator!=(const PosIdx other) const
+    size_t hash() const noexcept
     {
-        return id != other.id;
+        return std::hash<uint32_t>{}(id);
     }
 };
 
 inline PosIdx noPos = {};
 
 }
+
+namespace std {
+
+template<>
+struct hash<nix::PosIdx>
+{
+    std::size_t operator()(nix::PosIdx pos) const noexcept
+    {
+        return pos.hash();
+    }
+};
+
+} // namespace std

src/libexpr/pos-table.hh

@@ -1,10 +1,8 @@
 #pragma once
 
-#include <cinttypes>
-#include <numeric>
+#include <cstdint>
 #include <vector>
 
-#include "chunked-vector.hh"
 #include "pos-idx.hh"
 #include "position.hh"
 #include "sync.hh"

src/libexpr/primops.cc

@@ -1,11 +1,9 @@
-#include "archive.hh"
 #include "derivations.hh"
 #include "downstream-placeholder.hh"
 #include "eval-inline.hh"
 #include "eval.hh"
 #include "eval-settings.hh"
 #include "gc-small-vector.hh"
-#include "globals.hh"
 #include "json-to-value.hh"
 #include "names.hh"
 #include "path-references.hh"
@@ -15,7 +13,6 @@
 #include "value-to-json.hh"
 #include "value-to-xml.hh"
 #include "primops.hh"
-#include "fs-input-accessor.hh"
 #include "fetch-to-store.hh"
 
 #include <boost/container/small_vector.hpp>
@@ -27,6 +24,7 @@
 
 #include <algorithm>
 #include <cstring>
+#include <sstream>
 #include <regex>
 
 #ifndef _WIN32
@@ -78,8 +76,8 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS
 
     if (drvs.empty()) return {};
 
-    if (isIFD && !evalSettings.enableImportFromDerivation)
-        error<EvalError>(
+    if (isIFD && !settings.enableImportFromDerivation)
+        error<EvalBaseError>(
             "cannot build '%1%' during evaluation because the option 'allow-import-from-derivation' is disabled",
             drvs.begin()->to_string(*store)
         ).debugThrow();
@@ -428,7 +426,7 @@ static void prim_typeOf(EvalState & state, const PosIdx pos, Value * * args, Val
             t = args[0]->external()->typeOf();
             break;
         case nFloat: t = "float"; break;
-        case nThunk: abort();
+        case nThunk: unreachable();
     }
     v.mkString(t);
 }
@@ -589,9 +587,9 @@ struct CompareValues
     {
         try {
             if (v1->type() == nFloat && v2->type() == nInt)
-                return v1->fpoint() < v2->integer();
+                return v1->fpoint() < v2->integer().value;
             if (v1->type() == nInt && v2->type() == nFloat)
-                return v1->integer() < v2->fpoint();
+                return v1->integer().value < v2->fpoint();
             if (v1->type() != v2->type())
                 state.error<EvalError>("cannot compare %s with %s", showType(*v1), showType(*v2)).debugThrow();
             // Allow selecting a subset of enum values
@@ -721,7 +719,7 @@ static RegisterPrimOp primop_genericClosure(PrimOp {
     .doc = R"(
       `builtins.genericClosure` iteratively computes the transitive closure over an arbitrary relation defined by a function.
 
-      It takes *attrset* with two attributes named `startSet` and `operator`, and returns a list of attrbute sets:
+      It takes *attrset* with two attributes named `startSet` and `operator`, and returns a list of attribute sets:
 
       - `startSet`:
         The initial list of attribute sets.
@@ -733,11 +731,12 @@ static RegisterPrimOp primop_genericClosure(PrimOp {
       Each attribute set in the list `startSet` and the list returned by `operator` must have an attribute `key`, which must support equality comparison.
       The value of `key` can be one of the following types:
 
-      - [Number](@docroot@/language/values.md#type-number)
-      - [Boolean](@docroot@/language/values.md#type-boolean)
-      - [String](@docroot@/language/values.md#type-string)
-      - [Path](@docroot@/language/values.md#type-path)
-      - [List](@docroot@/language/values.md#list)
+      - [Int](@docroot@/language/types.md#type-int)
+      - [Float](@docroot@/language/types.md#type-float)
+      - [Boolean](@docroot@/language/types.md#type-boolean)
+      - [String](@docroot@/language/types.md#type-string)
+      - [Path](@docroot@/language/types.md#type-path)
+      - [List](@docroot@/language/types.md#list)
 
       The result is produced by calling the `operator` on each `item` that has not been called yet, including newly added items, until no new items are added.
       Items are compared by their `key` attribute.
@@ -780,15 +779,14 @@ static RegisterPrimOp primop_break({
     )",
     .fun = [](EvalState & state, const PosIdx pos, Value * * args, Value & v)
     {
-        if (state.debugRepl && !state.debugTraces.empty()) {
+        if (state.canDebug()) {
             auto error = Error(ErrorInfo {
                 .level = lvlInfo,
                 .msg = HintFmt("breakpoint reached"),
                 .pos = state.positions[pos],
             });
 
-            auto & dt = state.debugTraces.front();
-            state.runDebugRepl(&error, dt.env, dt.expr);
+            state.runDebugRepl(&error);
         }
 
         // Return the value we were passed.
@@ -807,7 +805,7 @@ static RegisterPrimOp primop_abort({
         NixStringContext context;
         auto s = state.coerceToString(pos, *args[0], context,
                 "while evaluating the error message passed to builtins.abort").toOwned();
-        state.error<Abort>("evaluation aborted with the following error message: '%1%'", s).debugThrow();
+        state.error<Abort>("evaluation aborted with the following error message: '%1%'", s).setIsFromExpr().debugThrow();
     }
 });
 
@@ -826,7 +824,7 @@ static RegisterPrimOp primop_throw({
       NixStringContext context;
       auto s = state.coerceToString(pos, *args[0], context,
               "while evaluating the error message passed to builtin.throw").toOwned();
-      state.error<ThrownError>(s).debugThrow();
+      state.error<ThrownError>(s).setIsFromExpr().debugThrow();
     }
 });
 
@@ -902,7 +900,7 @@ static void prim_tryEval(EvalState & state, const PosIdx pos, Value * * args, Va
     MaintainCount trylevel(state.trylevel);
 
     ReplExitStatus (* savedDebugRepl)(ref<EvalState> es, const ValMap & extraEnv) = nullptr;
-    if (state.debugRepl && evalSettings.ignoreExceptionsDuringTry)
+    if (state.debugRepl && state.settings.ignoreExceptionsDuringTry)
     {
         /* to prevent starting the repl from exceptions withing a tryEval, null it. */
         savedDebugRepl = state.debugRepl;
@@ -951,7 +949,7 @@ static RegisterPrimOp primop_tryEval({
 static void prim_getEnv(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
     std::string name(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.getEnv"));
-    v.mkString(evalSettings.restrictEval || evalSettings.pureEval ? "" : getEnv(name).value_or(""));
+    v.mkString(state.settings.restrictEval || state.settings.pureEval ? "" : getEnv(name).value_or(""));
 }
 
 static RegisterPrimOp primop_getEnv({
@@ -1018,9 +1016,8 @@ static void prim_trace(EvalState & state, const PosIdx pos, Value * * args, Valu
         printError("trace: %1%", args[0]->string_view());
     else
         printError("trace: %1%", ValuePrinter(state, *args[0]));
-    if (evalSettings.builtinsTraceDebugger && state.debugRepl && !state.debugTraces.empty()) {
-        const DebugTrace & last = state.debugTraces.front();
-        state.runDebugRepl(nullptr, last.env, last.expr);
+    if (state.settings.builtinsTraceDebugger) {
+        state.runDebugRepl(nullptr);
     }
     state.forceValue(*args[1], pos);
     v = *args[1];
@@ -1043,6 +1040,55 @@ static RegisterPrimOp primop_trace({
     .fun = prim_trace,
 });
 
+static void prim_warn(EvalState & state, const PosIdx pos, Value * * args, Value & v)
+{
+    // We only accept a string argument for now. The use case for pretty printing a value is covered by `trace`.
+    // By rejecting non-strings we allow future versions to add more features without breaking existing code.
+    auto msgStr = state.forceString(*args[0], pos, "while evaluating the first argument; the message passed to builtins.warn");
+
+    {
+        BaseError msg(std::string{msgStr});
+        msg.atPos(state.positions[pos]);
+        auto info = msg.info();
+        info.level = lvlWarn;
+        info.isFromExpr = true;
+        logWarning(info);
+    }
+
+    if (state.settings.builtinsAbortOnWarn) {
+        // Not an EvalError or subclass, which would cause the error to be stored in the eval cache.
+        state.error<EvalBaseError>("aborting to reveal stack trace of warning, as abort-on-warn is set").setIsFromExpr().debugThrow();
+    }
+    if (state.settings.builtinsTraceDebugger || state.settings.builtinsDebuggerOnWarn) {
+        state.runDebugRepl(nullptr);
+    }
+    state.forceValue(*args[1], pos);
+    v = *args[1];
+}
+
+static RegisterPrimOp primop_warn({
+    .name = "__warn",
+    .args = {"e1", "e2"},
+    .doc = R"(
+      Evaluate *e1*, which must be a string and print iton standard error as a warning.
+      Then return *e2*.
+      This function is useful for non-critical situations where attention is advisable.
+
+      If the
+      [`debugger-on-trace`](@docroot@/command-ref/conf-file.md#conf-debugger-on-trace)
+      or [`debugger-on-warn`](@docroot@/command-ref/conf-file.md#conf-debugger-on-warn)
+      option is set to `true` and the `--debugger` flag is given, the
+      interactive debugger will be started when `warn` is called (like
+      [`break`](@docroot@/language/builtins.md#builtins-break)).
+
+      If the
+      [`abort-on-warn`](@docroot@/command-ref/conf-file.md#conf-abort-on-warn)
+      option is set, the evaluation will be aborted after the warning is printed.
+      This is useful to reveal the stack trace of the warning, when the context is non-interactive and a debugger can not be launched.
+    )",
+    .fun = prim_warn,
+});
+
 
 /* Takes two arguments and evaluates to the second one. Used as the
  * builtins.traceVerbose implementation when --trace-verbose is not enabled
@@ -1116,12 +1162,34 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * *
     }
 }
 
+/**
+ * Early validation for the derivation name, for better error message.
+ * It is checked again when constructing store paths.
+ *
+ * @todo Check that the `.drv` suffix also fits.
+ */
+static void checkDerivationName(EvalState & state, std::string_view drvName)
+{
+    try {
+        checkName(drvName);
+    } catch (BadStorePathName & e) {
+        // "Please pass a different name": Users may not be aware that they can
+        //     pass a different one, in functions like `fetchurl` where the name
+        //     is optional.
+        // Note that Nixpkgs generally won't trigger this, because `mkDerivation`
+        // sanitizes the name.
+        state.error<EvalError>("invalid derivation name: %s. Please pass a different '%s'.", Uncolored(e.message()), "name").debugThrow();
+    }
+}
+
 static void derivationStrictInternal(
     EvalState & state,
     const std::string & drvName,
     const Bindings * attrs,
     Value & v)
 {
+    checkDerivationName(state, drvName);
+
     /* Check whether attributes should be passed as a JSON file. */
     using nlohmann::json;
     std::optional<json> jsonObject;
@@ -1156,13 +1224,13 @@ static void derivationStrictInternal(
 
     for (auto & i : attrs->lexicographicOrder(state.symbols)) {
         if (i->name == state.sIgnoreNulls) continue;
-        const std::string & key = state.symbols[i->name];
+        auto key = state.symbols[i->name];
         vomit("processing attribute '%1%'", key);
 
         auto handleHashMode = [&](const std::string_view s) {
             if (s == "recursive") {
                 // back compat, new name is "nar"
-                ingestionMethod = FileIngestionMethod::Recursive;
+                ingestionMethod = ContentAddressMethod::Raw::NixArchive;
             } else try {
                 ingestionMethod = ContentAddressMethod::parse(s);
             } catch (UsageError &) {
@@ -1170,9 +1238,9 @@ static void derivationStrictInternal(
                     "invalid value '%s' for 'outputHashMode' attribute", s
                 ).atPos(v).debugThrow();
             }
-            if (ingestionMethod == TextIngestionMethod {})
+            if (ingestionMethod == ContentAddressMethod::Raw::Text)
                 experimentalFeatureSettings.require(Xp::DynamicDerivations);
-            if (ingestionMethod == FileIngestionMethod::Git)
+            if (ingestionMethod == ContentAddressMethod::Raw::Git)
                 experimentalFeatureSettings.require(Xp::GitHashing);
         };
 
@@ -1185,11 +1253,11 @@ static void derivationStrictInternal(
                         .debugThrow();
                 /* !!! Check whether j is a valid attribute
                    name. */
-                /* Derivations cannot be named ‘drv’, because
-                   then we'd have an attribute ‘drvPath’ in
-                   the resulting set. */
-                if (j == "drv")
-                    state.error<EvalError>("invalid derivation output name 'drv'")
+                /* Derivations cannot be named ‘drvPath’, because
+                   we already have an attribute ‘drvPath’ in
+                   the resulting set (see state.sDrvPath). */
+                if (j == "drvPath")
+                    state.error<EvalError>("invalid derivation output name 'drvPath'")
                         .atPos(v)
                         .debugThrow();
                 outputs.insert(j);
@@ -1240,7 +1308,7 @@ static void derivationStrictInternal(
 
                     if (i->name == state.sStructuredAttrs) continue;
 
-                    (*jsonObject)[key] = printValueAsJSON(state, true, *i->value, pos, context);
+                    jsonObject->emplace(key, printValueAsJSON(state, true, *i->value, pos, context));
 
                     if (i->name == state.sBuilder)
                         drv.builder = state.forceString(*i->value, context, pos, context_below);
@@ -1261,6 +1329,20 @@ static void derivationStrictInternal(
                         handleOutputs(ss);
                     }
 
+                    if (i->name == state.sAllowedReferences)
+                        warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'allowedReferences'; use 'outputChecks.<output>.allowedReferences' instead", drvName);
+                    if (i->name == state.sAllowedRequisites)
+                        warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'allowedRequisites'; use 'outputChecks.<output>.allowedRequisites' instead", drvName);
+                    if (i->name == state.sDisallowedReferences)
+                        warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedReferences'; use 'outputChecks.<output>.disallowedReferences' instead", drvName);
+                    if (i->name == state.sDisallowedRequisites)
+                        warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedRequisites'; use 'outputChecks.<output>.disallowedRequisites' instead", drvName);
+                    if (i->name == state.sMaxSize)
+                        warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'maxSize'; use 'outputChecks.<output>.maxSize' instead", drvName);
+                    if (i->name == state.sMaxClosureSize)
+                        warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'maxClosureSize'; use 'outputChecks.<output>.maxClosureSize' instead", drvName);
+
+
                 } else {
                     auto s = state.coerceToString(pos, *i->value, context, context_below, true).toOwned();
                     drv.env.emplace(key, s);
@@ -1330,7 +1412,7 @@ static void derivationStrictInternal(
 
     /* Check whether the derivation name is valid. */
     if (isDerivation(drvName) &&
-        !(ingestionMethod == ContentAddressMethod { TextIngestionMethod { } } &&
+        !(ingestionMethod == ContentAddressMethod::Raw::Text &&
           outputs.size() == 1 &&
           *(outputs.begin()) == "out"))
     {
@@ -1352,7 +1434,7 @@ static void derivationStrictInternal(
 
         auto h = newHashAllowEmpty(*outputHash, outputHashAlgo);
 
-        auto method = ingestionMethod.value_or(FileIngestionMethod::Flat);
+        auto method = ingestionMethod.value_or(ContentAddressMethod::Raw::Flat);
 
         DerivationOutput::CAFixed dof {
             .ca = ContentAddress {
@@ -1371,7 +1453,7 @@ static void derivationStrictInternal(
                 .atPos(v).debugThrow();
 
         auto ha = outputHashAlgo.value_or(HashAlgorithm::SHA256);
-        auto method = ingestionMethod.value_or(FileIngestionMethod::Recursive);
+        auto method = ingestionMethod.value_or(ContentAddressMethod::Raw::NixArchive);
 
         for (auto & i : outputs) {
             drv.env[i] = hashPlaceholder(i);
@@ -1517,7 +1599,7 @@ static RegisterPrimOp primop_toPath({
    corner cases. */
 static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
-    if (evalSettings.pureEval)
+    if (state.settings.pureEval)
         state.error<EvalError>(
             "'%s' is not allowed in pure evaluation mode",
             "builtins.storePath"
@@ -1627,7 +1709,7 @@ static RegisterPrimOp primop_baseNameOf({
     .name = "baseNameOf",
     .args = {"x"},
     .doc = R"(
-      Return the *base name* of either a [path value](@docroot@/language/values.md#type-path) *x* or a string *x*, depending on which type is passed, and according to the following rules.
+      Return the *base name* of either a [path value](@docroot@/language/types.md#type-path) *x* or a string *x*, depending on which type is passed, and according to the following rules.
 
       For a path value, the *base name* is considered to be the part of the path after the last directory separator, including any file extensions.
       This is the simple case, as path values don't have trailing slashes.
@@ -1761,35 +1843,7 @@ static RegisterPrimOp primop_findFile(PrimOp {
     .doc = R"(
       Find *lookup-path* in *search-path*.
 
-      A search path is represented list of [attribute sets](./values.md#attribute-set) with two attributes:
-      - `prefix` is a relative path.
-      - `path` denotes a file system location
-      The exact syntax depends on the command line interface.
-
-      Examples of search path attribute sets:
-
-      - ```
-        {
-          prefix = "nixos-config";
-          path = "/etc/nixos/configuration.nix";
-        }
-        ```
-
-      - ```
-        {
-          prefix = "";
-          path = "/nix/var/nix/profiles/per-user/root/channels";
-        }
-        ```
-
-      The lookup algorithm checks each entry until a match is found, returning a [path value](@docroot@/language/values.html#type-path) of the match:
-
-      - If *lookup-path* matches `prefix`, then the remainder of *lookup-path* (the "suffix") is searched for within the directory denoted by `path`.
-        Note that the `path` may need to be downloaded at this point to look inside.
-      - If the suffix is found inside that directory, then the entry is a match.
-        The combined absolute path of the directory (now downloaded if need be) and the suffix is returned.
-
-      [Lookup path](@docroot@/language/constructs/lookup-path.md) expressions are [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath):
+      [Lookup path](@docroot@/language/constructs/lookup-path.md) expressions are [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and [`builtins.nixPath`](#builtins-nixPath):
 
       ```nix
       <nixpkgs>
@@ -1800,6 +1854,119 @@ static RegisterPrimOp primop_findFile(PrimOp {
       ```nix
       builtins.findFile builtins.nixPath "nixpkgs"
       ```
+
+      A search path is represented as a list of [attribute sets](./types.md#attribute-set) with two attributes:
+      - `prefix` is a relative path.
+      - `path` denotes a file system location
+
+      Examples of search path attribute sets:
+
+      - ```
+        {
+          prefix = "";
+          path = "/nix/var/nix/profiles/per-user/root/channels";
+        }
+        ```
+      - ```
+        {
+          prefix = "nixos-config";
+          path = "/etc/nixos/configuration.nix";
+        }
+        ```
+      - ```
+        {
+          prefix = "nixpkgs";
+          path = "https://github.com/NixOS/nixpkgs/tarballs/master";
+        }
+        ```
+      - ```
+        {
+          prefix = "nixpkgs";
+          path = "channel:nixpkgs-unstable";
+        }
+        ```
+      - ```
+        {
+          prefix = "flake-compat";
+          path = "flake:github:edolstra/flake-compat";
+        }
+        ```
+
+      The lookup algorithm checks each entry until a match is found, returning a [path value](@docroot@/language/types.md#type-path) of the match:
+
+      - If a prefix of `lookup-path` matches `prefix`, then the remainder of *lookup-path* (the "suffix") is searched for within the directory denoted by `path`.
+        The contents of `path` may need to be downloaded at this point to look inside.
+
+      - If the suffix is found inside that directory, then the entry is a match.
+        The combined absolute path of the directory (now downloaded if need be) and the suffix is returned.
+
+      > **Example**
+      >
+      > A *search-path* value
+      >
+      > ```
+      > [
+      >   {
+      >     prefix = "";
+      >     path = "/home/eelco/Dev";
+      >   }
+      >   {
+      >     prefix = "nixos-config";
+      >     path = "/etc/nixos";
+      >   }
+      > ]
+      > ```
+      >
+      > and a *lookup-path* value `"nixos-config"` will cause Nix to try `/home/eelco/Dev/nixos-config` and `/etc/nixos` in that order and return the first path that exists.
+
+      If `path` starts with `http://` or `https://`, it is interpreted as the URL of a tarball that will be downloaded and unpacked to a temporary location.
+      The tarball must consist of a single top-level directory.
+
+      The URLs of the tarballs from the official `nixos.org` channels can be abbreviated as `channel:<channel-name>`.
+      See [documentation on `nix-channel`](@docroot@/command-ref/nix-channel.md) for details about channels.
+
+      > **Example**
+      >
+      > These two search path entries are equivalent:
+      >
+      > - ```
+      >   {
+      >     prefix = "nixpkgs";
+      >     path = "channel:nixpkgs-unstable";
+      >   }
+      >   ```
+      > - ```
+      >   {
+      >     prefix = "nixpkgs";
+      >     path = "https://nixos.org/channels/nixos-unstable/nixexprs.tar.xz";
+      >   }
+      >   ```
+
+      Search paths can also point to source trees using [flake URLs](@docroot@/command-ref/new-cli/nix3-flake.md#url-like-syntax).
+
+
+      > **Example**
+      >
+      > The search path entry
+      >
+      > ```
+      > {
+      >   prefix = "nixpkgs";
+      >   path = "flake:nixpkgs";
+      > }
+      > ```
+      > specifies that the prefix `nixpkgs` shall refer to the source tree downloaded from the `nixpkgs` entry in the flake registry.
+      >
+      > Similarly
+      >
+      > ```
+      > {
+      >   prefix = "nixpkgs";
+      >   path = "flake:github:nixos/nixpkgs/nixos-22.05";
+      > }
+      > ```
+      >
+      > makes `<nixpkgs>` refer to a particular branch of the `NixOS/nixpkgs` repository on GitHub.
     )",
     .fun = prim_findFile,
 });
@@ -1828,12 +1995,12 @@ static RegisterPrimOp primop_hashFile({
     .fun = prim_hashFile,
 });
 
-static Value * fileTypeToString(EvalState & state, InputAccessor::Type type)
+static Value * fileTypeToString(EvalState & state, SourceAccessor::Type type)
 {
     return
-        type == InputAccessor::Type::tRegular ? &state.vStringRegular :
-        type == InputAccessor::Type::tDirectory ? &state.vStringDirectory :
-        type == InputAccessor::Type::tSymlink ? &state.vStringSymlink :
+        type == SourceAccessor::Type::tRegular ? &state.vStringRegular :
+        type == SourceAccessor::Type::tDirectory ? &state.vStringDirectory :
+        type == SourceAccessor::Type::tSymlink ? &state.vStringSymlink :
         &state.vStringUnknown;
 }
 
@@ -2147,7 +2314,7 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Val
         })
         : ({
             StringSource s { contents };
-            state.store->addToStoreFromDump(s, name, FileSerialisationMethod::Flat, TextIngestionMethod {}, HashAlgorithm::SHA256, refs, state.repair);
+            state.store->addToStoreFromDump(s, name, FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, refs, state.repair);
         });
 
     /* Note: we don't need to add `context' to the context of the
@@ -2210,7 +2377,7 @@ static RegisterPrimOp primop_toFile({
       ```
 
       Note that `${configFile}` is a
-      [string interpolation](@docroot@/language/values.md#type-string), so the result of the
+      [string interpolation](@docroot@/language/types.md#type-string), so the result of the
       expression `configFile`
       (i.e., a path like `/nix/store/m7p7jfny445k...-foo.conf`) will be
       spliced into the resulting string.
@@ -2262,7 +2429,7 @@ static void addPath(
     std::string_view name,
     SourcePath path,
     Value * filterFun,
-    FileIngestionMethod method,
+    ContentAddressMethod method,
     const std::optional<Hash> expectedHash,
     Value & v,
     const NixStringContext & context)
@@ -2294,11 +2461,10 @@ static void addPath(
 
         std::optional<StorePath> expectedStorePath;
         if (expectedHash)
-            expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo {
-                .method = method,
-                .hash = *expectedHash,
-                .references = {},
-            });
+            expectedStorePath = state.store->makeFixedOutputPathFromCA(name, ContentAddressWithReferences::fromParts(
+                method,
+                *expectedHash,
+                {}));
 
         if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
             auto dstPath = fetchToStore(
@@ -2331,7 +2497,7 @@ static void prim_filterSource(EvalState & state, const PosIdx pos, Value * * arg
         "while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'");
     state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource");
 
-    addPath(state, pos, path.baseName(), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v, context);
+    addPath(state, pos, path.baseName(), path, args[0], ContentAddressMethod::Raw::NixArchive, std::nullopt, v, context);
 }
 
 static RegisterPrimOp primop_filterSource({
@@ -2394,7 +2560,7 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value
     std::optional<SourcePath> path;
     std::string name;
     Value * filterFun = nullptr;
-    auto method = FileIngestionMethod::Recursive;
+    auto method = ContentAddressMethod::Raw::NixArchive;
     std::optional<Hash> expectedHash;
     NixStringContext context;
 
@@ -2409,7 +2575,9 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value
         else if (n == "filter")
             state.forceFunction(*(filterFun = attr.value), attr.pos, "while evaluating the `filter` parameter passed to builtins.path");
         else if (n == "recursive")
-            method = FileIngestionMethod { state.forceBool(*attr.value, attr.pos, "while evaluating the `recursive` attribute passed to builtins.path") };
+            method = state.forceBool(*attr.value, attr.pos, "while evaluating the `recursive` attribute passed to builtins.path")
+                ? ContentAddressMethod::Raw::NixArchive
+                : ContentAddressMethod::Raw::Flat;
         else if (n == "sha256")
             expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `sha256` attribute passed to builtins.path"), HashAlgorithm::SHA256);
         else
@@ -2594,13 +2762,13 @@ static struct LazyPosAcessors {
     PrimOp primop_lineOfPos{
         .arity = 1,
         .fun = [] (EvalState & state, PosIdx pos, Value * * args, Value & v) {
-            v.mkInt(state.positions[PosIdx(args[0]->integer())].line);
+            v.mkInt(state.positions[PosIdx(args[0]->integer().value)].line);
         }
     };
     PrimOp primop_columnOfPos{
         .arity = 1,
         .fun = [] (EvalState & state, PosIdx pos, Value * * args, Value & v) {
-            v.mkInt(state.positions[PosIdx(args[0]->integer())].column);
+            v.mkInt(state.positions[PosIdx(args[0]->integer().value)].column);
         }
     };
 
@@ -3076,7 +3244,8 @@ static void elemAt(EvalState & state, const PosIdx pos, Value & list, int n, Val
 /* Return the n-1'th element of a list. */
 static void prim_elemAt(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
-    elemAt(state, pos, *args[0], state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.elemAt"), v);
+    NixInt::Inner elem = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.elemAt").value;
+    elemAt(state, pos, *args[0], elem, v);
 }
 
 static RegisterPrimOp primop_elemAt({
@@ -3370,10 +3539,12 @@ static RegisterPrimOp primop_all({
 
 static void prim_genList(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
-    auto len = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.genList");
+    auto len_ = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.genList").value;
 
-    if (len < 0)
-        state.error<EvalError>("cannot create list of size %1%", len).atPos(pos).debugThrow();
+    if (len_ < 0)
+        state.error<EvalError>("cannot create list of size %1%", len_).atPos(pos).debugThrow();
+
+    size_t len = size_t(len_);
 
     // More strict than striclty (!) necessary, but acceptable
     // as evaluating map without accessing any values makes little sense.
@@ -3630,9 +3801,17 @@ static void prim_add(EvalState & state, const PosIdx pos, Value * * args, Value
     if (args[0]->type() == nFloat || args[1]->type() == nFloat)
         v.mkFloat(state.forceFloat(*args[0], pos, "while evaluating the first argument of the addition")
                 + state.forceFloat(*args[1], pos, "while evaluating the second argument of the addition"));
-    else
-        v.mkInt(  state.forceInt(*args[0], pos, "while evaluating the first argument of the addition")
-                + state.forceInt(*args[1], pos, "while evaluating the second argument of the addition"));
+    else {
+        auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument of the addition");
+        auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument of the addition");
+
+        auto result_ = i1 + i2;
+        if (auto result = result_.valueChecked(); result.has_value()) {
+            v.mkInt(*result);
+        } else {
+            state.error<EvalError>("integer overflow in adding %1% + %2%", i1, i2).atPos(pos).debugThrow();
+        }
+    }
 }
 
 static RegisterPrimOp primop_add({
@@ -3651,9 +3830,18 @@ static void prim_sub(EvalState & state, const PosIdx pos, Value * * args, Value
     if (args[0]->type() == nFloat || args[1]->type() == nFloat)
         v.mkFloat(state.forceFloat(*args[0], pos, "while evaluating the first argument of the subtraction")
                 - state.forceFloat(*args[1], pos, "while evaluating the second argument of the subtraction"));
-    else
-        v.mkInt(  state.forceInt(*args[0], pos, "while evaluating the first argument of the subtraction")
-                - state.forceInt(*args[1], pos, "while evaluating the second argument of the subtraction"));
+    else {
+        auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument of the subtraction");
+        auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument of the subtraction");
+
+        auto result_ = i1 - i2;
+
+        if (auto result = result_.valueChecked(); result.has_value()) {
+            v.mkInt(*result);
+        } else {
+            state.error<EvalError>("integer overflow in subtracting %1% - %2%", i1, i2).atPos(pos).debugThrow();
+        }
+    }
 }
 
 static RegisterPrimOp primop_sub({
@@ -3672,9 +3860,18 @@ static void prim_mul(EvalState & state, const PosIdx pos, Value * * args, Value
     if (args[0]->type() == nFloat || args[1]->type() == nFloat)
         v.mkFloat(state.forceFloat(*args[0], pos, "while evaluating the first of the multiplication")
                 * state.forceFloat(*args[1], pos, "while evaluating the second argument of the multiplication"));
-    else
-        v.mkInt(  state.forceInt(*args[0], pos, "while evaluating the first argument of the multiplication")
-                * state.forceInt(*args[1], pos, "while evaluating the second argument of the multiplication"));
+    else {
+        auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument of the multiplication");
+        auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument of the multiplication");
+
+        auto result_ = i1 * i2;
+
+        if (auto result = result_.valueChecked(); result.has_value()) {
+            v.mkInt(*result);
+        } else {
+            state.error<EvalError>("integer overflow in multiplying %1% * %2%", i1, i2).atPos(pos).debugThrow();
+        }
+    }
 }
 
 static RegisterPrimOp primop_mul({
@@ -3701,10 +3898,12 @@ static void prim_div(EvalState & state, const PosIdx pos, Value * * args, Value
         NixInt i1 = state.forceInt(*args[0], pos, "while evaluating the first operand of the division");
         NixInt i2 = state.forceInt(*args[1], pos, "while evaluating the second operand of the division");
         /* Avoid division overflow as it might raise SIGFPE. */
-        if (i1 == std::numeric_limits<NixInt>::min() && i2 == -1)
-            state.error<EvalError>("overflow in integer division").atPos(pos).debugThrow();
-
-        v.mkInt(i1 / i2);
+        auto result_ = i1 / i2;
+        if (auto result = result_.valueChecked(); result.has_value()) {
+            v.mkInt(*result);
+        } else {
+            state.error<EvalError>("integer overflow in dividing %1% / %2%", i1, i2).atPos(pos).debugThrow();
+        }
     }
 }
 
@@ -3719,8 +3918,9 @@ static RegisterPrimOp primop_div({
 
 static void prim_bitAnd(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
-    v.mkInt(state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitAnd")
-            & state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitAnd"));
+    auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitAnd");
+    auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitAnd");
+    v.mkInt(i1.value & i2.value);
 }
 
 static RegisterPrimOp primop_bitAnd({
@@ -3734,8 +3934,10 @@ static RegisterPrimOp primop_bitAnd({
 
 static void prim_bitOr(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
-    v.mkInt(state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitOr")
-            | state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitOr"));
+    auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitOr");
+    auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitOr");
+
+    v.mkInt(i1.value | i2.value);
 }
 
 static RegisterPrimOp primop_bitOr({
@@ -3749,8 +3951,10 @@ static RegisterPrimOp primop_bitOr({
 
 static void prim_bitXor(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
-    v.mkInt(state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitXor")
-            ^ state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitXor"));
+    auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitXor");
+    auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitXor");
+
+    v.mkInt(i1.value ^ i2.value);
 }
 
 static RegisterPrimOp primop_bitXor({
@@ -3830,13 +4034,19 @@ static RegisterPrimOp primop_toString({
    non-negative. */
 static void prim_substring(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
-    int start = state.forceInt(*args[0], pos, "while evaluating the first argument (the start offset) passed to builtins.substring");
+    NixInt::Inner start = state.forceInt(*args[0], pos, "while evaluating the first argument (the start offset) passed to builtins.substring").value;
 
     if (start < 0)
         state.error<EvalError>("negative start position in 'substring'").atPos(pos).debugThrow();
 
 
-    int len = state.forceInt(*args[1], pos, "while evaluating the second argument (the substring length) passed to builtins.substring");
+    NixInt::Inner len = state.forceInt(*args[1], pos, "while evaluating the second argument (the substring length) passed to builtins.substring").value;
+
+    // Negative length may be idiomatically passed to builtins.substring to get
+    // the tail of the string.
+    if (len < 0) {
+        len = std::numeric_limits<NixInt::Inner>::max();
+    }
 
     // Special-case on empty substring to avoid O(n) strlen
     // This allows for the use of empty substrings to efficently capture string context
@@ -3879,7 +4089,7 @@ static void prim_stringLength(EvalState & state, const PosIdx pos, Value * * arg
 {
     NixStringContext context;
     auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.stringLength");
-    v.mkInt(s->size());
+    v.mkInt(NixInt::Inner(s->size()));
 }
 
 static RegisterPrimOp primop_stringLength({
@@ -4014,17 +4224,23 @@ static RegisterPrimOp primop_convertHash({
 
 struct RegexCache
 {
-    // TODO use C++20 transparent comparison when available
-    std::unordered_map<std::string_view, std::regex> cache;
-    std::list<std::string> keys;
+    struct State
+    {
+        // TODO use C++20 transparent comparison when available
+        std::unordered_map<std::string_view, std::regex> cache;
+        std::list<std::string> keys;
+    };
+
+    Sync<State> state_;
 
     std::regex get(std::string_view re)
     {
-        auto it = cache.find(re);
-        if (it != cache.end())
+        auto state(state_.lock());
+        auto it = state->cache.find(re);
+        if (it != state->cache.end())
             return it->second;
-        keys.emplace_back(re);
-        return cache.emplace(keys.back(), std::regex(keys.back(), std::regex::extended)).first->second;
+        state->keys.emplace_back(re);
+        return state->cache.emplace(state->keys.back(), std::regex(state->keys.back(), std::regex::extended)).first->second;
     }
 };
 
@@ -4357,7 +4573,8 @@ static void prim_compareVersions(EvalState & state, const PosIdx pos, Value * *
 {
     auto version1 = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.compareVersions");
     auto version2 = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.compareVersions");
-    v.mkInt(compareVersions(version1, version2));
+    auto result = compareVersions(version1, version2);
+    v.mkInt(result < 0 ? -1 : result > 0 ? 1 : 0);
 }
 
 static RegisterPrimOp primop_compareVersions({
@@ -4429,7 +4646,7 @@ void EvalState::createBaseEnv()
     addConstant("builtins", v, {
         .type = nAttrs,
         .doc = R"(
-          Contains all the [built-in functions](@docroot@/language/builtins.md) and values.
+          Contains all the built-in functions and values.
 
           Since built-in functions were added over time, [testing for attributes](./operators.md#has-attribute) in `builtins` can be used for graceful fallback on older Nix installations:
 
@@ -4449,7 +4666,7 @@ void EvalState::createBaseEnv()
           It can be returned by
           [comparison operators](@docroot@/language/operators.md#Comparison)
           and used in
-          [conditional expressions](@docroot@/language/constructs.md#Conditionals).
+          [conditional expressions](@docroot@/language/syntax.md#Conditionals).
 
           The name `true` is not special, and can be shadowed:
 
@@ -4469,7 +4686,7 @@ void EvalState::createBaseEnv()
           It can be returned by
           [comparison operators](@docroot@/language/operators.md#Comparison)
           and used in
-          [conditional expressions](@docroot@/language/constructs.md#Conditionals).
+          [conditional expressions](@docroot@/language/syntax.md#Conditionals).
 
           The name `false` is not special, and can be shadowed:
 
@@ -4494,7 +4711,7 @@ void EvalState::createBaseEnv()
         )",
     });
 
-    if (!evalSettings.pureEval) {
+    if (!settings.pureEval) {
         v.mkInt(time(0));
     }
     addConstant("__currentTime", v, {
@@ -4516,13 +4733,13 @@ void EvalState::createBaseEnv()
           1683705525
           ```
 
-          The [store path](@docroot@/glossary.md#gloss-store-path) of a derivation depending on `currentTime` will differ for each evaluation, unless both evaluate `builtins.currentTime` in the same second.
+          The [store path](@docroot@/store/store-path.md) of a derivation depending on `currentTime` will differ for each evaluation, unless both evaluate `builtins.currentTime` in the same second.
         )",
         .impureOnly = true,
     });
 
-    if (!evalSettings.pureEval)
-        v.mkString(evalSettings.getCurrentSystem());
+    if (!settings.pureEval)
+        v.mkString(settings.getCurrentSystem());
     addConstant("__currentSystem", v, {
         .type = nString,
         .doc = R"(
@@ -4602,7 +4819,7 @@ void EvalState::createBaseEnv()
 
 #ifndef _WIN32 // TODO implement on Windows
     // Miscellaneous
-    if (evalSettings.enableNativeCode) {
+    if (settings.enableNativeCode) {
         addPrimOp({
             .name = "__importNative",
             .arity = 2,
@@ -4625,7 +4842,7 @@ void EvalState::createBaseEnv()
           error if `--trace-verbose` is enabled. Then return *e2*. This function
           is useful for debugging.
         )",
-        .fun = evalSettings.traceVerbose ? prim_trace : prim_second,
+        .fun = settings.traceVerbose ? prim_trace : prim_second,
     });
 
     /* Add a value containing the current Nix expression search path. */
@@ -4640,7 +4857,17 @@ void EvalState::createBaseEnv()
     addConstant("__nixPath", v, {
         .type = nList,
         .doc = R"(
-          The value of the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path): a list of search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md).
+          A list of search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md).
+          Its value is primarily determined by the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path), which are
+          - Overridden by the [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) environment variable or the `--nix-path` option
+          - Extended by the [`-I` option](@docroot@/command-ref/opt-common.md#opt-I) or `--extra-nix-path`
+
+          > **Example**
+          >
+          > ```bash
+          > $ NIX_PATH= nix-instantiate --eval --expr "builtins.nixPath" -I foo=bar --no-pure-eval
+          > [ { path = "bar"; prefix = "foo"; } ]
+          > ```
 
           Lookup path expressions are [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and
           [`builtins.findFile`](./builtins.html#builtins-findFile):

src/libexpr/primops/context.cc

@@ -14,8 +14,11 @@ static void prim_unsafeDiscardStringContext(EvalState & state, const PosIdx pos,
 
 static RegisterPrimOp primop_unsafeDiscardStringContext({
     .name = "__unsafeDiscardStringContext",
-    .arity = 1,
-    .fun = prim_unsafeDiscardStringContext
+    .args = {"s"},
+    .doc = R"(
+        Discard the [string context](@docroot@/language/string-context.md) from a value that can be coerced to a string.
+    )",
+    .fun = prim_unsafeDiscardStringContext,
 });
 
 
@@ -75,7 +78,11 @@ static RegisterPrimOp primop_unsafeDiscardOutputDependency({
     .name = "__unsafeDiscardOutputDependency",
     .args = {"s"},
     .doc = R"(
-      Create a copy of the given string where every "derivation deep" string context element is turned into a constant string context element.
+      Create a copy of the given string where every
+      [derivation deep](@docroot@/language/string-context.md#string-context-element-derivation-deep)
+      string context element is turned into a
+      [constant](@docroot@/language/string-context.md#string-context-element-constant)
+      string context element.
 
       This is the opposite of [`builtins.addDrvOutputDependencies`](#builtins-addDrvOutputDependencies).
 
@@ -137,7 +144,11 @@ static RegisterPrimOp primop_addDrvOutputDependencies({
     .name = "__addDrvOutputDependencies",
     .args = {"s"},
     .doc = R"(
-      Create a copy of the given string where a single constant string context element is turned into a "derivation deep" string context element.
+      Create a copy of the given string where a single
+      [constant](@docroot@/language/string-context.md#string-context-element-constant)
+      string context element is turned into a
+      [derivation deep](@docroot@/language/string-context.md#string-context-element-derivation-deep)
+      string context element.
 
       The store path that is the constant string context element should point to a valid derivation, and end in `.drv`.
 

src/libexpr/primops/derivation.nix

@@ -1,6 +1,31 @@
-/* This is the implementation of the ‘derivation’ builtin function.
-   It's actually a wrapper around the ‘derivationStrict’ primop. */
+# This is the implementation of the ‘derivation’ builtin function.
+# It's actually a wrapper around the ‘derivationStrict’ primop.
+# Note that the following comment will be shown in :doc in the repl, but not in the manual.
 
+/**
+  Create a derivation.
+
+  # Inputs
+
+  The single argument is an attribute set that describes what to build and how to build it.
+  See https://nix.dev/manual/nix/2.23/language/derivations
+
+  # Output
+
+  The result is an attribute set that describes the derivation.
+  Notably it contains the outputs, which in the context of the Nix language are special strings that refer to the output paths, which may not yet exist.
+  The realisation of these outputs only occurs when needed; for example
+
+    * When `nix-build` or a similar command is run, it realises the outputs that were requested on its command line.
+      See https://nix.dev/manual/nix/2.23/command-ref/nix-build
+
+    * When `import`, `readFile`, `readDir` or some other functions are called, they have to realise the outputs they depend on.
+      This is referred to as "import from derivation".
+      See https://nix.dev/manual/nix/2.23/language/import-from-derivation
+
+  Note that `derivation` is very bare-bones, and provides almost no commands during the build.
+  Most likely, you'll want to use functions like `stdenv.mkDerivation` in Nixpkgs to set up a basic environment.
+*/
 drvAttrs @ { outputs ? [ "out" ], ... }:
 
 let

src/libexpr/primops/fetchMercurial.cc

@@ -53,7 +53,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a
     // whitelist. Ah well.
     state.checkURI(url);
 
-    if (evalSettings.pureEval && !rev)
+    if (state.settings.pureEval && !rev)
         throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision");
 
     fetchers::Attrs attrs;
@@ -62,7 +62,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a
     attrs.insert_or_assign("name", std::string(name));
     if (ref) attrs.insert_or_assign("ref", *ref);
     if (rev) attrs.insert_or_assign("rev", rev->gitRev());
-    auto input = fetchers::Input::fromAttrs(std::move(attrs));
+    auto input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs));
 
     auto [storePath, input2] = input.fetchToStore(state.store);
 

src/libexpr/primops/fetchTree.cc

@@ -1,4 +1,4 @@
-#include "libfetchers/attrs.hh"
+#include "attrs.hh"
 #include "primops.hh"
 #include "eval-inline.hh"
 #include "eval-settings.hh"
@@ -85,7 +85,7 @@ static void fetchTree(
     Value & v,
     const FetchTreeParams & params = FetchTreeParams{}
 ) {
-    fetchers::Input input;
+    fetchers::Input input { state.fetchSettings };
     NixStringContext context;
     std::optional<std::string> type;
     if (params.isFetchGit) type = "git";
@@ -122,9 +122,15 @@ static void fetchTree(
             }
             else if (attr.value->type() == nBool)
                 attrs.emplace(state.symbols[attr.name], Explicit<bool>{attr.value->boolean()});
-            else if (attr.value->type() == nInt)
-                attrs.emplace(state.symbols[attr.name], uint64_t(attr.value->integer()));
-            else if (state.symbols[attr.name] == "publicKeys") {
+            else if (attr.value->type() == nInt) {
+                auto intValue = attr.value->integer().value;
+
+                if (intValue < 0) {
+                    state.error<EvalError>("negative value given for fetchTree attr %1%: %2%", state.symbols[attr.name], intValue).atPos(pos).debugThrow();
+                }
+
+                attrs.emplace(state.symbols[attr.name], uint64_t(intValue));
+            } else if (state.symbols[attr.name] == "publicKeys") {
                 experimentalFeatureSettings.require(Xp::VerifiedFetches);
                 attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, pos, context).dump());
             }
@@ -137,13 +143,18 @@ static void fetchTree(
             attrs.emplace("exportIgnore", Explicit<bool>{true});
         }
 
+        // fetchTree should fetch git repos with shallow = true by default
+        if (type == "git" && !params.isFetchGit && !attrs.contains("shallow")) {
+            attrs.emplace("shallow", Explicit<bool>{true});
+        }
+
         if (!params.allowNameArgument)
             if (auto nameIter = attrs.find("name"); nameIter != attrs.end())
                 state.error<EvalError>(
                     "attribute 'name' isn’t supported in call to 'fetchTree'"
                 ).atPos(pos).debugThrow();
 
-        input = fetchers::Input::fromAttrs(std::move(attrs));
+        input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs));
     } else {
         auto url = state.coerceToString(pos, *args[0], context,
                 "while evaluating the first argument passed to the fetcher",
@@ -156,20 +167,20 @@ static void fetchTree(
             if (!attrs.contains("exportIgnore") && (!attrs.contains("submodules") || !*fetchers::maybeGetBoolAttr(attrs, "submodules"))) {
                 attrs.emplace("exportIgnore", Explicit<bool>{true});
             }
-            input = fetchers::Input::fromAttrs(std::move(attrs));
+            input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs));
         } else {
             if (!experimentalFeatureSettings.isEnabled(Xp::Flakes))
                 state.error<EvalError>(
                     "passing a string argument to 'fetchTree' requires the 'flakes' experimental feature"
                 ).atPos(pos).debugThrow();
-            input = fetchers::Input::fromURL(url);
+            input = fetchers::Input::fromURL(state.fetchSettings, url);
         }
     }
 
-    if (!evalSettings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes))
+    if (!state.settings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes))
         input = lookupInRegistries(state.store, input).first;
 
-    if (evalSettings.pureEval && !input.isLocked()) {
+    if (state.settings.pureEval && !input.isLocked()) {
         auto fetcher = "fetchTree";
         if (params.isFetchGit)
             fetcher = "fetchGit";
@@ -200,8 +211,8 @@ static RegisterPrimOp primop_fetchTree({
     .doc = R"(
       Fetch a file system tree or a plain file using one of the supported backends and return an attribute set with:
 
-      - the resulting fixed-output [store path](@docroot@/glossary.md#gloss-store-path)
-      - the corresponding [NAR](@docroot@/glossary.md#gloss-nar) hash
+      - the resulting fixed-output [store path](@docroot@/store/store-path.md)
+      - the corresponding [NAR](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) hash
       - backend-specific metadata (currently not documented). <!-- TODO: document output attributes -->
 
       *input* must be an attribute set with the following attributes:
@@ -320,6 +331,8 @@ static RegisterPrimOp primop_fetchTree({
 
         - `ref` (String, optional)
 
+          By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
+
           A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name.
 
           Default: `"HEAD"`
@@ -333,8 +346,9 @@ static RegisterPrimOp primop_fetchTree({
         - `shallow` (Bool, optional)
 
           Make a shallow clone when fetching the Git tree.
+          When this is enabled, the options `ref` and `allRefs` have no effect anymore.
 
-          Default: `false`
+          Default: `true`
 
         - `submodules` (Bool, optional)
 
@@ -344,8 +358,11 @@ static RegisterPrimOp primop_fetchTree({
 
         - `allRefs` (Bool, optional)
 
-          If set to `true`, always fetch the entire repository, even if the latest commit is still in the cache.
-          Otherwise, only the latest commit is fetched if it is not already cached.
+          By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
+
+          Whether to fetch all references (eg. branches and tags) of the repository.
+          With this argument being true, it's possible to load a `rev` from *any* `ref`.
+          (Without setting this option, only `rev`s from the specified `ref` are supported).
 
           Default: `false`
 
@@ -372,7 +389,7 @@ static RegisterPrimOp primop_fetchTree({
       - `"mercurial"`
 
      *input* can also be a [URL-like reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references).
-     The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-flakes) to be enabled.
+     The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/development/experimental-features.md#xp-feature-flakes) to be enabled.
 
       > **Example**
       >
@@ -420,7 +437,10 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
 
     state.forceValue(*args[0], pos);
 
-    if (args[0]->type() == nAttrs) {
+    bool isArgAttrs = args[0]->type() == nAttrs;
+    bool nameAttrPassed = false;
+
+    if (isArgAttrs) {
 
         for (auto & attr : *args[0]->attrs()) {
             std::string_view n(state.symbols[attr.name]);
@@ -428,8 +448,10 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
                 url = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the url we should fetch");
             else if (n == "sha256")
                 expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the sha256 of the content we should fetch"), HashAlgorithm::SHA256);
-            else if (n == "name")
+            else if (n == "name") {
+                nameAttrPassed = true;
                 name = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the name of the content we should fetch");
+            }
             else
                 state.error<EvalError>("unsupported argument '%s' to '%s'", n, who)
                 .atPos(pos).debugThrow();
@@ -442,14 +464,27 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
         url = state.forceStringNoCtx(*args[0], pos, "while evaluating the url we should fetch");
 
     if (who == "fetchTarball")
-        url = evalSettings.resolvePseudoUrl(*url);
+        url = state.settings.resolvePseudoUrl(*url);
 
     state.checkURI(*url);
 
     if (name == "")
         name = baseNameOf(*url);
 
-    if (evalSettings.pureEval && !expectedHash)
+    try {
+        checkName(name);
+    } catch (BadStorePathName & e) {
+        auto resolution =
+            nameAttrPassed ? HintFmt("Please change the value for the 'name' attribute passed to '%s', so that it can create a valid store path.", who) :
+            isArgAttrs ? HintFmt("Please add a valid 'name' attribute to the argument for '%s', so that it can create a valid store path.", who) :
+            HintFmt("Please pass an attribute set with 'url' and 'name' attributes to '%s',  so that it can create a valid store path.", who);
+
+        state.error<EvalError>(
+            std::string("invalid store path name when fetching URL '%s': %s. %s"), *url, Uncolored(e.message()), Uncolored(resolution.str()))
+        .atPos(pos).debugThrow();
+    }
+
+    if (state.settings.pureEval && !expectedHash)
         state.error<EvalError>("in pure evaluation mode, '%s' requires a 'sha256' argument", who).atPos(pos).debugThrow();
 
     // early exit if pinned and already in the store
@@ -457,7 +492,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
         auto expectedPath = state.store->makeFixedOutputPath(
             name,
             FixedOutputInfo {
-                .method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat,
+                .method = unpack ? FileIngestionMethod::NixArchive : FileIngestionMethod::Flat,
                 .hash = *expectedHash,
                 .references = {}
             });
@@ -472,7 +507,11 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
     //       https://github.com/NixOS/nix/issues/4313
     auto storePath =
         unpack
-        ? fetchToStore(*state.store, fetchers::downloadTarball(*url).accessor, FetchMode::Copy, name)
+        ? fetchToStore(
+            *state.store,
+            fetchers::downloadTarball(state.store, state.fetchSettings, *url),
+            FetchMode::Copy,
+            name)
         : fetchers::downloadFile(state.store, *url, name).storePath;
 
     if (expectedHash) {
@@ -500,9 +539,19 @@ static void prim_fetchurl(EvalState & state, const PosIdx pos, Value * * args, V
 
 static RegisterPrimOp primop_fetchurl({
     .name = "__fetchurl",
-    .args = {"url"},
+    .args = {"arg"},
     .doc = R"(
       Download the specified URL and return the path of the downloaded file.
+      `arg` can be either a string denoting the URL, or an attribute set with the following attributes:
+
+      - `url`
+
+        The URL of the file to download.
+
+      - `name` (default: the last path component of the URL)
+
+        A name for the file in the store. This can be useful if the URL has any
+        characters that are invalid for the store.
 
       Not available in [restricted evaluation mode](@docroot@/command-ref/conf-file.md#conf-restrict-eval).
     )",
@@ -520,11 +569,11 @@ static RegisterPrimOp primop_fetchTarball({
     .doc = R"(
       Download the specified URL, unpack it and return the path of the
       unpacked tree. The file must be a tape archive (`.tar`) compressed
-      with `gzip`, `bzip2` or `xz`. The top-level path component of the
-      files in the tarball is removed, so it is best if the tarball
-      contains a single directory at top level. The typical use of the
-      function is to obtain external Nix expression dependencies, such as
-      a particular version of Nixpkgs, e.g.
+      with `gzip`, `bzip2` or `xz`. If the tarball consists of a
+      single directory, then the top-level path component of the files
+      in the tarball is removed. The typical use of the function is to
+      obtain external Nix expression dependencies, such as a
+      particular version of Nixpkgs, e.g.
 
       ```nix
       with import (fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz) {};
@@ -599,6 +648,8 @@ static RegisterPrimOp primop_fetchGit({
 
         [Git reference]: https://git-scm.com/book/en/v2/Git-Internals-Git-References
 
+        This option has no effect once `shallow` cloning is enabled.
+
         By default, the `ref` value is prefixed with `refs/heads/`.
         As of 2.3.0, Nix will not prefix `refs/heads/` if `ref` starts with `refs/`.
 
@@ -616,23 +667,25 @@ static RegisterPrimOp primop_fetchGit({
       - `shallow` (default: `false`)
 
         Make a shallow clone when fetching the Git tree.
-
+        When this is enabled, the options `ref` and `allRefs` have no effect anymore.
       - `allRefs`
 
-        Whether to fetch all references of the repository.
-        With this argument being true, it's possible to load a `rev` from *any* `ref`
+        Whether to fetch all references (eg. branches and tags) of the repository.
+        With this argument being true, it's possible to load a `rev` from *any* `ref`.
         (by default only `rev`s from the specified `ref` are supported).
 
+        This option has no effect once `shallow` cloning is enabled.
+
       - `verifyCommit` (default: `true` if `publicKey` or `publicKeys` are provided, otherwise `false`)
 
         Whether to check `rev` for a signature matching `publicKey` or `publicKeys`.
         If `verifyCommit` is enabled, then `fetchGit` cannot use a local repository with uncommitted changes.
-        Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches).
+        Requires the [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches).
 
       - `publicKey`
 
         The public key against which `rev` is verified if `verifyCommit` is enabled.
-        Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches).
+        Requires the [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches).
 
       - `keytype` (default: `"ssh-ed25519"`)
 
@@ -644,7 +697,7 @@ static RegisterPrimOp primop_fetchGit({
         - `"ssh-ed25519"`
         - `"ssh-ed25519-sk"`
         - `"ssh-rsa"`
-        Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches).
+        Requires the [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches).
 
       - `publicKeys`
 
@@ -658,7 +711,7 @@ static RegisterPrimOp primop_fetchGit({
         }
         ```
 
-        Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches).
+        Requires the [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches).
 
 
       Here are some examples of how to use `fetchGit`.

src/libexpr/primops/fromTOML.cc

@@ -1,10 +1,10 @@
 #include "primops.hh"
 #include "eval-inline.hh"
 
-#include "../../toml11/toml.hpp"
-
 #include <sstream>
 
+#include <toml.hpp>
+
 namespace nix {
 
 static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, Value & val)

src/libexpr/primops/meson.build

@@ -0,0 +1,12 @@
+generated_headers += gen_header.process(
+  'derivation.nix',
+  preserve_path_from: meson.project_source_root(),
+)
+
+sources += files(
+  'context.cc',
+  'fetchClosure.cc',
+  'fetchMercurial.cc',
+  'fetchTree.cc',
+  'fromTOML.cc',
+)

src/libexpr/print-ambiguous.cc

@@ -94,7 +94,7 @@ void printAmbiguous(
         break;
     default:
         printError("Nix evaluator internal error: printAmbiguous: invalid value type");
-        abort();
+        unreachable();
     }
 }
 

src/libexpr/print.cc

@@ -1,5 +1,6 @@
 #include <limits>
 #include <unordered_set>
+#include <sstream>
 
 #include "print.hh"
 #include "ansicolor.hh"
@@ -162,8 +163,8 @@ private:
     EvalState & state;
     PrintOptions options;
     std::optional<ValuesSeen> seen;
-    size_t attrsPrinted = 0;
-    size_t listItemsPrinted = 0;
+    size_t totalAttrsPrinted = 0;
+    size_t totalListItemsPrinted = 0;
     std::string indent;
 
     void increaseIndent()
@@ -271,22 +272,36 @@ private:
 
     void printDerivation(Value & v)
     {
-        NixStringContext context;
-        std::string storePath;
-        if (auto i = v.attrs()->get(state.sDrvPath))
-            storePath = state.store->printStorePath(state.coerceToStorePath(i->pos, *i->value, context, "while evaluating the drvPath of a derivation"));
+        std::optional<StorePath> storePath;
+        if (auto i = v.attrs()->get(state.sDrvPath)) {
+            NixStringContext context;
+            storePath = state.coerceToStorePath(i->pos, *i->value, context, "while evaluating the drvPath of a derivation");
+        }
+
+        /* This unfortunately breaks printing nested values because of
+           how the pretty printer is used (when pretting printing and warning
+           to same terminal / std stream). */
+#if 0
+        if (storePath && !storePath->isDerivation())
+            warn(
+                "drvPath attribute '%s' is not a valid store path to a derivation, this value not work properly",
+                state.store->printStorePath(*storePath));
+#endif
 
         if (options.ansiColors)
             output << ANSI_GREEN;
         output << "«derivation";
-        if (!storePath.empty()) {
-            output << " " << storePath;
+        if (storePath) {
+            output << " " << state.store->printStorePath(*storePath);
         }
         output << "»";
         if (options.ansiColors)
             output << ANSI_NORMAL;
     }
 
+    /**
+     * @note This may force items.
+     */
     bool shouldPrettyPrintAttrs(AttrVec & v)
     {
         if (!options.shouldPrettyPrint() || v.empty()) {
@@ -303,6 +318,9 @@ private:
             return true;
         }
 
+        // It is ok to force the item(s) here, because they will be printed anyway.
+        state.forceValue(*item, item->determinePos(noPos));
+
         // Pretty-print single-item attrsets only if they contain nested
         // structures.
         auto itemType = item->type();
@@ -333,11 +351,13 @@ private:
 
             auto prettyPrint = shouldPrettyPrintAttrs(sorted);
 
+            size_t currentAttrsPrinted = 0;
+
             for (auto & i : sorted) {
                 printSpace(prettyPrint);
 
-                if (attrsPrinted >= options.maxAttrs) {
-                    printElided(sorted.size() - attrsPrinted, "attribute", "attributes");
+                if (totalAttrsPrinted >= options.maxAttrs) {
+                    printElided(sorted.size() - currentAttrsPrinted, "attribute", "attributes");
                     break;
                 }
 
@@ -345,7 +365,8 @@ private:
                 output << " = ";
                 print(*i.second, depth + 1);
                 output << ";";
-                attrsPrinted++;
+                totalAttrsPrinted++;
+                currentAttrsPrinted++;
             }
 
             decreaseIndent();
@@ -356,6 +377,9 @@ private:
         }
     }
 
+    /**
+     * @note This may force items.
+     */
     bool shouldPrettyPrintList(std::span<Value * const> list)
     {
         if (!options.shouldPrettyPrint() || list.empty()) {
@@ -372,6 +396,9 @@ private:
             return true;
         }
 
+        // It is ok to force the item(s) here, because they will be printed anyway.
+        state.forceValue(*item, item->determinePos(noPos));
+
         // Pretty-print single-item lists only if they contain nested
         // structures.
         auto itemType = item->type();
@@ -390,11 +417,14 @@ private:
             output << "[";
             auto listItems = v.listItems();
             auto prettyPrint = shouldPrettyPrintList(listItems);
+
+            size_t currentListItemsPrinted = 0;
+
             for (auto elem : listItems) {
                 printSpace(prettyPrint);
 
-                if (listItemsPrinted >= options.maxListItems) {
-                    printElided(listItems.size() - listItemsPrinted, "item", "items");
+                if (totalListItemsPrinted >= options.maxListItems) {
+                    printElided(listItems.size() - currentListItemsPrinted, "item", "items");
                     break;
                 }
 
@@ -403,7 +433,8 @@ private:
                 } else {
                     printNullptr();
                 }
-                listItemsPrinted++;
+                totalListItemsPrinted++;
+                currentListItemsPrinted++;
             }
 
             decreaseIndent();
@@ -444,7 +475,7 @@ private:
             else
                 output << "primop";
         } else {
-            abort();
+            unreachable();
         }
 
         output << "»";
@@ -473,7 +504,7 @@ private:
             if (options.ansiColors)
                     output << ANSI_NORMAL;
         } else {
-            abort();
+            unreachable();
         }
     }
 
@@ -576,8 +607,8 @@ public:
 
     void print(Value & v)
     {
-        attrsPrinted = 0;
-        listItemsPrinted = 0;
+        totalAttrsPrinted = 0;
+        totalListItemsPrinted = 0;
         indent.clear();
 
         if (options.trackRepeated) {

src/libexpr/symbol-table.hh

@@ -7,6 +7,7 @@
 
 #include "types.hh"
 #include "chunked-vector.hh"
+#include "error.hh"
 
 namespace nix {
 
@@ -30,9 +31,9 @@ public:
         return *s == s2;
     }
 
-    operator const std::string & () const
+    const char * c_str() const
     {
-        return *s;
+        return s->c_str();
     }
 
     operator const std::string_view () const
@@ -41,6 +42,11 @@ public:
     }
 
     friend std::ostream & operator <<(std::ostream & os, const SymbolStr & symbol);
+
+    bool empty() const
+    {
+        return s->empty();
+    }
 };
 
 /**
@@ -62,9 +68,10 @@ public:
 
     explicit operator bool() const { return id > 0; }
 
-    bool operator<(const Symbol other) const { return id < other.id; }
+    auto operator<=>(const Symbol other) const { return id <=> other.id; }
     bool operator==(const Symbol other) const { return id == other.id; }
-    bool operator!=(const Symbol other) const { return id != other.id; }
+
+    friend class std::hash<Symbol>;
 };
 
 /**
@@ -109,7 +116,7 @@ public:
     SymbolStr operator[](Symbol s) const
     {
         if (s.id == 0 || s.id > store.size())
-            abort();
+            unreachable();
         return SymbolStr(store[s.id - 1]);
     }
 
@@ -128,3 +135,12 @@ public:
 };
 
 }
+
+template<>
+struct std::hash<nix::Symbol>
+{
+    std::size_t operator()(const nix::Symbol & s) const noexcept
+    {
+        return std::hash<decltype(s.id)>{}(s.id);
+    }
+};

src/libexpr/value-to-json.cc

@@ -22,7 +22,7 @@ json printValueAsJSON(EvalState & state, bool strict,
     switch (v.type()) {
 
         case nInt:
-            out = v.integer();
+            out = v.integer().value;
             break;
 
         case nBool:
@@ -58,7 +58,7 @@ json printValueAsJSON(EvalState & state, bool strict,
                 out = json::object();
                 for (auto & a : v.attrs()->lexicographicOrder(state.symbols)) {
                     try {
-                        out[state.symbols[a->name]] = printValueAsJSON(state, strict, *a->value, a->pos, context, copyToStore);
+                        out.emplace(state.symbols[a->name], printValueAsJSON(state, strict, *a->value, a->pos, context, copyToStore));
                     } catch (Error & e) {
                         e.addTrace(state.positions[a->pos],
                             HintFmt("while evaluating attribute '%1%'", state.symbols[a->name]));

src/libexpr/value-to-xml.cc

@@ -9,7 +9,7 @@
 namespace nix {
 
 
-static XMLAttrs singletonAttrs(const std::string & name, const std::string & value)
+static XMLAttrs singletonAttrs(const std::string & name, std::string_view value)
 {
     XMLAttrs attrs;
     attrs[name] = value;

src/libexpr/value.hh

@@ -2,14 +2,13 @@
 ///@file
 
 #include <cassert>
-#include <climits>
 #include <span>
 
 #include "symbol-table.hh"
 #include "value/context.hh"
-#include "input-accessor.hh"
 #include "source-path.hh"
 #include "print-options.hh"
+#include "checked-arithmetic.hh"
 
 #if HAVE_BOEHMGC
 #include <gc/gc_allocator.h>
@@ -23,6 +22,7 @@ class BindingsBuilder;
 
 
 typedef enum {
+    tUninitialized = 0,
     tInt = 1,
     tBool,
     tString,
@@ -74,8 +74,8 @@ class EvalState;
 class XMLWriter;
 class Printer;
 
-typedef int64_t NixInt;
-typedef double NixFloat;
+using NixInt = checked::Checked<int64_t>;
+using NixFloat = double;
 
 /**
  * External values must descend from ExternalValueBase, so that
@@ -112,7 +112,7 @@ class ExternalValueBase
      * Compare to another value of the same type. Defaults to uncomparable,
      * i.e. always false.
      */
-    virtual bool operator ==(const ExternalValueBase & b) const;
+    virtual bool operator ==(const ExternalValueBase & b) const noexcept;
 
     /**
      * Print the value as JSON. Defaults to unconvertable, i.e. throws an error
@@ -166,7 +166,7 @@ public:
 struct Value
 {
 private:
-    InternalType internalType;
+    InternalType internalType = tUninitialized;
 
     friend std::string showType(const Value & v);
 
@@ -216,7 +216,7 @@ public:
     };
 
     struct Path {
-        InputAccessor * accessor;
+        SourceAccessor * accessor;
         const char * path;
     };
 
@@ -270,6 +270,7 @@ public:
     inline ValueType type(bool invalidIsThunk = false) const
     {
         switch (internalType) {
+            case tUninitialized: break;
             case tInt: return nInt;
             case tBool: return nBool;
             case tString: return nString;
@@ -285,7 +286,7 @@ public:
         if (invalidIsThunk)
             return nThunk;
         else
-            abort();
+            unreachable();
     }
 
     inline void finishValue(InternalType newType, Payload newPayload)
@@ -294,6 +295,21 @@ public:
         internalType = newType;
     }
 
+    /**
+     * A value becomes valid when it is initialized. We don't use this
+     * in the evaluator; only in the bindings, where the slight extra
+     * cost is warranted because of inexperienced callers.
+     */
+    inline bool isValid() const
+    {
+        return internalType != tUninitialized;
+    }
+
+    inline void mkInt(NixInt::Inner n)
+    {
+        mkInt(NixInt{n});
+    }
+
     inline void mkInt(NixInt n)
     {
         finishValue(tInt, { .integer = n });
@@ -315,15 +331,15 @@ public:
 
     void mkStringMove(const char * s, const NixStringContext & context);
 
-    inline void mkString(const Symbol & s)
+    inline void mkString(const SymbolStr & s)
     {
-        mkString(((const std::string &) s).c_str());
+        mkString(s.c_str());
     }
 
     void mkPath(const SourcePath & path);
     void mkPath(std::string_view path);
 
-    inline void mkPath(InputAccessor * accessor, const char * path)
+    inline void mkPath(SourceAccessor * accessor, const char * path)
     {
         finishValue(tPath, { .path = { .accessor = accessor, .path = path } });
     }
@@ -438,7 +454,7 @@ public:
         return std::string_view(payload.string.c_str);
     }
 
-    const char * const c_str() const
+    const char * c_str() const
     {
         assert(internalType == tString);
         return payload.string.c_str;
@@ -484,11 +500,11 @@ void Value::mkBlackhole()
 
 #if HAVE_BOEHMGC
 typedef std::vector<Value *, traceable_allocator<Value *>> ValueVector;
-typedef std::map<Symbol, Value *, std::less<Symbol>, traceable_allocator<std::pair<const Symbol, Value *>>> ValueMap;
+typedef std::unordered_map<Symbol, Value *, std::hash<Symbol>, std::equal_to<Symbol>, traceable_allocator<std::pair<const Symbol, Value *>>> ValueMap;
 typedef std::map<Symbol, ValueVector, std::less<Symbol>, traceable_allocator<std::pair<const Symbol, ValueVector>>> ValueVectorMap;
 #else
 typedef std::vector<Value *> ValueVector;
-typedef std::map<Symbol, Value *> ValueMap;
+typedef std::unordered_map<Symbol, Value *> ValueMap;
 typedef std::map<Symbol, ValueVector> ValueVectorMap;
 #endif
 

src/libfetchers/.version

@@ -0,0 +1 @@
+../../.version

src/libfetchers/attrs.cc

@@ -33,7 +33,7 @@ nlohmann::json attrsToJSON(const Attrs & attrs)
             json[attr.first] = *v;
         } else if (auto v = std::get_if<Explicit<bool>>(&attr.second)) {
             json[attr.first] = v->t;
-        } else abort();
+        } else unreachable();
     }
     return json;
 }
@@ -99,7 +99,7 @@ std::map<std::string, std::string> attrsToQuery(const Attrs & attrs)
             query.insert_or_assign(attr.first, *v);
         } else if (auto v = std::get_if<Explicit<bool>>(&attr.second)) {
             query.insert_or_assign(attr.first, v->t ? "1" : "0");
-        } else abort();
+        } else unreachable();
     }
     return query;
 }

src/libfetchers/build-utils-meson

@@ -0,0 +1 @@
+../../build-utils-meson

src/libfetchers/cache.cc

@@ -11,12 +11,11 @@ namespace nix::fetchers {
 static const char * schema = R"sql(
 
 create table if not exists Cache (
-    input     text not null,
-    info      text not null,
-    path      text not null,
-    immutable integer not null,
+    domain    text not null,
+    key       text not null,
+    value     text not null,
     timestamp integer not null,
-    primary key (input)
+    primary key (domain, key)
 );
 )sql";
 
@@ -28,7 +27,7 @@ struct CacheImpl : Cache
     struct State
     {
         SQLite db;
-        SQLiteStmt add, lookup;
+        SQLiteStmt upsert, lookup;
     };
 
     Sync<State> _state;
@@ -37,137 +36,130 @@ struct CacheImpl : Cache
     {
         auto state(_state.lock());
 
-        auto dbPath = getCacheDir() + "/nix/fetcher-cache-v1.sqlite";
+        auto dbPath = getCacheDir() + "/fetcher-cache-v2.sqlite";
         createDirs(dirOf(dbPath));
 
         state->db = SQLite(dbPath);
         state->db.isCache();
         state->db.exec(schema);
 
-        state->add.create(state->db,
-            "insert or replace into Cache(input, info, path, immutable, timestamp) values (?, ?, ?, ?, ?)");
+        state->upsert.create(state->db,
+            "insert or replace into Cache(domain, key, value, timestamp) values (?, ?, ?, ?)");
 
         state->lookup.create(state->db,
-            "select info, path, immutable, timestamp from Cache where input = ?");
+            "select value, timestamp from Cache where domain = ? and key = ?");
     }
 
     void upsert(
-        const Attrs & inAttrs,
-        const Attrs & infoAttrs) override
+        const Key & key,
+        const Attrs & value) override
     {
-        _state.lock()->add.use()
-            (attrsToJSON(inAttrs).dump())
-            (attrsToJSON(infoAttrs).dump())
-            ("") // no path
-            (false)
+        _state.lock()->upsert.use()
+            (key.first)
+            (attrsToJSON(key.second).dump())
+            (attrsToJSON(value).dump())
             (time(0)).exec();
     }
 
-    std::optional<Attrs> lookup(const Attrs & inAttrs) override
+    std::optional<Attrs> lookup(
+        const Key & key) override
     {
-        if (auto res = lookupExpired(inAttrs))
-            return std::move(res->infoAttrs);
+        if (auto res = lookupExpired(key))
+            return std::move(res->value);
         return {};
     }
 
-    std::optional<Attrs> lookupWithTTL(const Attrs & inAttrs) override
+    std::optional<Attrs> lookupWithTTL(
+        const Key & key) override
     {
-        if (auto res = lookupExpired(inAttrs)) {
+        if (auto res = lookupExpired(key)) {
             if (!res->expired)
-                return std::move(res->infoAttrs);
-            debug("ignoring expired cache entry '%s'",
-                attrsToJSON(inAttrs).dump());
-        }
-        return {};
-    }
-
-    std::optional<Result2> lookupExpired(const Attrs & inAttrs) override
-    {
-        auto state(_state.lock());
-
-        auto inAttrsJSON = attrsToJSON(inAttrs).dump();
-
-        auto stmt(state->lookup.use()(inAttrsJSON));
-        if (!stmt.next()) {
-            debug("did not find cache entry for '%s'", inAttrsJSON);
-            return {};
-        }
-
-        auto infoJSON = stmt.getStr(0);
-        auto locked = stmt.getInt(2) != 0;
-        auto timestamp = stmt.getInt(3);
-
-        debug("using cache entry '%s' -> '%s'", inAttrsJSON, infoJSON);
-
-        return Result2 {
-            .expired = !locked && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)),
-            .infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJSON)),
-        };
-    }
-
-    void add(
-        Store & store,
-        const Attrs & inAttrs,
-        const Attrs & infoAttrs,
-        const StorePath & storePath,
-        bool locked) override
-    {
-        _state.lock()->add.use()
-            (attrsToJSON(inAttrs).dump())
-            (attrsToJSON(infoAttrs).dump())
-            (store.printStorePath(storePath))
-            (locked)
-            (time(0)).exec();
-    }
-
-    std::optional<std::pair<Attrs, StorePath>> lookup(
-        Store & store,
-        const Attrs & inAttrs) override
-    {
-        if (auto res = lookupExpired(store, inAttrs)) {
-            if (!res->expired)
-                return std::make_pair(std::move(res->infoAttrs), std::move(res->storePath));
-            debug("ignoring expired cache entry '%s'",
-                attrsToJSON(inAttrs).dump());
+                return std::move(res->value);
+            debug("ignoring expired cache entry '%s:%s'",
+                key.first, attrsToJSON(key.second).dump());
         }
         return {};
     }
 
     std::optional<Result> lookupExpired(
-        Store & store,
-        const Attrs & inAttrs) override
+        const Key & key) override
     {
         auto state(_state.lock());
 
-        auto inAttrsJSON = attrsToJSON(inAttrs).dump();
+        auto keyJSON = attrsToJSON(key.second).dump();
 
-        auto stmt(state->lookup.use()(inAttrsJSON));
+        auto stmt(state->lookup.use()(key.first)(keyJSON));
         if (!stmt.next()) {
-            debug("did not find cache entry for '%s'", inAttrsJSON);
+            debug("did not find cache entry for '%s:%s'", key.first, keyJSON);
             return {};
         }
 
-        auto infoJSON = stmt.getStr(0);
-        auto storePath = store.parseStorePath(stmt.getStr(1));
-        auto locked = stmt.getInt(2) != 0;
-        auto timestamp = stmt.getInt(3);
+        auto valueJSON = stmt.getStr(0);
+        auto timestamp = stmt.getInt(1);
 
-        store.addTempRoot(storePath);
-        if (!store.isValidPath(storePath)) {
-            // FIXME: we could try to substitute 'storePath'.
-            debug("ignoring disappeared cache entry '%s'", inAttrsJSON);
-            return {};
-        }
-
-        debug("using cache entry '%s' -> '%s', '%s'",
-            inAttrsJSON, infoJSON, store.printStorePath(storePath));
+        debug("using cache entry '%s:%s' -> '%s'", key.first, keyJSON, valueJSON);
 
         return Result {
-            .expired = !locked && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)),
-            .infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJSON)),
-            .storePath = std::move(storePath)
+            .expired = settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0),
+            .value = jsonToAttrs(nlohmann::json::parse(valueJSON)),
         };
     }
+
+    void upsert(
+        Key key,
+        Store & store,
+        Attrs value,
+        const StorePath & storePath) override
+    {
+        /* Add the store prefix to the cache key to handle multiple
+           store prefixes. */
+        key.second.insert_or_assign("store", store.storeDir);
+
+        value.insert_or_assign("storePath", (std::string) storePath.to_string());
+
+        upsert(key, value);
+    }
+
+    std::optional<ResultWithStorePath> lookupStorePath(
+        Key key,
+        Store & store) override
+    {
+        key.second.insert_or_assign("store", store.storeDir);
+
+        auto res = lookupExpired(key);
+        if (!res) return std::nullopt;
+
+        auto storePathS = getStrAttr(res->value, "storePath");
+        res->value.erase("storePath");
+
+        ResultWithStorePath res2(*res, StorePath(storePathS));
+
+        store.addTempRoot(res2.storePath);
+        if (!store.isValidPath(res2.storePath)) {
+            // FIXME: we could try to substitute 'storePath'.
+            debug("ignoring disappeared cache entry '%s:%s' -> '%s'",
+                key.first,
+                attrsToJSON(key.second).dump(),
+                store.printStorePath(res2.storePath));
+            return std::nullopt;
+        }
+
+        debug("using cache entry '%s:%s' -> '%s', '%s'",
+            key.first,
+            attrsToJSON(key.second).dump(),
+            attrsToJSON(res2.value).dump(),
+            store.printStorePath(res2.storePath));
+
+        return res2;
+    }
+
+    std::optional<ResultWithStorePath> lookupStorePathWithTTL(
+        Key key,
+        Store & store) override
+    {
+        auto res = lookupStorePath(std::move(key), store);
+        return res && !res->expired ? res : std::nullopt;
+    }
 };
 
 ref<Cache> getCache()

src/libfetchers/cache.hh

@@ -15,61 +15,80 @@ struct Cache
     virtual ~Cache() { }
 
     /**
-     * Add a value to the cache. The cache is an arbitrary mapping of
-     * Attrs to Attrs.
+     * A domain is a partition of the key/value cache for a particular
+     * purpose, e.g. git revision to revcount.
+     */
+    using Domain = std::string_view;
+
+    /**
+     * A cache key is a domain and an arbitrary set of attributes.
+     */
+    using Key = std::pair<Domain, Attrs>;
+
+    /**
+     * Add a key/value pair to the cache.
      */
     virtual void upsert(
-        const Attrs & inAttrs,
-        const Attrs & infoAttrs) = 0;
+        const Key & key,
+        const Attrs & value) = 0;
 
     /**
      * Look up a key with infinite TTL.
      */
     virtual std::optional<Attrs> lookup(
-        const Attrs & inAttrs) = 0;
+        const Key & key) = 0;
 
     /**
      * Look up a key. Return nothing if its TTL has exceeded
      * `settings.tarballTTL`.
      */
     virtual std::optional<Attrs> lookupWithTTL(
-        const Attrs & inAttrs) = 0;
+        const Key & key) = 0;
 
-    struct Result2
+    struct Result
     {
         bool expired = false;
-        Attrs infoAttrs;
+        Attrs value;
     };
 
     /**
      * Look up a key. Return a bool denoting whether its TTL has
      * exceeded `settings.tarballTTL`.
      */
-    virtual std::optional<Result2> lookupExpired(
-        const Attrs & inAttrs) = 0;
+    virtual std::optional<Result> lookupExpired(
+        const Key & key) = 0;
 
-    /* Old cache for things that have a store path. */
-    virtual void add(
+    /**
+     * Insert a cache entry that has a store path associated with
+     * it. Such cache entries are always considered stale if the
+     * associated store path is invalid.
+     */
+    virtual void upsert(
+        Key key,
         Store & store,
-        const Attrs & inAttrs,
-        const Attrs & infoAttrs,
-        const StorePath & storePath,
-        bool locked) = 0;
+        Attrs value,
+        const StorePath & storePath) = 0;
 
-    virtual std::optional<std::pair<Attrs, StorePath>> lookup(
-        Store & store,
-        const Attrs & inAttrs) = 0;
-
-    struct Result
+    struct ResultWithStorePath : Result
     {
-        bool expired = false;
-        Attrs infoAttrs;
         StorePath storePath;
     };
 
-    virtual std::optional<Result> lookupExpired(
-        Store & store,
-        const Attrs & inAttrs) = 0;
+    /**
+     * Look up a store path in the cache. The returned store path will
+     * be valid, but it may be expired.
+     */
+    virtual std::optional<ResultWithStorePath> lookupStorePath(
+        Key key,
+        Store & store) = 0;
+
+    /**
+     * Look up a store path in the cache. Return nothing if its TTL
+     * has exceeded `settings.tarballTTL`.
+     */
+    virtual std::optional<ResultWithStorePath> lookupStorePathWithTTL(
+        Key key,
+        Store & store) = 0;
 };
 
 ref<Cache> getCache();

src/libfetchers/fetch-settings.cc

@@ -1,13 +1,9 @@
 #include "fetch-settings.hh"
 
-namespace nix {
+namespace nix::fetchers {
 
-FetchSettings::FetchSettings()
+Settings::Settings()
 {
 }
 
-FetchSettings fetchSettings;
-
-static GlobalConfig::Register rFetchSettings(&fetchSettings);
-
 }