Merge branch 'master' into lfs

src/build-remote/build-remote.cc

@@ -11,11 +11,13 @@
 
 #include "machines.hh"
 #include "shared.hh"
+#include "plugin.hh"
 #include "pathlocks.hh"
 #include "globals.hh"
 #include "serialise.hh"
 #include "build-result.hh"
 #include "store-api.hh"
+#include "strings.hh"
 #include "derivations.hh"
 #include "local-store.hh"
 #include "legacy.hh"
@@ -37,7 +39,7 @@ static std::string currentLoad;
 
 static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot)
 {
-    return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true);
+    return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri.render()), slot), true);
 }
 
 static bool allSupportedLocally(Store & store, const std::set<std::string>& requiredFeatures) {
@@ -135,7 +137,7 @@ static int main_build_remote(int argc, char * * argv)
                 Machine * bestMachine = nullptr;
                 uint64_t bestLoad = 0;
                 for (auto & m : machines) {
-                    debug("considering building on remote machine '%s'", m.storeUri);
+                    debug("considering building on remote machine '%s'", m.storeUri.render());
 
                     if (m.enabled &&
                         m.systemSupported(neededSystem) &&
@@ -202,7 +204,7 @@ static int main_build_remote(int argc, char * * argv)
                         else
                             drvstr = "<unknown>";
 
-                        auto error = HintFmt(errorText);
+                        auto error = HintFmt::fromFormatString(errorText);
                         error
                             % drvstr
                             % neededSystem
@@ -232,17 +234,16 @@ static int main_build_remote(int argc, char * * argv)
                 lock = -1;
 
                 try {
+                    storeUri = bestMachine->storeUri.render();
 
-                    Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri));
+                    Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", storeUri));
 
                     sshStore = bestMachine->openStore();
                     sshStore->connect();
-                    storeUri = bestMachine->storeUri;
-
                 } catch (std::exception & e) {
                     auto msg = chomp(drainFD(5, false));
                     printError("cannot build on '%s': %s%s",
-                        bestMachine->storeUri, e.what(),
+                        storeUri, e.what(),
                         msg.empty() ? "" : ": " + msg);
                     bestMachine->enabled = false;
                     continue;
@@ -262,7 +263,20 @@ connected:
         auto inputs = readStrings<PathSet>(source);
         auto wantedOutputs = readStrings<StringSet>(source);
 
-        AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true);
+        AutoCloseFD uploadLock;
+        {
+            auto setUpdateLock = [&](auto && fileName){
+                uploadLock = openLockFile(currentLoad + "/" + escapeUri(fileName) + ".upload-lock", true);
+            };
+            try {
+                setUpdateLock(storeUri);
+            } catch (SysError & e) {
+                if (e.errNo != ENAMETOOLONG) throw;
+                // Try again hashing the store URL so we have a shorter path
+                auto h = hashString(HashAlgorithm::MD5, storeUri);
+                setUpdateLock(h.to_string(HashFormat::Base64, false));
+            }
+        }
 
         {
             Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri));

src/external-api-docs/.gitignore

@@ -0,0 +1,3 @@
+/doxygen.cfg
+/html
+/latex

src/external-api-docs/.version

@@ -0,0 +1 @@
+../../.version

src/external-api-docs/README.md

@@ -0,0 +1,121 @@
+# Getting started
+
+> **Warning** These bindings are **experimental**, which means they can change
+> at any time or be removed outright; nevertheless the plan is to provide a
+> stable external C API to the Nix language and the Nix store.
+
+The language library allows evaluating Nix expressions and interacting with Nix
+language values. The Nix store API is still rudimentary, and only allows
+initialising and connecting to a store for the Nix language evaluator to
+interact with.
+
+Currently there are two ways to interface with the Nix language evaluator
+programmatically:
+
+1. Embedding the evaluator
+2. Writing language plug-ins
+
+Embedding means you link the Nix C libraries in your program and use them from
+there. Adding a plug-in means you make a library that gets loaded by the Nix
+language evaluator, specified through a configuration option.
+
+Many of the components and mechanisms involved are not yet documented, therefore
+please refer to the [Nix source code](https://github.com/NixOS/nix/) for
+details. Additions to in-code documentation and the reference manual are highly
+appreciated.
+
+The following examples, for simplicity, don't include error handling. See the
+[Handling errors](@ref errors) section for more information.
+
+# Embedding the Nix Evaluator{#nix_evaluator_example}
+
+In this example we programmatically start the Nix language evaluator with a
+dummy store (that has no store paths and cannot be written to), and evaluate the
+Nix expression `builtins.nixVersion`.
+
+**main.c:**
+
+```C
+#include <nix_api_util.h>
+#include <nix_api_expr.h>
+#include <nix_api_value.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+// NOTE: This example lacks all error handling. Production code must check for
+// errors, as some return values will be undefined.
+
+void my_get_string_cb(const char * start, unsigned int n, void * user_data)
+{
+    *((char **) user_data) = strdup(start);
+}
+
+int main()
+{
+    nix_libexpr_init(NULL);
+
+    Store * store = nix_store_open(NULL, "dummy://", NULL);
+    EvalState * state = nix_state_create(NULL, NULL, store); // empty search path (NIX_PATH)
+    Value * value = nix_alloc_value(NULL, state);
+
+    nix_expr_eval_from_string(NULL, state, "builtins.nixVersion", ".", value);
+    nix_value_force(NULL, state, value);
+
+    char * version;
+    nix_get_string(NULL, value, my_get_string_cb, &version);
+    printf("Nix version: %s\n", version);
+
+    free(version);
+    nix_gc_decref(NULL, value);
+    nix_state_free(state);
+    nix_store_free(store);
+    return 0;
+}
+```
+
+**Usage:**
+
+```ShellSession
+$ gcc main.c $(pkg-config nix-expr-c --libs --cflags) -o main
+$ ./main
+Nix version: 2.17
+```
+
+# Writing a Nix language plug-in
+
+In this example we add a custom primitive operation (_primop_) to `builtins`. It
+will increment the argument if it is an integer and throw an error otherwise.
+
+**plugin.c:**
+
+```C
+#include <nix_api_util.h>
+#include <nix_api_expr.h>
+#include <nix_api_value.h>
+
+void increment(void* user_data, nix_c_context* ctx, EvalState* state, Value** args, Value* v) {
+    nix_value_force(NULL, state, args[0]);
+    if (nix_get_type(NULL, args[0]) == NIX_TYPE_INT) {
+      nix_init_int(NULL, v, nix_get_int(NULL, args[0]) + 1);
+    } else {
+      nix_set_err_msg(ctx, NIX_ERR_UNKNOWN, "First argument should be an integer.");
+    }
+}
+
+void nix_plugin_entry() {
+  const char* args[] = {"n", NULL};
+  PrimOp *p = nix_alloc_primop(NULL, increment, 1, "increment", args, "Example custom built-in function: increments an integer", NULL);
+  nix_register_primop(NULL, p);
+  nix_gc_decref(NULL, p);
+}
+```
+
+**Usage:**
+
+```ShellSession
+$ gcc plugin.c $(pkg-config nix-expr-c --libs --cflags) -shared -o plugin.so
+$ nix --plugin-files ./plugin.so repl
+nix-repl> builtins.increment 1
+2
+```

src/external-api-docs/doxygen.cfg.in

@@ -0,0 +1,58 @@
+# Doxyfile 1.9.5
+
+# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by
+# double-quotes, unless you are using Doxywizard) that should identify the
+# project for which the documentation is generated. This name is used in the
+# title of most generated pages and in a few other places.
+# The default value is: My Project.
+
+PROJECT_NAME           = "Nix"
+
+# The PROJECT_NUMBER tag can be used to enter a project or revision number. This
+# could be handy for archiving the generated documentation or if some version
+# control system is used.
+
+PROJECT_NUMBER         = @PROJECT_NUMBER@
+
+OUTPUT_DIRECTORY       = @OUTPUT_DIRECTORY@
+
+# Using the PROJECT_BRIEF tag one can provide an optional one line description
+# for a project that appears at the top of each page and should give viewer a
+# quick idea about the purpose of the project. Keep the description short.
+
+PROJECT_BRIEF          = "Nix, the purely functional package manager: C API (experimental)"
+
+# If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output.
+# The default value is: YES.
+
+GENERATE_LATEX         = NO
+
+# The INPUT tag is used to specify the files and/or directories that contain
+# documented source files. You may enter file names like myfile.cpp or
+# directories like /usr/src/myproject. Separate the files or directories with
+# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING
+# Note: If this tag is empty the current directory is searched.
+
+# FIXME Make this list more maintainable somehow. We could maybe generate this
+# in the Makefile, but we would need to change how `.in` files are preprocessed
+# so they can expand variables despite configure variables.
+
+INPUT                  = \
+  @src@/src/libutil-c \
+  @src@/src/libexpr-c \
+  @src@/src/libstore-c \
+  @src@/doc/external-api/README.md
+
+FILE_PATTERNS = nix_api_*.h *.md
+
+# The INCLUDE_PATH tag can be used to specify one or more directories that
+# contain include files that are not input files but should be processed by the
+# preprocessor. Note that the INCLUDE_PATH is not recursive, so the setting of
+# RECURSIVE has no effect here.
+# This tag requires that the tag SEARCH_INCLUDES is set to YES.
+
+EXCLUDE_PATTERNS = *_internal.h
+GENERATE_TREEVIEW = YES
+OPTIMIZE_OUTPUT_FOR_C  = YES
+
+USE_MDFILE_AS_MAINPAGE = doc/external-api/README.md

src/external-api-docs/meson.build

@@ -0,0 +1,31 @@
+project('nix-external-api-docs',
+  version : files('.version'),
+  meson_version : '>= 1.1',
+  license : 'LGPL-2.1-or-later',
+)
+
+fs = import('fs')
+
+doxygen_cfg = configure_file(
+  input : 'doxygen.cfg.in',
+  output : 'doxygen.cfg',
+  configuration : {
+    'PROJECT_NUMBER': meson.project_version(),
+    'OUTPUT_DIRECTORY' : meson.current_build_dir(),
+    'src' : fs.parent(fs.parent(meson.project_source_root())),
+  },
+)
+
+doxygen = find_program('doxygen', native : true, required : true)
+
+custom_target(
+  'external-api-docs',
+  command : [ doxygen , doxygen_cfg ],
+  input : [
+    doxygen_cfg,
+  ],
+  output : 'html',
+  install : true,
+  install_dir : get_option('datadir') / 'doc/nix/external-api',
+  build_always_stale : true,
+)

src/external-api-docs/package.nix

@@ -0,0 +1,59 @@
+{ lib
+, mkMesonDerivation
+
+, meson
+, ninja
+, doxygen
+
+# Configuration Options
+
+, version
+}:
+
+let
+  inherit (lib) fileset;
+in
+
+mkMesonDerivation (finalAttrs: {
+  pname = "nix-external-api-docs";
+  inherit version;
+
+  workDir = ./.;
+  fileset =
+    let
+      cpp = fileset.fileFilter (file: file.hasExt "cc" || file.hasExt "h");
+    in
+    fileset.unions [
+      ./.version
+      ../../.version
+      ./meson.build
+      ./doxygen.cfg.in
+      ./README.md
+      # Source is not compiled, but still must be available for Doxygen
+      # to gather comments.
+      (cpp ../libexpr-c)
+      (cpp ../libstore-c)
+      (cpp ../libutil-c)
+    ];
+
+  nativeBuildInputs = [
+    meson
+    ninja
+    doxygen
+  ];
+
+  preConfigure =
+    ''
+      chmod u+w ./.version
+      echo ${finalAttrs.version} > ./.version
+    '';
+
+  postInstall = ''
+    mkdir -p ''${!outputDoc}/nix-support
+    echo "doc external-api-docs $out/share/doc/nix/external-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products
+  '';
+
+  meta = {
+    platforms = lib.platforms.all;
+  };
+})

src/internal-api-docs/.gitignore

@@ -0,0 +1,3 @@
+/doxygen.cfg
+/html
+/latex

src/internal-api-docs/.version

@@ -0,0 +1 @@
+../../.version

src/internal-api-docs/doxygen.cfg.in

@@ -0,0 +1,99 @@
+# Doxyfile 1.9.5
+
+# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by
+# double-quotes, unless you are using Doxywizard) that should identify the
+# project for which the documentation is generated. This name is used in the
+# title of most generated pages and in a few other places.
+# The default value is: My Project.
+
+PROJECT_NAME           = "Nix"
+
+# The PROJECT_NUMBER tag can be used to enter a project or revision number. This
+# could be handy for archiving the generated documentation or if some version
+# control system is used.
+
+PROJECT_NUMBER         = @PROJECT_NUMBER@
+
+OUTPUT_DIRECTORY       = @OUTPUT_DIRECTORY@
+
+# Using the PROJECT_BRIEF tag one can provide an optional one line description
+# for a project that appears at the top of each page and should give viewer a
+# quick idea about the purpose of the project. Keep the description short.
+
+PROJECT_BRIEF          = "Nix, the purely functional package manager; unstable internal interfaces"
+
+# If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output.
+# The default value is: YES.
+
+GENERATE_LATEX         = NO
+
+# The INPUT tag is used to specify the files and/or directories that contain
+# documented source files. You may enter file names like myfile.cpp or
+# directories like /usr/src/myproject. Separate the files or directories with
+# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING
+# Note: If this tag is empty the current directory is searched.
+
+# FIXME Make this list more maintainable somehow. We could maybe generate this
+# in the Makefile, but we would need to change how `.in` files are preprocessed
+# so they can expand variables despite configure variables.
+
+INPUT                  = \
+  @src@/libcmd \
+  @src@/libexpr \
+  @src@/libexpr/flake \
+  @src@/nix-expr-tests \
+  @src@/nix-expr-tests/value \
+  @src@/nix-expr-test-support/test \
+  @src@/nix-expr-test-support/test/value \
+  @src@/libexpr/value \
+  @src@/libfetchers \
+  @src@/libmain \
+  @src@/libstore \
+  @src@/libstore/build \
+  @src@/libstore/builtins \
+  @src@/nix-store-tests \
+  @src@/nix-store-test-support/test \
+  @src@/libutil \
+  @src@/nix-util-tests \
+  @src@/nix-util-test-support/test \
+  @src@/nix \
+  @src@/nix-env \
+  @src@/nix-store
+
+# If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names
+# in the source code. If set to NO, only conditional compilation will be
+# performed. Macro expansion can be done in a controlled way by setting
+# EXPAND_ONLY_PREDEF to YES.
+# The default value is: NO.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+MACRO_EXPANSION        = YES
+
+# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then
+# the macro expansion is limited to the macros specified with the PREDEFINED and
+# EXPAND_AS_DEFINED tags.
+# The default value is: NO.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+EXPAND_ONLY_PREDEF     = YES
+
+# The INCLUDE_PATH tag can be used to specify one or more directories that
+# contain include files that are not input files but should be processed by the
+# preprocessor. Note that the INCLUDE_PATH is not recursive, so the setting of
+# RECURSIVE has no effect here.
+# This tag requires that the tag SEARCH_INCLUDES is set to YES.
+
+INCLUDE_PATH           =
+
+# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this
+# tag can be used to specify a list of macro names that should be expanded. The
+# macro definition that is found in the sources will be used. Use the PREDEFINED
+# tag if you want to use a different macro definition that overrules the
+# definition found in the source code.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+EXPAND_AS_DEFINED      = \
+  DECLARE_COMMON_SERIALISER \
+  DECLARE_WORKER_SERIALISER \
+  DECLARE_SERVE_SERIALISER \
+  LENGTH_PREFIXED_PROTO_HELPER

src/internal-api-docs/meson.build

@@ -0,0 +1,31 @@
+project('nix-internal-api-docs',
+  version : files('.version'),
+  meson_version : '>= 1.1',
+  license : 'LGPL-2.1-or-later',
+)
+
+fs = import('fs')
+
+doxygen_cfg = configure_file(
+  input : 'doxygen.cfg.in',
+  output : 'doxygen.cfg',
+  configuration : {
+    'PROJECT_NUMBER': meson.project_version(),
+    'OUTPUT_DIRECTORY' : meson.current_build_dir(),
+    'src' : fs.parent(fs.parent(meson.project_source_root())) / 'src',
+  },
+)
+
+doxygen = find_program('doxygen', native : true, required : true)
+
+custom_target(
+  'internal-api-docs',
+  command : [ doxygen , doxygen_cfg ],
+  input : [
+    doxygen_cfg,
+  ],
+  output : 'html',
+  install : true,
+  install_dir : get_option('datadir') / 'doc/nix/internal-api',
+  build_always_stale : true,
+)

src/internal-api-docs/package.nix

@@ -0,0 +1,54 @@
+{ lib
+, mkMesonDerivation
+
+, meson
+, ninja
+, doxygen
+
+# Configuration Options
+
+, version
+}:
+
+let
+  inherit (lib) fileset;
+in
+
+mkMesonDerivation (finalAttrs: {
+  pname = "nix-internal-api-docs";
+  inherit version;
+
+  workDir = ./.;
+  fileset = let
+    cpp = fileset.fileFilter (file: file.hasExt "cc" || file.hasExt "hh");
+  in fileset.unions [
+    ./.version
+    ../../.version
+    ./meson.build
+    ./doxygen.cfg.in
+    # Source is not compiled, but still must be available for Doxygen
+    # to gather comments.
+    (cpp ../.)
+  ];
+
+  nativeBuildInputs = [
+    meson
+    ninja
+    doxygen
+  ];
+
+  preConfigure =
+    ''
+      chmod u+w ./.version
+      echo ${finalAttrs.version} > ./.version
+    '';
+
+  postInstall = ''
+    mkdir -p ''${!outputDoc}/nix-support
+    echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products
+  '';
+
+  meta = {
+    platforms = lib.platforms.all;
+  };
+})

src/libcmd/.version

@@ -0,0 +1 @@
+../../.version

src/libcmd/build-utils-meson

@@ -0,0 +1 @@
+../../build-utils-meson

src/libcmd/built-path.cc

@@ -1,6 +1,7 @@
 #include "built-path.hh"
 #include "derivations.hh"
 #include "store-api.hh"
+#include "comparator.hh"
 
 #include <nlohmann/json.hpp>
 
@@ -8,30 +9,24 @@
 
 namespace nix {
 
-#define CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, COMPARATOR) \
-    bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \
-    { \
-        const MY_TYPE* me = this; \
-        auto fields1 = std::tie(*me->drvPath, me->FIELD); \
-        me = &other; \
-        auto fields2 = std::tie(*me->drvPath, me->FIELD); \
-        return fields1 COMPARATOR fields2; \
-    }
-#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \
-    CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, ==) \
-    CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \
-    CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <)
+// Custom implementation to avoid `ref` ptr equality
+GENERATE_CMP_EXT(
+    ,
+    std::strong_ordering,
+    SingleBuiltPathBuilt,
+    *me->drvPath,
+    me->output);
 
-#define FIELD_TYPE std::pair<std::string, StorePath>
-CMP(SingleBuiltPath, SingleBuiltPathBuilt, output)
-#undef FIELD_TYPE
+// Custom implementation to avoid `ref` ptr equality
 
-#define FIELD_TYPE std::map<std::string, StorePath>
-CMP(SingleBuiltPath, BuiltPathBuilt, outputs)
-#undef FIELD_TYPE
-
-#undef CMP
-#undef CMP_ONE
+// TODO no `GENERATE_CMP_EXT` because no `std::set::operator<=>` on
+// Darwin, per header.
+GENERATE_EQUAL(
+    ,
+    BuiltPathBuilt ::,
+    BuiltPathBuilt,
+    *me->drvPath,
+    me->outputs);
 
 StorePath SingleBuiltPath::outPath() const
 {

src/libcmd/built-path.hh

@@ -18,7 +18,8 @@ struct SingleBuiltPathBuilt {
     static SingleBuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view);
     nlohmann::json toJSON(const StoreDirConfig & store) const;
 
-    DECLARE_CMP(SingleBuiltPathBuilt);
+    bool operator ==(const SingleBuiltPathBuilt &) const noexcept;
+    std::strong_ordering operator <=>(const SingleBuiltPathBuilt &) const noexcept;
 };
 
 using _SingleBuiltPathRaw = std::variant<
@@ -33,6 +34,9 @@ struct SingleBuiltPath : _SingleBuiltPathRaw {
     using Opaque = DerivedPathOpaque;
     using Built = SingleBuiltPathBuilt;
 
+    bool operator == (const SingleBuiltPath &) const = default;
+    auto operator <=> (const SingleBuiltPath &) const = default;
+
     inline const Raw & raw() const {
         return static_cast<const Raw &>(*this);
     }
@@ -59,11 +63,13 @@ struct BuiltPathBuilt {
     ref<SingleBuiltPath> drvPath;
     std::map<std::string, StorePath> outputs;
 
+    bool operator == (const BuiltPathBuilt &) const noexcept;
+    // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet.
+    //std::strong_ordering operator <=> (const BuiltPathBuilt &) const noexcept;
+
     std::string to_string(const StoreDirConfig & store) const;
     static BuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view);
     nlohmann::json toJSON(const StoreDirConfig & store) const;
-
-    DECLARE_CMP(BuiltPathBuilt);
 };
 
 using _BuiltPathRaw = std::variant<
@@ -82,6 +88,10 @@ struct BuiltPath : _BuiltPathRaw {
     using Opaque = DerivedPathOpaque;
     using Built = BuiltPathBuilt;
 
+    bool operator == (const BuiltPath &) const = default;
+    // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet.
+    //auto operator <=> (const BuiltPath &) const = default;
+
     inline const Raw & raw() const {
         return static_cast<const Raw &>(*this);
     }

src/libcmd/command.cc

@@ -1,3 +1,5 @@
+#include <nlohmann/json.hpp>
+
 #include "command.hh"
 #include "markdown.hh"
 #include "store-api.hh"
@@ -6,8 +8,7 @@
 #include "nixexpr.hh"
 #include "profiles.hh"
 #include "repl.hh"
-
-#include <nlohmann/json.hpp>
+#include "strings.hh"
 
 extern char * * environ __attribute__((weak));
 
@@ -127,12 +128,12 @@ ref<EvalState> EvalCommand::getEvalState()
     if (!evalState) {
         evalState =
             #if HAVE_BOEHMGC
-            std::allocate_shared<EvalState>(traceable_allocator<EvalState>(),
-                searchPath, getEvalStore(), getStore())
+            std::allocate_shared<EvalState>(
+                traceable_allocator<EvalState>(),
             #else
             std::make_shared<EvalState>(
-                searchPath, getEvalStore(), getStore())
             #endif
+                lookupPath, getEvalStore(), fetchSettings, evalSettings, getStore())
             ;
 
         evalState->repair = repair;
@@ -148,7 +149,7 @@ MixOperateOnOptions::MixOperateOnOptions()
 {
     addFlag({
         .longName = "derivation",
-        .description = "Operate on the [store derivation](../../glossary.md#gloss-store-derivation) rather than its outputs.",
+        .description = "Operate on the [store derivation](@docroot@/glossary.md#gloss-store-derivation) rather than its outputs.",
         .category = installablesCategory,
         .handler = {&operateOn, OperateOn::Derivation},
     });

src/libcmd/common-eval-args.cc

@@ -1,18 +1,57 @@
+#include "fetch-settings.hh"
 #include "eval-settings.hh"
 #include "common-eval-args.hh"
 #include "shared.hh"
+#include "config-global.hh"
 #include "filetransfer.hh"
 #include "eval.hh"
 #include "fetchers.hh"
 #include "registry.hh"
 #include "flake/flakeref.hh"
+#include "flake/settings.hh"
 #include "store-api.hh"
 #include "command.hh"
 #include "tarball.hh"
 #include "fetch-to-store.hh"
+#include "compatibility-settings.hh"
+#include "eval-settings.hh"
 
 namespace nix {
 
+fetchers::Settings fetchSettings;
+
+static GlobalConfig::Register rFetchSettings(&fetchSettings);
+
+EvalSettings evalSettings {
+    settings.readOnlyMode,
+    {
+        {
+            "flake",
+            [](ref<Store> store, std::string_view rest) {
+                experimentalFeatureSettings.require(Xp::Flakes);
+                // FIXME `parseFlakeRef` should take a `std::string_view`.
+                auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false);
+                debug("fetching flake search path element '%s''", rest);
+                auto storePath = flakeRef.resolve(store).fetchTree(store).first;
+                return store->toRealPath(storePath);
+            },
+        },
+    },
+};
+
+static GlobalConfig::Register rEvalSettings(&evalSettings);
+
+
+flake::Settings flakeSettings;
+
+static GlobalConfig::Register rFlakeSettings(&flakeSettings);
+
+
+CompatibilitySettings compatibilitySettings {};
+
+static GlobalConfig::Register rCompatibilitySettings(&compatibilitySettings);
+
+
 MixEvalArgs::MixEvalArgs()
 {
     addFlag({
@@ -20,7 +59,7 @@ MixEvalArgs::MixEvalArgs()
         .description = "Pass the value *expr* as the argument *name* to Nix functions.",
         .category = category,
         .labels = {"name", "expr"},
-        .handler = {[&](std::string name, std::string expr) { autoArgs[name] = 'E' + expr; }}
+        .handler = {[&](std::string name, std::string expr) { autoArgs.insert_or_assign(name, AutoArg{AutoArgExpr{expr}}); }}
     });
 
     addFlag({
@@ -28,87 +67,40 @@ MixEvalArgs::MixEvalArgs()
         .description = "Pass the string *string* as the argument *name* to Nix functions.",
         .category = category,
         .labels = {"name", "string"},
-        .handler = {[&](std::string name, std::string s) { autoArgs[name] = 'S' + s; }},
+        .handler = {[&](std::string name, std::string s) { autoArgs.insert_or_assign(name, AutoArg{AutoArgString{s}}); }},
+    });
+
+    addFlag({
+        .longName = "arg-from-file",
+        .description = "Pass the contents of file *path* as the argument *name* to Nix functions.",
+        .category = category,
+        .labels = {"name", "path"},
+        .handler = {[&](std::string name, std::string path) { autoArgs.insert_or_assign(name, AutoArg{AutoArgFile{path}}); }},
+        .completer = completePath
+    });
+
+    addFlag({
+        .longName = "arg-from-stdin",
+        .description = "Pass the contents of stdin as the argument *name* to Nix functions.",
+        .category = category,
+        .labels = {"name"},
+        .handler = {[&](std::string name) { autoArgs.insert_or_assign(name, AutoArg{AutoArgStdin{}}); }},
     });
 
     addFlag({
         .longName = "include",
         .shortName = 'I',
         .description = R"(
-  Add *path* to the Nix search path. The Nix search path is
-  initialized from the colon-separated [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) environment
-  variable, and is used to look up the location of Nix expressions using [paths](@docroot@/language/values.md#type-path) enclosed in angle
-  brackets (i.e., `<nixpkgs>`).
+  Add *path* to search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md)
 
-  For instance, passing
+  This option may be given multiple times.
 
-  ```
-  -I /home/eelco/Dev
-  -I /etc/nixos
-  ```
-
-  will cause Nix to look for paths relative to `/home/eelco/Dev` and
-  `/etc/nixos`, in that order. This is equivalent to setting the
-  `NIX_PATH` environment variable to
-
-  ```
-  /home/eelco/Dev:/etc/nixos
-  ```
-
-  It is also possible to match paths against a prefix. For example,
-  passing
-
-  ```
-  -I nixpkgs=/home/eelco/Dev/nixpkgs-branch
-  -I /etc/nixos
-  ```
-
-  will cause Nix to search for `<nixpkgs/path>` in
-  `/home/eelco/Dev/nixpkgs-branch/path` and `/etc/nixos/nixpkgs/path`.
-
-  If a path in the Nix search path starts with `http://` or `https://`,
-  it is interpreted as the URL of a tarball that will be downloaded and
-  unpacked to a temporary location. The tarball must consist of a single
-  top-level directory. For example, passing
-
-  ```
-  -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/master.tar.gz
-  ```
-
-  tells Nix to download and use the current contents of the `master`
-  branch in the `nixpkgs` repository.
-
-  The URLs of the tarballs from the official `nixos.org` channels
-  (see [the manual page for `nix-channel`](../nix-channel.md)) can be
-  abbreviated as `channel:<channel-name>`.  For instance, the
-  following two flags are equivalent:
-
-  ```
-  -I nixpkgs=channel:nixos-21.05
-  -I nixpkgs=https://nixos.org/channels/nixos-21.05/nixexprs.tar.xz
-  ```
-
-  You can also fetch source trees using [flake URLs](./nix3-flake.md#url-like-syntax) and add them to the
-  search path. For instance,
-
-  ```
-  -I nixpkgs=flake:nixpkgs
-  ```
-
-  specifies that the prefix `nixpkgs` shall refer to the source tree
-  downloaded from the `nixpkgs` entry in the flake registry. Similarly,
-
-  ```
-  -I nixpkgs=flake:github:NixOS/nixpkgs/nixos-22.05
-  ```
-
-  makes `<nixpkgs>` refer to a particular branch of the
-  `NixOS/nixpkgs` repository on GitHub.
+  Paths added through `-I` take precedence over the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path) and the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH).
   )",
         .category = category,
         .labels = {"path"},
         .handler = {[&](std::string s) {
-            searchPath.elements.emplace_back(SearchPath::Elem::parse(s));
+            lookupPath.elements.emplace_back(LookupPath::Elem::parse(s));
         }}
     });
 
@@ -127,8 +119,8 @@ MixEvalArgs::MixEvalArgs()
         .category = category,
         .labels = {"original-ref", "resolved-ref"},
         .handler = {[&](std::string _from, std::string _to) {
-            auto from = parseFlakeRef(_from, absPath("."));
-            auto to = parseFlakeRef(_to, absPath("."));
+            auto from = parseFlakeRef(fetchSettings, _from, absPath("."));
+            auto to = parseFlakeRef(fetchSettings, _to, absPath("."));
             fetchers::Attrs extraAttrs;
             if (to.subdir != "") extraAttrs["dir"] = to.subdir;
             fetchers::overrideRegistry(from.input, to.input, extraAttrs);
@@ -154,13 +146,23 @@ MixEvalArgs::MixEvalArgs()
 Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
 {
     auto res = state.buildBindings(autoArgs.size());
-    for (auto & i : autoArgs) {
+    for (auto & [name, arg] : autoArgs) {
         auto v = state.allocValue();
-        if (i.second[0] == 'E')
-            state.mkThunk_(*v, state.parseExprFromString(i.second.substr(1), state.rootPath(".")));
-        else
-            v->mkString(((std::string_view) i.second).substr(1));
-        res.insert(state.symbols.create(i.first), v);
+        std::visit(overloaded {
+            [&](const AutoArgExpr & arg) {
+                state.mkThunk_(*v, state.parseExprFromString(arg.expr, compatibilitySettings.nixShellShebangArgumentsRelativeToScript ? state.rootPath(absPath(getCommandBaseDir())) : state.rootPath(".")));
+            },
+            [&](const AutoArgString & arg) {
+                v->mkString(arg.s);
+            },
+            [&](const AutoArgFile & arg) {
+                v->mkString(readFile(arg.path.string()));
+            },
+            [&](const AutoArgStdin & arg) {
+                v->mkString(readFile(STDIN_FILENO));
+            }
+        }, arg);
+        res.insert(state.symbols.create(name), v);
     }
     return res.finish();
 }
@@ -176,7 +178,7 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas
 
     else if (hasPrefix(s, "flake:")) {
         experimentalFeatureSettings.require(Xp::Flakes);
-        auto flakeRef = parseFlakeRef(std::string(s.substr(6)), {}, true, false);
+        auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false);
         auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first;
         return state.rootPath(CanonPath(state.store->toRealPath(storePath)));
     }

src/libcmd/common-eval-args.hh

@@ -6,13 +6,42 @@
 #include "common-args.hh"
 #include "search-path.hh"
 
+#include <filesystem>
+
 namespace nix {
 
 class Store;
+
+namespace fetchers { struct Settings; }
+
 class EvalState;
+struct EvalSettings;
+struct CompatibilitySettings;
 class Bindings;
 struct SourcePath;
 
+namespace flake { struct Settings; }
+
+/**
+ * @todo Get rid of global setttings variables
+ */
+extern fetchers::Settings fetchSettings;
+
+/**
+ * @todo Get rid of global setttings variables
+ */
+extern EvalSettings evalSettings;
+
+/**
+ * @todo Get rid of global setttings variables
+ */
+extern flake::Settings flakeSettings;
+
+/**
+ * Settings that control behaviors that have changed since Nix 2.3.
+ */
+extern CompatibilitySettings compatibilitySettings;
+
 struct MixEvalArgs : virtual Args, virtual MixRepair
 {
     static constexpr auto category = "Common evaluation options";
@@ -21,14 +50,24 @@ struct MixEvalArgs : virtual Args, virtual MixRepair
 
     Bindings * getAutoArgs(EvalState & state);
 
-    SearchPath searchPath;
+    LookupPath lookupPath;
 
     std::optional<std::string> evalStoreUrl;
 
 private:
-    std::map<std::string, std::string> autoArgs;
+    struct AutoArgExpr { std::string expr; };
+    struct AutoArgString { std::string s; };
+    struct AutoArgFile { std::filesystem::path path; };
+    struct AutoArgStdin { };
+
+    using AutoArg = std::variant<AutoArgExpr, AutoArgString, AutoArgFile, AutoArgStdin>;
+
+    std::map<std::string, AutoArg> autoArgs;
 };
 
+/**
+ * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory)
+ */
 SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * baseDir = nullptr);
 
 }

src/libcmd/compatibility-settings.hh

@@ -0,0 +1,36 @@
+#pragma once
+#include "config.hh"
+
+namespace nix {
+struct CompatibilitySettings : public Config
+{
+
+    CompatibilitySettings() = default;
+
+    // Added in Nix 2.24, July 2024.
+    Setting<bool> nixShellAlwaysLooksForShellNix{this, true, "nix-shell-always-looks-for-shell-nix", R"(
+        Before Nix 2.24, [`nix-shell`](@docroot@/command-ref/nix-shell.md) would only look at `shell.nix` if it was in the working directory - when no file was specified.
+
+        Since Nix 2.24, `nix-shell` always looks for a `shell.nix`, whether that's in the working directory, or in a directory that was passed as an argument.
+
+        You may set this to `false` to temporarily revert to the behavior of Nix 2.23 and older.
+
+        Using this setting is not recommended.
+        It will be deprecated and removed.
+    )"};
+
+    // Added in Nix 2.24, July 2024.
+    Setting<bool> nixShellShebangArgumentsRelativeToScript{
+        this, true, "nix-shell-shebang-arguments-relative-to-script", R"(
+        Before Nix 2.24, relative file path expressions in arguments in a `nix-shell` shebang were resolved relative to the working directory.
+
+        Since Nix 2.24, `nix-shell` resolves these paths in a manner that is relative to the [base directory](@docroot@/glossary.md#gloss-base-directory), defined as the script's directory.
+
+        You may set this to `false` to temporarily revert to the behavior of Nix 2.23 and older.
+
+        Using this setting is not recommended.
+        It will be deprecated and removed.
+    )"};
+};
+
+};

src/libcmd/installable-attr-path.cc

@@ -75,6 +75,8 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths()
                 std::set<std::string> outputsToInstall;
                 for (auto & output : packageInfo.queryOutputs(false, true))
                     outputsToInstall.insert(output.first);
+                if (outputsToInstall.empty())
+                    outputsToInstall.insert("out");
                 return OutputsSpec::Names { std::move(outputsToInstall) };
             },
             [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec {

src/libcmd/installable-flake.cc

@@ -43,20 +43,6 @@ std::vector<std::string> InstallableFlake::getActualAttrPaths()
     return res;
 }
 
-Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake)
-{
-    auto vFlake = state.allocValue();
-
-    callFlake(state, lockedFlake, *vFlake);
-
-    auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs"));
-    assert(aOutputs);
-
-    state.forceValue(*aOutputs->value, aOutputs->value->determinePos(noPos));
-
-    return aOutputs->value;
-}
-
 static std::string showAttrPaths(const std::vector<std::string> & paths)
 {
     std::string s;
@@ -106,9 +92,14 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths()
             fmt("while evaluating the flake output attribute '%s'", attrPath)))
         {
             return { *derivedPathWithInfo };
+        } else {
+            throw Error(
+                "expected flake output attribute '%s' to be a derivation or path but found %s: %s",
+                attrPath,
+                showType(v),
+                ValuePrinter(*this->state, v, errorPrintOptions)
+            );
         }
-        else
-            throw Error("flake output attribute '%s' is not a derivation or path", attrPath);
     }
 
     auto drvPath = attr->forceDerivation();
@@ -205,7 +196,8 @@ std::shared_ptr<flake::LockedFlake> InstallableFlake::getLockedFlake() const
         flake::LockFlags lockFlagsApplyConfig = lockFlags;
         // FIXME why this side effect?
         lockFlagsApplyConfig.applyNixConfig = true;
-        _lockedFlake = std::make_shared<flake::LockedFlake>(lockFlake(*state, flakeRef, lockFlagsApplyConfig));
+        _lockedFlake = std::make_shared<flake::LockedFlake>(lockFlake(
+            flakeSettings, *state, flakeRef, lockFlagsApplyConfig));
     }
     return _lockedFlake;
 }

src/libcmd/installable-flake.hh

@@ -1,6 +1,7 @@
 #pragma once
 ///@file
 
+#include "common-eval-args.hh"
 #include "installable-value.hh"
 
 namespace nix {
@@ -52,8 +53,6 @@ struct InstallableFlake : InstallableValue
 
     std::vector<std::string> getActualAttrPaths();
 
-    Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake);
-
     DerivedPathsWithInfo toDerivedPaths() override;
 
     std::pair<Value *, PosIdx> toValue(EvalState & state) override;
@@ -80,7 +79,7 @@ struct InstallableFlake : InstallableValue
  */
 static inline FlakeRef defaultNixpkgsFlakeRef()
 {
-    return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}});
+    return FlakeRef::fromAttrs(fetchSettings, {{"type","indirect"}, {"id", "nixpkgs"}});
 }
 
 ref<eval_cache::EvalCache> openEvalCache(

src/libcmd/installable-value.hh

@@ -66,7 +66,7 @@ struct ExtraPathInfoValue : ExtraPathInfo
 };
 
 /**
- * An Installable which corresponds a Nix langauge value, in addition to
+ * An Installable which corresponds a Nix language value, in addition to
  * a collection of \ref DerivedPath "derived paths".
  */
 struct InstallableValue : Installable

src/libcmd/installables.cc

@@ -27,6 +27,8 @@
 
 #include <nlohmann/json.hpp>
 
+#include "strings-inline.hh"
+
 namespace nix {
 
 void completeFlakeInputPath(
@@ -129,7 +131,7 @@ MixFlakeOptions::MixFlakeOptions()
             lockFlags.writeLockFile = false;
             lockFlags.inputOverrides.insert_or_assign(
                 flake::parseInputPath(inputPath),
-                parseFlakeRef(flakeRef, absPath(getCommandBaseDir()), true));
+                parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir()), true));
         }},
         .completer = {[&](AddCompletions & completions, size_t n, std::string_view prefix) {
             if (n == 0) {
@@ -146,7 +148,7 @@ MixFlakeOptions::MixFlakeOptions()
         .category = category,
         .labels = {"flake-lock-path"},
         .handler = {[&](std::string lockFilePath) {
-            lockFlags.referenceLockFilePath = lockFilePath;
+            lockFlags.referenceLockFilePath = {getFSSourceAccessor(), CanonPath(absPath(lockFilePath))};
         }},
         .completer = completePath
     });
@@ -170,14 +172,15 @@ MixFlakeOptions::MixFlakeOptions()
         .handler = {[&](std::string flakeRef) {
             auto evalState = getEvalState();
             auto flake = flake::lockFlake(
+                flakeSettings,
                 *evalState,
-                parseFlakeRef(flakeRef, absPath(getCommandBaseDir())),
+                parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir())),
                 { .writeLockFile = false });
             for (auto & [inputName, input] : flake.lockFile.root->inputs) {
                 auto input2 = flake.lockFile.findInput({inputName}); // resolve 'follows' nodes
                 if (auto input3 = std::dynamic_pointer_cast<const flake::LockedNode>(input2)) {
                     overrideRegistry(
-                        fetchers::Input::fromAttrs({{"type","indirect"}, {"id", inputName}}),
+                        fetchers::Input::fromAttrs(fetchSettings, {{"type","indirect"}, {"id", inputName}}),
                         input3->lockedRef.input,
                         {});
                 }
@@ -288,11 +291,11 @@ void SourceExprCommand::completeInstallable(AddCompletions & completions, std::s
             state->autoCallFunction(*autoArgs, v1, v2);
 
             if (v2.type() == nAttrs) {
-                for (auto & i : *v2.attrs) {
-                    std::string name = state->symbols[i.name];
+                for (auto & i : *v2.attrs()) {
+                    std::string_view name = state->symbols[i.name];
                     if (name.find(searchWord) == 0) {
                         if (prefix_ == "")
-                            completions.add(name);
+                            completions.add(std::string(name));
                         else
                             completions.add(prefix_ + "." + name);
                     }
@@ -338,10 +341,11 @@ void completeFlakeRefWithFragment(
             auto flakeRefS = std::string(prefix.substr(0, hash));
 
             // TODO: ideally this would use the command base directory instead of assuming ".".
-            auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath("."));
+            auto flakeRef = parseFlakeRef(fetchSettings, expandTilde(flakeRefS), absPath("."));
 
             auto evalCache = openEvalCache(*evalState,
-                std::make_shared<flake::LockedFlake>(lockFlake(*evalState, flakeRef, lockFlags)));
+                std::make_shared<flake::LockedFlake>(lockFlake(
+                    flakeSettings, *evalState, flakeRef, lockFlags)));
 
             auto root = evalCache->getRoot();
 
@@ -372,6 +376,7 @@ void completeFlakeRefWithFragment(
                         auto attrPath2 = (*attr)->getAttrPath(attr2);
                         /* Strip the attrpath prefix. */
                         attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size());
+                        // FIXME: handle names with dots
                         completions.add(flakeRefS + "#" + prefixRoot + concatStringsSep(".", evalState->symbols.resolve(attrPath2)));
                     }
                 }
@@ -403,7 +408,7 @@ void completeFlakeRef(AddCompletions & completions, ref<Store> store, std::strin
     Args::completeDir(completions, 0, prefix);
 
     /* Look for registry entries that match the prefix. */
-    for (auto & registry : fetchers::getRegistries(store)) {
+    for (auto & registry : fetchers::getRegistries(fetchSettings, store)) {
         for (auto & entry : registry->entries) {
             auto from = entry.from.to_string();
             if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) {
@@ -442,13 +447,10 @@ ref<eval_cache::EvalCache> openEvalCache(
     EvalState & state,
     std::shared_ptr<flake::LockedFlake> lockedFlake)
 {
-    auto fingerprint = lockedFlake->getFingerprint();
-    return make_ref<nix::eval_cache::EvalCache>(
-        evalSettings.useEvalCache && evalSettings.pureEval
-            ? std::optional { std::cref(fingerprint) }
-            : std::nullopt,
-        state,
-        [&state, lockedFlake]()
+    auto fingerprint = evalSettings.useEvalCache && evalSettings.pureEval
+        ? lockedFlake->getFingerprint(state.store)
+        : std::nullopt;
+    auto rootLoader = [&state, lockedFlake]()
         {
             /* For testing whether the evaluation cache is
                complete. */
@@ -460,11 +462,21 @@ ref<eval_cache::EvalCache> openEvalCache(
 
             state.forceAttrs(*vFlake, noPos, "while parsing cached flake data");
 
-            auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs"));
+            auto aOutputs = vFlake->attrs()->get(state.symbols.create("outputs"));
             assert(aOutputs);
 
             return aOutputs->value;
-        });
+        };
+
+    if (fingerprint) {
+        auto search = state.evalCaches.find(fingerprint.value());
+        if (search == state.evalCaches.end()) {
+            search = state.evalCaches.emplace(fingerprint.value(), make_ref<nix::eval_cache::EvalCache>(fingerprint, state, rootLoader)).first;
+        }
+        return search->second;
+    } else {
+        return make_ref<nix::eval_cache::EvalCache>(std::nullopt, state, rootLoader);
+    }
 }
 
 Installables SourceExprCommand::parseInstallables(
@@ -527,7 +539,8 @@ Installables SourceExprCommand::parseInstallables(
             }
 
             try {
-                auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, absPath(getCommandBaseDir()));
+                auto [flakeRef, fragment] = parseFlakeRefWithFragment(
+                    fetchSettings, std::string { prefix }, absPath(getCommandBaseDir()));
                 result.push_back(make_ref<InstallableFlake>(
                         this,
                         getEvalState(),
@@ -594,6 +607,37 @@ std::vector<BuiltPathWithResult> Installable::build(
     return res;
 }
 
+static void throwBuildErrors(
+    std::vector<KeyedBuildResult> & buildResults,
+    const Store & store)
+{
+    std::vector<KeyedBuildResult> failed;
+    for (auto & buildResult : buildResults) {
+        if (!buildResult.success()) {
+            failed.push_back(buildResult);
+        }
+    }
+
+    auto failedResult = failed.begin();
+    if (failedResult != failed.end()) {
+        if (failed.size() == 1) {
+            failedResult->rethrow();
+        } else {
+            StringSet failedPaths;
+            for (; failedResult != failed.end(); failedResult++) {
+                if (!failedResult->errorMsg.empty()) {
+                    logError(ErrorInfo{
+                        .level = lvlError,
+                        .msg = failedResult->errorMsg,
+                    });
+                }
+                failedPaths.insert(failedResult->path.to_string(store));
+            }
+            throw Error("build of %s failed", concatStringsSep(", ", quoteStrings(failedPaths)));
+        }
+    }
+}
+
 std::vector<std::pair<ref<Installable>, BuiltPathWithResult>> Installable::build2(
     ref<Store> evalStore,
     ref<Store> store,
@@ -655,10 +699,9 @@ std::vector<std::pair<ref<Installable>, BuiltPathWithResult>> Installable::build
         if (settings.printMissing)
             printMissing(store, pathsToBuild, lvlInfo);
 
-        for (auto & buildResult : store->buildPathsWithResults(pathsToBuild, bMode, evalStore)) {
-            if (!buildResult.success())
-                buildResult.rethrow();
-
+        auto buildResults = store->buildPathsWithResults(pathsToBuild, bMode, evalStore);
+        throwBuildErrors(buildResults, *store);
+        for (auto & buildResult : buildResults) {
             for (auto & aux : backmap[buildResult.path]) {
                 std::visit(overloaded {
                     [&](const DerivedPath::Built & bfd) {
@@ -814,6 +857,7 @@ std::vector<FlakeRef> RawInstallablesCommand::getFlakeRefsForCompletion()
     std::vector<FlakeRef> res;
     for (auto i : rawInstallables)
         res.push_back(parseFlakeRefWithFragment(
+            fetchSettings,
             expandTilde(i),
             absPath(getCommandBaseDir())).first);
     return res;
@@ -836,6 +880,7 @@ std::vector<FlakeRef> InstallableCommand::getFlakeRefsForCompletion()
 {
     return {
         parseFlakeRefWithFragment(
+            fetchSettings,
             expandTilde(_installable),
             absPath(getCommandBaseDir())).first
     };

src/libcmd/local.mk

@@ -6,10 +6,10 @@ libcmd_DIR := $(d)
 
 libcmd_SOURCES := $(wildcard $(d)/*.cc)
 
-libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers
+libcmd_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) $(INCLUDE_libflake) $(INCLUDE_libmain)
 
 libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) $(THREAD_LDFLAGS)
 
-libcmd_LIBS = libstore libutil libexpr libmain libfetchers
+libcmd_LIBS = libutil libstore libfetchers libflake libexpr libmain
 
 $(eval $(call install-file-in, $(buildprefix)$(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644))

src/libcmd/markdown.cc

@@ -1,21 +1,23 @@
 #include "markdown.hh"
-#include "util.hh"
+#include "environment-variables.hh"
+#include "error.hh"
 #include "finally.hh"
 #include "terminal.hh"
 
-#include <sys/queue.h>
 #if HAVE_LOWDOWN
-#include <lowdown.h>
+#  include <sys/queue.h>
+#  include <lowdown.h>
 #endif
 
 namespace nix {
 
-std::string renderMarkdownToTerminal(std::string_view markdown)
-{
 #if HAVE_LOWDOWN
+static std::string doRenderMarkdownToTerminal(std::string_view markdown)
+{
     int windowWidth = getWindowSize().second;
 
-    struct lowdown_opts opts {
+    struct lowdown_opts opts
+    {
         .type = LOWDOWN_TERM,
         .maxdepth = 20,
         .cols = (size_t) std::max(windowWidth - 5, 60),
@@ -50,10 +52,22 @@ std::string renderMarkdownToTerminal(std::string_view markdown)
     if (!rndr_res)
         throw Error("allocation error while rendering Markdown");
 
-    return filterANSIEscapes(std::string(buf->data, buf->size), !shouldANSI());
-#else
-    return std::string(markdown);
-#endif
+    return filterANSIEscapes(std::string(buf->data, buf->size), !isTTY());
 }
 
+std::string renderMarkdownToTerminal(std::string_view markdown)
+{
+    if (auto e = getEnv("_NIX_TEST_RAW_MARKDOWN"); e && *e == "1")
+        return std::string(markdown);
+    else
+        return doRenderMarkdownToTerminal(markdown);
 }
+
+#else
+std::string renderMarkdownToTerminal(std::string_view markdown)
+{
+    return std::string(markdown);
+}
+#endif
+
+} // namespace nix

src/libcmd/markdown.hh

@@ -1,10 +1,17 @@
 #pragma once
 ///@file
 
-#include "types.hh"
+#include <string_view>
 
 namespace nix {
 
+/**
+ * Render the given Markdown text to the terminal.
+ *
+ * If Nix is compiled without Markdown support, this function will return the input text as-is.
+ *
+ * The renderer takes into account the terminal width, and wraps text accordingly.
+ */
 std::string renderMarkdownToTerminal(std::string_view markdown);
 
 }

src/libcmd/meson.build

@@ -0,0 +1,130 @@
+project('nix-cmd', 'cpp',
+  version : files('.version'),
+  default_options : [
+    'cpp_std=c++2a',
+    # TODO(Qyriad): increase the warning level
+    'warning_level=1',
+    'debug=true',
+    'optimization=2',
+    'errorlogs=true', # Please print logs for tests that fail
+  ],
+  meson_version : '>= 1.1',
+  license : 'LGPL-2.1-or-later',
+)
+
+cxx = meson.get_compiler('cpp')
+
+subdir('build-utils-meson/deps-lists')
+
+configdata = configuration_data()
+
+deps_private_maybe_subproject = [
+]
+deps_public_maybe_subproject = [
+  dependency('nix-util'),
+  dependency('nix-store'),
+  dependency('nix-fetchers'),
+  dependency('nix-expr'),
+  dependency('nix-flake'),
+  dependency('nix-main'),
+]
+subdir('build-utils-meson/subprojects')
+
+subdir('build-utils-meson/threads')
+
+nlohmann_json = dependency('nlohmann_json', version : '>= 3.9')
+deps_public += nlohmann_json
+
+lowdown = dependency('lowdown', version : '>= 0.9.0', required : get_option('markdown'))
+deps_private += lowdown
+configdata.set('HAVE_LOWDOWN', lowdown.found().to_int())
+
+readline_flavor = get_option('readline-flavor')
+if readline_flavor == 'editline'
+  editline = dependency('libeditline', 'editline', version : '>=1.14')
+  deps_private += editline
+elif readline_flavor == 'readline'
+  readline = dependency('readline')
+  deps_private += readline
+  configdata.set(
+    'USE_READLINE',
+    1,
+    description: 'Use readline instead of editline',
+  )
+else
+  error('illegal editline flavor', readline_flavor)
+endif
+
+config_h = configure_file(
+  configuration : configdata,
+  output : 'config-cmd.hh',
+)
+
+add_project_arguments(
+  # TODO(Qyriad): Yes this is how the autoconf+Make system did it.
+  # It would be nice for our headers to be idempotent instead.
+  '-include', 'config-util.hh',
+  '-include', 'config-store.hh',
+  # '-include', 'config-fetchers.h',
+  '-include', 'config-expr.hh',
+  '-include', 'config-main.hh',
+  '-include', 'config-cmd.hh',
+  language : 'cpp',
+)
+
+subdir('build-utils-meson/diagnostics')
+
+sources = files(
+  'built-path.cc',
+  'command-installable-value.cc',
+  'command.cc',
+  'common-eval-args.cc',
+  'editor-for.cc',
+  'installable-attr-path.cc',
+  'installable-derived-path.cc',
+  'installable-flake.cc',
+  'installable-value.cc',
+  'installables.cc',
+  'legacy.cc',
+  'markdown.cc',
+  'misc-store-flags.cc',
+  'network-proxy.cc',
+  'repl-interacter.cc',
+  'repl.cc',
+)
+
+include_dirs = [include_directories('.')]
+
+headers = [config_h] + files(
+  'built-path.hh',
+  'command-installable-value.hh',
+  'command.hh',
+  'common-eval-args.hh',
+  'compatibility-settings.hh',
+  'editor-for.hh',
+  'installable-attr-path.hh',
+  'installable-derived-path.hh',
+  'installable-flake.hh',
+  'installable-value.hh',
+  'installables.hh',
+  'legacy.hh',
+  'markdown.hh',
+  'misc-store-flags.hh',
+  'network-proxy.hh',
+  'repl-interacter.hh',
+  'repl.hh',
+)
+
+this_library = library(
+  'nixcmd',
+  sources,
+  dependencies : deps_public + deps_private + deps_other,
+  prelink : true, # For C++ static initializers
+  install : true,
+)
+
+install_headers(headers, subdir : 'nix', preserve_path : true)
+
+libraries_private = []
+
+subdir('build-utils-meson/export')

src/libcmd/meson.options

@@ -0,0 +1,15 @@
+# vim: filetype=meson
+
+option(
+  'markdown',
+  type: 'feature',
+  description: 'Enable Markdown rendering in the Nix binary (requires lowdown)',
+)
+
+option(
+  'readline-flavor',
+  type : 'combo',
+  choices : ['editline', 'readline'],
+  value : 'editline',
+  description : 'Which library to use for nice line editing with the Nix language REPL',
+)

src/libcmd/misc-store-flags.cc

@@ -81,9 +81,15 @@ Args::Flag fileIngestionMethod(FileIngestionMethod * method)
     How to compute the hash of the input.
     One of:
 
-    - `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function.
+    - `nar` (the default):
+      Serialises the input as a
+      [Nix Archive](@docroot@/store/file-system-object/content-address.md#serial-nix-archive)
+      and passes that to the hash function.
 
-    - `flat`: Assumes that the input is a single file and directly passes it to the hash function;
+    - `flat`:
+      Assumes that the input is a single file and
+      [directly passes](@docroot@/store/file-system-object/content-address.md#serial-flat)
+      it to the hash function.
         )",
         .labels = {"file-ingestion-method"},
         .handler = {[method](std::string s) {
@@ -101,15 +107,23 @@ Args::Flag contentAddressMethod(ContentAddressMethod * method)
     How to compute the content-address of the store object.
     One of:
 
-    - `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function.
+    - [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive)
+      (the default):
+      Serialises the input as a
+      [Nix Archive](@docroot@/store/file-system-object/content-address.md#serial-nix-archive)
+      and passes that to the hash function.
 
-    - `flat`: Assumes that the input is a single file and directly passes it to the hash function;
+    - [`flat`](@docroot@/store/store-object/content-address.md#method-flat):
+      Assumes that the input is a single file and
+      [directly passes](@docroot@/store/file-system-object/content-address.md#serial-flat)
+      it to the hash function.
 
-    - `text`: Like `flat`, but used for
-      [derivations](@docroot@/glossary.md#store-derivation) serialized in store object and 
+    - [`text`](@docroot@/store/store-object/content-address.md#method-text):
+      Like `flat`, but used for
+      [derivations](@docroot@/glossary.md#store-derivation) serialized in store object and
       [`builtins.toFile`](@docroot@/language/builtins.html#builtins-toFile).
       For advanced use-cases only;
-      for regular usage prefer `nar` and `flat.
+      for regular usage prefer `nar` and `flat`.
         )",
         .labels = {"content-address-method"},
         .handler = {[method](std::string s) {

src/libcmd/network-proxy.cc

@@ -0,0 +1,50 @@
+#include "network-proxy.hh"
+
+#include <algorithm>
+
+#include "environment-variables.hh"
+
+namespace nix {
+
+static const StringSet lowercaseVariables{"http_proxy", "https_proxy", "ftp_proxy", "all_proxy", "no_proxy"};
+
+static StringSet getAllVariables()
+{
+    StringSet variables = lowercaseVariables;
+    for (const auto & variable : lowercaseVariables) {
+        std::string upperVariable;
+        std::transform(
+            variable.begin(), variable.end(), upperVariable.begin(), [](unsigned char c) { return std::toupper(c); });
+        variables.insert(std::move(upperVariable));
+    }
+    return variables;
+}
+
+const StringSet networkProxyVariables = getAllVariables();
+
+static StringSet getExcludingNoProxyVariables()
+{
+    static const StringSet excludeVariables{"no_proxy", "NO_PROXY"};
+    StringSet variables;
+    std::set_difference(
+        networkProxyVariables.begin(),
+        networkProxyVariables.end(),
+        excludeVariables.begin(),
+        excludeVariables.end(),
+        std::inserter(variables, variables.begin()));
+    return variables;
+}
+
+static const StringSet excludingNoProxyVariables = getExcludingNoProxyVariables();
+
+bool haveNetworkProxyConnection()
+{
+    for (const auto & variable : excludingNoProxyVariables) {
+        if (getEnv(variable).has_value()) {
+            return true;
+        }
+    }
+    return false;
+}
+
+}

src/libcmd/network-proxy.hh

@@ -0,0 +1,22 @@
+#pragma once
+///@file
+
+#include "types.hh"
+
+namespace nix {
+
+/**
+ * Environment variables relating to network proxying. These are used by
+ * a few misc commands.
+ *
+ * See the Environment section of https://curl.se/docs/manpage.html for details.
+ */
+extern const StringSet networkProxyVariables;
+
+/**
+ * Heuristically check if there is a proxy connection by checking for defined
+ * proxy variables.
+ */
+bool haveNetworkProxyConnection();
+
+}

src/libcmd/package.nix

@@ -0,0 +1,104 @@
+{ lib
+, stdenv
+, mkMesonDerivation
+, releaseTools
+
+, meson
+, ninja
+, pkg-config
+
+, nix-util
+, nix-store
+, nix-fetchers
+, nix-expr
+, nix-flake
+, nix-main
+, editline
+, readline
+, lowdown
+, nlohmann_json
+
+# Configuration Options
+
+, version
+
+# Whether to enable Markdown rendering in the Nix binary.
+, enableMarkdown ? !stdenv.hostPlatform.isWindows
+
+# Which interactive line editor library to use for Nix's repl.
+#
+# Currently supported choices are:
+#
+# - editline (default)
+# - readline
+, readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline"
+}:
+
+let
+  inherit (lib) fileset;
+in
+
+mkMesonDerivation (finalAttrs: {
+  pname = "nix-cmd";
+  inherit version;
+
+  workDir = ./.;
+  fileset = fileset.unions [
+    ../../build-utils-meson
+    ./build-utils-meson
+    ../../.version
+    ./.version
+    ./meson.build
+    ./meson.options
+    (fileset.fileFilter (file: file.hasExt "cc") ./.)
+    (fileset.fileFilter (file: file.hasExt "hh") ./.)
+  ];
+
+  outputs = [ "out" "dev" ];
+
+  nativeBuildInputs = [
+    meson
+    ninja
+    pkg-config
+  ];
+
+  buildInputs = [
+    ({ inherit editline readline; }.${readlineFlavor})
+  ] ++ lib.optional enableMarkdown lowdown;
+
+  propagatedBuildInputs = [
+    nix-util
+    nix-store
+    nix-fetchers
+    nix-expr
+    nix-flake
+    nix-main
+    nlohmann_json
+  ];
+
+  preConfigure =
+    # "Inline" .version so it's not a symlink, and includes the suffix.
+    # Do the meson utils, without modification.
+    ''
+      chmod u+w ./.version
+      echo ${version} > ../../.version
+    '';
+
+  mesonFlags = [
+    (lib.mesonEnable "markdown" enableMarkdown)
+    (lib.mesonOption "readline-flavor" readlineFlavor)
+  ];
+
+  env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) {
+    LDFLAGS = "-fuse-ld=gold";
+  };
+
+  separateDebugInfo = !stdenv.hostPlatform.isStatic;
+
+  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
+
+  meta = {
+    platforms = lib.platforms.unix ++ lib.platforms.windows;
+  };
+
+})

src/libcmd/repl-interacter.cc

@@ -0,0 +1,206 @@
+#include <cstdio>
+
+#ifdef USE_READLINE
+#include <readline/history.h>
+#include <readline/readline.h>
+#else
+// editline < 1.15.2 don't wrap their API for C++ usage
+// (added in https://github.com/troglobit/editline/commit/91398ceb3427b730995357e9d120539fb9bb7461).
+// This results in linker errors due to to name-mangling of editline C symbols.
+// For compatibility with these versions, we wrap the API here
+// (wrapping multiple times on newer versions is no problem).
+extern "C" {
+#include <editline.h>
+}
+#endif
+
+#include "signals.hh"
+#include "finally.hh"
+#include "repl-interacter.hh"
+#include "file-system.hh"
+#include "repl.hh"
+#include "environment-variables.hh"
+
+namespace nix {
+
+namespace {
+// Used to communicate to NixRepl::getLine whether a signal occurred in ::readline.
+volatile sig_atomic_t g_signal_received = 0;
+
+void sigintHandler(int signo)
+{
+    g_signal_received = signo;
+}
+};
+
+static detail::ReplCompleterMixin * curRepl; // ugly
+
+#ifndef USE_READLINE
+static char * completionCallback(char * s, int * match)
+{
+    auto possible = curRepl->completePrefix(s);
+    if (possible.size() == 1) {
+        *match = 1;
+        auto * res = strdup(possible.begin()->c_str() + strlen(s));
+        if (!res)
+            throw Error("allocation failure");
+        return res;
+    } else if (possible.size() > 1) {
+        auto checkAllHaveSameAt = [&](size_t pos) {
+            auto & first = *possible.begin();
+            for (auto & p : possible) {
+                if (p.size() <= pos || p[pos] != first[pos])
+                    return false;
+            }
+            return true;
+        };
+        size_t start = strlen(s);
+        size_t len = 0;
+        while (checkAllHaveSameAt(start + len))
+            ++len;
+        if (len > 0) {
+            *match = 1;
+            auto * res = strdup(std::string(*possible.begin(), start, len).c_str());
+            if (!res)
+                throw Error("allocation failure");
+            return res;
+        }
+    }
+
+    *match = 0;
+    return nullptr;
+}
+
+static int listPossibleCallback(char * s, char *** avp)
+{
+    auto possible = curRepl->completePrefix(s);
+
+    if (possible.size() > (std::numeric_limits<int>::max() / sizeof(char *)))
+        throw Error("too many completions");
+
+    int ac = 0;
+    char ** vp = nullptr;
+
+    auto check = [&](auto * p) {
+        if (!p) {
+            if (vp) {
+                while (--ac >= 0)
+                    free(vp[ac]);
+                free(vp);
+            }
+            throw Error("allocation failure");
+        }
+        return p;
+    };
+
+    vp = check((char **) malloc(possible.size() * sizeof(char *)));
+
+    for (auto & p : possible)
+        vp[ac++] = check(strdup(p.c_str()));
+
+    *avp = vp;
+
+    return ac;
+}
+#endif
+
+ReadlineLikeInteracter::Guard ReadlineLikeInteracter::init(detail::ReplCompleterMixin * repl)
+{
+    // Allow nix-repl specific settings in .inputrc
+    rl_readline_name = "nix-repl";
+    try {
+        createDirs(dirOf(historyFile));
+    } catch (SystemError & e) {
+        logWarning(e.info());
+    }
+#ifndef USE_READLINE
+    el_hist_size = 1000;
+#endif
+    read_history(historyFile.c_str());
+    auto oldRepl = curRepl;
+    curRepl = repl;
+    Guard restoreRepl([oldRepl] { curRepl = oldRepl; });
+#ifndef USE_READLINE
+    rl_set_complete_func(completionCallback);
+    rl_set_list_possib_func(listPossibleCallback);
+#endif
+    return restoreRepl;
+}
+
+static constexpr const char * promptForType(ReplPromptType promptType)
+{
+    switch (promptType) {
+    case ReplPromptType::ReplPrompt:
+        return "nix-repl> ";
+    case ReplPromptType::ContinuationPrompt:
+        return "          ";
+    }
+    assert(false);
+}
+
+bool ReadlineLikeInteracter::getLine(std::string & input, ReplPromptType promptType)
+{
+#ifndef _WIN32 // TODO use more signals.hh for this
+    struct sigaction act, old;
+    sigset_t savedSignalMask, set;
+
+    auto setupSignals = [&]() {
+        act.sa_handler = sigintHandler;
+        sigfillset(&act.sa_mask);
+        act.sa_flags = 0;
+        if (sigaction(SIGINT, &act, &old))
+            throw SysError("installing handler for SIGINT");
+
+        sigemptyset(&set);
+        sigaddset(&set, SIGINT);
+        if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask))
+            throw SysError("unblocking SIGINT");
+    };
+    auto restoreSignals = [&]() {
+        if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr))
+            throw SysError("restoring signals");
+
+        if (sigaction(SIGINT, &old, 0))
+            throw SysError("restoring handler for SIGINT");
+    };
+
+    setupSignals();
+#endif
+    char * s = readline(promptForType(promptType));
+    Finally doFree([&]() { free(s); });
+#ifndef _WIN32 // TODO use more signals.hh for this
+    restoreSignals();
+#endif
+
+    if (g_signal_received) {
+        g_signal_received = 0;
+        input.clear();
+        return true;
+    }
+
+    // editline doesn't echo the input to the output when non-interactive, unlike readline
+    // this results in a different behavior when running tests. The echoing is
+    // quite useful for reading the test output, so we add it here.
+    if (auto e = getEnv("_NIX_TEST_REPL_ECHO"); s && e && *e == "1")
+    {
+#ifndef USE_READLINE
+        // This is probably not right for multi-line input, but we don't use that
+        // in the characterisation tests, so it's fine.
+        std::cout << promptForType(promptType) << s << std::endl;
+#endif
+    }
+
+    if (!s)
+        return false;
+    input += s;
+    input += '\n';
+
+    return true;
+}
+
+ReadlineLikeInteracter::~ReadlineLikeInteracter()
+{
+    write_history(historyFile.c_str());
+}
+
+};

src/libcmd/repl-interacter.hh

@@ -0,0 +1,48 @@
+#pragma once
+/// @file
+
+#include "finally.hh"
+#include "types.hh"
+#include <functional>
+#include <string>
+
+namespace nix {
+
+namespace detail {
+/** Provides the completion hooks for the repl, without exposing its complete
+ * internals. */
+struct ReplCompleterMixin {
+    virtual StringSet completePrefix(const std::string & prefix) = 0;
+};
+};
+
+enum class ReplPromptType {
+    ReplPrompt,
+    ContinuationPrompt,
+};
+
+class ReplInteracter
+{
+public:
+    using Guard = Finally<std::function<void()>>;
+
+    virtual Guard init(detail::ReplCompleterMixin * repl) = 0;
+    /** Returns a boolean of whether the interacter got EOF */
+    virtual bool getLine(std::string & input, ReplPromptType promptType) = 0;
+    virtual ~ReplInteracter(){};
+};
+
+class ReadlineLikeInteracter : public virtual ReplInteracter
+{
+    std::string historyFile;
+public:
+    ReadlineLikeInteracter(std::string historyFile)
+        : historyFile(historyFile)
+    {
+    }
+    virtual Guard init(detail::ReplCompleterMixin * repl) override;
+    virtual bool getLine(std::string & input, ReplPromptType promptType) override;
+    virtual ~ReadlineLikeInteracter() override;
+};
+
+};

src/libcmd/repl.cc

@@ -1,34 +1,17 @@
 #include <iostream>
 #include <cstdlib>
 #include <cstring>
-#include <climits>
-
-#include <setjmp.h>
-
-#ifdef USE_READLINE
-#include <readline/history.h>
-#include <readline/readline.h>
-#else
-// editline < 1.15.2 don't wrap their API for C++ usage
-// (added in https://github.com/troglobit/editline/commit/91398ceb3427b730995357e9d120539fb9bb7461).
-// This results in linker errors due to to name-mangling of editline C symbols.
-// For compatibility with these versions, we wrap the API here
-// (wrapping multiple times on newer versions is no problem).
-extern "C" {
-#include <editline.h>
-}
-#endif
 
+#include "repl-interacter.hh"
 #include "repl.hh"
 
 #include "ansicolor.hh"
-#include "signals.hh"
 #include "shared.hh"
+#include "config-global.hh"
 #include "eval.hh"
-#include "eval-cache.hh"
-#include "eval-inline.hh"
 #include "eval-settings.hh"
 #include "attr-path.hh"
+#include "signals.hh"
 #include "store-api.hh"
 #include "log-store.hh"
 #include "common-eval-args.hh"
@@ -38,18 +21,21 @@ extern "C" {
 #include "flake/flake.hh"
 #include "flake/lockfile.hh"
 #include "users.hh"
-#include "terminal.hh"
 #include "editor-for.hh"
 #include "finally.hh"
 #include "markdown.hh"
 #include "local-fs-store.hh"
 #include "print.hh"
+#include "ref.hh"
+#include "value.hh"
 
 #if HAVE_BOEHMGC
 #define GC_INCLUDE_NEW
 #include <gc/gc_cpp.h>
 #endif
 
+#include "strings.hh"
+
 namespace nix {
 
 /**
@@ -75,6 +61,7 @@ enum class ProcessLineResult {
 
 struct NixRepl
     : AbstractNixRepl
+    , detail::ReplCompleterMixin
     #if HAVE_BOEHMGC
     , gc
     #endif
@@ -90,17 +77,16 @@ struct NixRepl
     int displ;
     StringSet varNames;
 
-    const Path historyFile;
+    std::unique_ptr<ReplInteracter> interacter;
 
-    NixRepl(const SearchPath & searchPath, nix::ref<Store> store,ref<EvalState> state,
+    NixRepl(const LookupPath & lookupPath, nix::ref<Store> store,ref<EvalState> state,
             std::function<AnnotatedValues()> getValues);
-    virtual ~NixRepl();
+    virtual ~NixRepl() = default;
 
     ReplExitStatus mainLoop() override;
     void initEnv() override;
 
-    StringSet completePrefix(const std::string & prefix);
-    bool getLine(std::string & input, const std::string & prompt);
+    virtual StringSet completePrefix(const std::string & prefix) override;
     StorePath getDerivationPath(Value & v);
     ProcessLineResult processLine(std::string line);
 
@@ -123,7 +109,8 @@ struct NixRepl
             .force = true,
             .derivationPaths = true,
             .maxDepth = maxDepth,
-            .prettyIndent = 2
+            .prettyIndent = 2,
+            .errors = ErrorPrintBehavior::ThrowTopLevel,
         });
     }
 };
@@ -137,111 +124,33 @@ std::string removeWhitespace(std::string s)
 }
 
 
-NixRepl::NixRepl(const SearchPath & searchPath, nix::ref<Store> store, ref<EvalState> state,
+NixRepl::NixRepl(const LookupPath & lookupPath, nix::ref<Store> store, ref<EvalState> state,
             std::function<NixRepl::AnnotatedValues()> getValues)
     : AbstractNixRepl(state)
     , debugTraceIndex(0)
     , getValues(getValues)
     , staticEnv(new StaticEnv(nullptr, state->staticBaseEnv.get()))
-    , historyFile(getDataDir() + "/nix/repl-history")
+    , interacter(make_unique<ReadlineLikeInteracter>(getDataDir() + "/nix/repl-history"))
 {
 }
 
-
-NixRepl::~NixRepl()
-{
-    write_history(historyFile.c_str());
-}
-
 void runNix(Path program, const Strings & args,
     const std::optional<std::string> & input = {})
 {
     auto subprocessEnv = getEnv();
     subprocessEnv["NIX_CONFIG"] = globalConfig.toKeyValue();
-
+    //isInteractive avoid grabling interactive commands
     runProgram2(RunOptions {
         .program = settings.nixBinDir+ "/" + program,
         .args = args,
         .environment = subprocessEnv,
         .input = input,
+        .isInteractive = true,
     });
 
     return;
 }
 
-static NixRepl * curRepl; // ugly
-
-static char * completionCallback(char * s, int *match) {
-  auto possible = curRepl->completePrefix(s);
-  if (possible.size() == 1) {
-    *match = 1;
-    auto *res = strdup(possible.begin()->c_str() + strlen(s));
-    if (!res) throw Error("allocation failure");
-    return res;
-  } else if (possible.size() > 1) {
-    auto checkAllHaveSameAt = [&](size_t pos) {
-      auto &first = *possible.begin();
-      for (auto &p : possible) {
-        if (p.size() <= pos || p[pos] != first[pos])
-          return false;
-      }
-      return true;
-    };
-    size_t start = strlen(s);
-    size_t len = 0;
-    while (checkAllHaveSameAt(start + len)) ++len;
-    if (len > 0) {
-      *match = 1;
-      auto *res = strdup(std::string(*possible.begin(), start, len).c_str());
-      if (!res) throw Error("allocation failure");
-      return res;
-    }
-  }
-
-  *match = 0;
-  return nullptr;
-}
-
-static int listPossibleCallback(char *s, char ***avp) {
-  auto possible = curRepl->completePrefix(s);
-
-  if (possible.size() > (INT_MAX / sizeof(char*)))
-    throw Error("too many completions");
-
-  int ac = 0;
-  char **vp = nullptr;
-
-  auto check = [&](auto *p) {
-    if (!p) {
-      if (vp) {
-        while (--ac >= 0)
-          free(vp[ac]);
-        free(vp);
-      }
-      throw Error("allocation failure");
-    }
-    return p;
-  };
-
-  vp = check((char **)malloc(possible.size() * sizeof(char*)));
-
-  for (auto & p : possible)
-    vp[ac++] = check(strdup(p.c_str()));
-
-  *avp = vp;
-
-  return ac;
-}
-
-namespace {
-    // Used to communicate to NixRepl::getLine whether a signal occurred in ::readline.
-    volatile sig_atomic_t g_signal_received = 0;
-
-    void sigintHandler(int signo) {
-        g_signal_received = signo;
-    }
-}
-
 static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positions, const DebugTrace & dt)
 {
     if (dt.isError)
@@ -281,24 +190,7 @@ ReplExitStatus NixRepl::mainLoop()
 
     loadFiles();
 
-    // Allow nix-repl specific settings in .inputrc
-    rl_readline_name = "nix-repl";
-    try {
-        createDirs(dirOf(historyFile));
-    } catch (SystemError & e) {
-        logWarning(e.info());
-    }
-#ifndef USE_READLINE
-    el_hist_size = 1000;
-#endif
-    read_history(historyFile.c_str());
-    auto oldRepl = curRepl;
-    curRepl = this;
-    Finally restoreRepl([&] { curRepl = oldRepl; });
-#ifndef USE_READLINE
-    rl_set_complete_func(completionCallback);
-    rl_set_list_possib_func(listPossibleCallback);
-#endif
+    auto _guard = interacter->init(static_cast<detail::ReplCompleterMixin *>(this));
 
     std::string input;
 
@@ -307,7 +199,7 @@ ReplExitStatus NixRepl::mainLoop()
         logger->pause();
         // When continuing input from previous lines, don't print a prompt, just align to the same
         // number of chars as the prompt.
-        if (!getLine(input, input.empty() ? "nix-repl> " : "          ")) {
+        if (!interacter->getLine(input, input.empty() ? ReplPromptType::ReplPrompt : ReplPromptType::ContinuationPrompt)) {
             // Ctrl-D should exit the debugger.
             state->debugStop = false;
             logger->cout("");
@@ -325,7 +217,7 @@ ReplExitStatus NixRepl::mainLoop()
                 case ProcessLineResult::PromptAgain:
                     break;
                 default:
-                    abort();
+                    unreachable();
             }
         } catch (ParseError & e) {
             if (e.msg().find("unexpected end of file") != std::string::npos) {
@@ -336,13 +228,7 @@ ReplExitStatus NixRepl::mainLoop()
               printMsg(lvlError, e.msg());
             }
         } catch (EvalError & e) {
-            // in debugger mode, an EvalError should trigger another repl session.
-            // when that session returns the exception will land here.  No need to show it again;
-            // show the error for this repl session instead.
-            if (state->debugRepl && !state->debugTraces.empty())
-                showDebugTrace(std::cout, state->positions, state->debugTraces.front());
-            else
-                printMsg(lvlError, e.msg());
+            printMsg(lvlError, e.msg());
         } catch (Error & e) {
             printMsg(lvlError, e.msg());
         } catch (Interrupted & e) {
@@ -356,51 +242,6 @@ ReplExitStatus NixRepl::mainLoop()
     }
 }
 
-
-bool NixRepl::getLine(std::string & input, const std::string & prompt)
-{
-    struct sigaction act, old;
-    sigset_t savedSignalMask, set;
-
-    auto setupSignals = [&]() {
-        act.sa_handler = sigintHandler;
-        sigfillset(&act.sa_mask);
-        act.sa_flags = 0;
-        if (sigaction(SIGINT, &act, &old))
-            throw SysError("installing handler for SIGINT");
-
-        sigemptyset(&set);
-        sigaddset(&set, SIGINT);
-        if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask))
-            throw SysError("unblocking SIGINT");
-    };
-    auto restoreSignals = [&]() {
-        if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr))
-            throw SysError("restoring signals");
-
-        if (sigaction(SIGINT, &old, 0))
-            throw SysError("restoring handler for SIGINT");
-    };
-
-    setupSignals();
-    char * s = readline(prompt.c_str());
-    Finally doFree([&]() { free(s); });
-    restoreSignals();
-
-    if (g_signal_received) {
-        g_signal_received = 0;
-        input.clear();
-        return true;
-    }
-
-    if (!s)
-      return false;
-    input += s;
-    input += '\n';
-    return true;
-}
-
-
 StringSet NixRepl::completePrefix(const std::string & prefix)
 {
     StringSet completions;
@@ -421,11 +262,14 @@ StringSet NixRepl::completePrefix(const std::string & prefix)
         try {
             auto dir = std::string(cur, 0, slash);
             auto prefix2 = std::string(cur, slash + 1);
-            for (auto & entry : readDirectory(dir == "" ? "/" : dir)) {
-                if (entry.name[0] != '.' && hasPrefix(entry.name, prefix2))
-                    completions.insert(prev + dir + "/" + entry.name);
+            for (auto & entry : std::filesystem::directory_iterator{dir == "" ? "/" : dir}) {
+                checkInterrupt();
+                auto name = entry.path().filename().string();
+                if (name[0] != '.' && hasPrefix(name, prefix2))
+                    completions.insert(prev + entry.path().string());
             }
         } catch (Error &) {
+        } catch (std::filesystem::filesystem_error &) {
         }
     } else if ((dot = cur.rfind('.')) == std::string::npos) {
         /* This is a variable name; look it up in the current scope. */
@@ -452,7 +296,7 @@ StringSet NixRepl::completePrefix(const std::string & prefix)
             e->eval(*state, *env, v);
             state->forceAttrs(v, noPos, "while evaluating an attrset for the purpose of completion (this error should not be displayed; file an issue?)");
 
-            for (auto & i : *v.attrs) {
+            for (auto & i : *v.attrs()) {
                 std::string_view name = state->symbols[i.name];
                 if (name.substr(0, cur2.size()) != cur2) continue;
                 completions.insert(concatStrings(prev, expr, ".", name));
@@ -464,6 +308,8 @@ StringSet NixRepl::completePrefix(const std::string & prefix)
             // Quietly ignore evaluation errors.
         } catch (BadURL & e) {
             // Quietly ignore BadURL flake-related errors.
+        } catch (FileNotFound & e) {
+            // Quietly ignore non-existent file beeing `import`-ed.
         }
     }
 
@@ -519,7 +365,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
     if (line.empty())
         return ProcessLineResult::PromptAgain;
 
-    _isInterrupted = false;
+    setInterrupted(false);
 
     std::string command, arg;
 
@@ -548,6 +394,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
              << "  :l, :load <path>             Load Nix expression and add it to scope\n"
              << "  :lf, :load-flake <ref>       Load Nix flake and add it to scope\n"
              << "  :p, :print <expr>            Evaluate and print expression recursively\n"
+             << "                               Strings are printed directly, without escaping.\n"
              << "  :q, :quit                    Exit nix-repl\n"
              << "  :r, :reload                  Reload all files\n"
              << "  :sh <expr>                   Build dependencies of derivation, then start\n"
@@ -651,7 +498,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
                 auto path = state->coerceToPath(noPos, v, context, "while evaluating the filename to edit");
                 return {path, 0};
             } else if (v.isLambda()) {
-                auto pos = state->positions[v.lambda.fun->pos];
+                auto pos = state->positions[v.payload.lambda.fun->pos];
                 if (auto path = std::get_if<SourcePath>(&pos.origin))
                     return {*path, pos.line};
                 else
@@ -669,7 +516,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
 
         // runProgram redirects stdout to a StringSink,
         // using runProgram2 to allow editors to display their UI
-        runProgram2(RunOptions { .program = editor, .searchPath = true, .args = args });
+        runProgram2(RunOptions { .program = editor, .lookupPath = true, .args = args , .isInteractive = true });
 
         // Reload right after exiting the editor
         state->resetFileCache();
@@ -755,7 +602,11 @@ ProcessLineResult NixRepl::processLine(std::string line)
     else if (command == ":p" || command == ":print") {
         Value v;
         evalString(arg, v);
-        printValue(std::cout, v);
+        if (v.type() == nString) {
+            std::cout << v.string_view();
+        } else {
+            printValue(std::cout, v);
+        }
         std::cout << std::endl;
     }
 
@@ -766,6 +617,35 @@ ProcessLineResult NixRepl::processLine(std::string line)
 
     else if (command == ":doc") {
         Value v;
+
+        auto expr = parseString(arg);
+        std::string fallbackName;
+        PosIdx fallbackPos;
+        DocComment fallbackDoc;
+        if (auto select = dynamic_cast<ExprSelect *>(expr)) {
+            Value vAttrs;
+            auto name = select->evalExceptFinalSelect(*state, *env, vAttrs);
+            fallbackName = state->symbols[name];
+
+            state->forceAttrs(vAttrs, noPos, "while evaluating an attribute set to look for documentation");
+            auto attrs = vAttrs.attrs();
+            assert(attrs);
+            auto attr = attrs->get(name);
+            if (!attr) {
+                // When missing, trigger the normal exception
+                // e.g. :doc builtins.foo
+                // behaves like
+                // nix-repl> builtins.foo      
+                // error: attribute 'foo' missing
+                evalString(arg, v);
+                assert(false);
+            }
+            if (attr->pos) {
+                fallbackPos = attr->pos;
+                fallbackDoc = state->getDocCommentForPos(fallbackPos);
+            }
+        }
+
         evalString(arg, v);
         if (auto doc = state->getDoc(v)) {
             std::string markdown;
@@ -783,6 +663,19 @@ ProcessLineResult NixRepl::processLine(std::string line)
             markdown += stripIndentation(doc->doc);
 
             logger->cout(trim(renderMarkdownToTerminal(markdown)));
+        } else if (fallbackPos) {
+            std::stringstream ss;
+            ss << "Attribute `" << fallbackName << "`\n\n";
+            ss << "  … defined at " << state->positions[fallbackPos] << "\n\n";
+            if (fallbackDoc) {
+                ss << fallbackDoc.getInnerText(state->positions);
+            } else {
+                ss << "No documentation found.\n\n";
+            }
+
+            auto markdown = ss.str();
+            logger->cout(trim(renderMarkdownToTerminal(markdown)));
+
         } else
             throw Error("value does not have documentation");
     }
@@ -840,14 +733,14 @@ void NixRepl::loadFlake(const std::string & flakeRefS)
     if (flakeRefS.empty())
         throw Error("cannot use ':load-flake' without a path specified. (Use '.' for the current working directory.)");
 
-    auto flakeRef = parseFlakeRef(flakeRefS, absPath("."), true);
+    auto flakeRef = parseFlakeRef(fetchSettings, flakeRefS, absPath("."), true);
     if (evalSettings.pureEval && !flakeRef.input.isLocked())
         throw Error("cannot use ':load-flake' on locked flake reference '%s' (use --impure to override)", flakeRefS);
 
     Value v;
 
     flake::callFlake(*state,
-        flake::lockFlake(*state, flakeRef,
+        flake::lockFlake(flakeSettings, *state, flakeRef,
             flake::LockFlags {
                 .updateLockFile = false,
                 .useRegistries = !evalSettings.pureEval,
@@ -899,17 +792,17 @@ void NixRepl::loadFiles()
 void NixRepl::addAttrsToScope(Value & attrs)
 {
     state->forceAttrs(attrs, [&]() { return attrs.determinePos(noPos); }, "while evaluating an attribute set to be merged in the global scope");
-    if (displ + attrs.attrs->size() >= envSize)
+    if (displ + attrs.attrs()->size() >= envSize)
         throw Error("environment full; cannot add more variables");
 
-    for (auto & i : *attrs.attrs) {
+    for (auto & i : *attrs.attrs()) {
         staticEnv->vars.emplace_back(i.name, displ);
         env->values[displ++] = i.value;
         varNames.emplace(state->symbols[i.name]);
     }
     staticEnv->sort();
     staticEnv->deduplicate();
-    notice("Added %1% variables.", attrs.attrs->size());
+    notice("Added %1% variables.", attrs.attrs()->size());
 }
 
 
@@ -941,11 +834,11 @@ void NixRepl::evalString(std::string s, Value & v)
 
 
 std::unique_ptr<AbstractNixRepl> AbstractNixRepl::create(
-   const SearchPath & searchPath, nix::ref<Store> store, ref<EvalState> state,
+   const LookupPath & lookupPath, nix::ref<Store> store, ref<EvalState> state,
    std::function<AnnotatedValues()> getValues)
 {
     return std::make_unique<NixRepl>(
-        searchPath,
+        lookupPath,
         openStore(),
         state,
         getValues
@@ -961,9 +854,9 @@ ReplExitStatus AbstractNixRepl::runSimple(
         NixRepl::AnnotatedValues values;
         return values;
     };
-    SearchPath searchPath = {};
+    LookupPath lookupPath = {};
     auto repl = std::make_unique<NixRepl>(
-            searchPath,
+            lookupPath,
             openStore(),
             evalState,
             getValues

src/libcmd/repl.hh

@@ -3,11 +3,6 @@
 
 #include "eval.hh"
 
-#if HAVE_BOEHMGC
-#define GC_INCLUDE_NEW
-#include <gc/gc_cpp.h>
-#endif
-
 namespace nix {
 
 struct AbstractNixRepl
@@ -25,7 +20,7 @@ struct AbstractNixRepl
     typedef std::vector<std::pair<Value*,std::string>> AnnotatedValues;
 
     static std::unique_ptr<AbstractNixRepl> create(
-        const SearchPath & searchPath, nix::ref<Store> store, ref<EvalState> state,
+        const LookupPath & lookupPath, nix::ref<Store> store, ref<EvalState> state,
         std::function<AnnotatedValues()> getValues);
 
     static ReplExitStatus runSimple(

src/libexpr-c/.version

@@ -0,0 +1 @@
+../../.version

src/libexpr-c/build-utils-meson

@@ -0,0 +1 @@
+../../build-utils-meson

src/libexpr-c/local.mk

@@ -0,0 +1,25 @@
+libraries += libexprc
+
+libexprc_NAME = libnixexprc
+
+libexprc_DIR := $(d)
+
+libexprc_SOURCES := \
+  $(wildcard $(d)/*.cc) \
+
+# Not just for this library itself, but also for downstream libraries using this library
+
+INCLUDE_libexprc := -I $(d)
+libexprc_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libutilc) \
+                     $(INCLUDE_libfetchers) \
+                     $(INCLUDE_libstore) $(INCLUDE_libstorec) \
+                     $(INCLUDE_libexpr) $(INCLUDE_libexprc)
+
+libexprc_LIBS = libutil libutilc libstore libstorec libfetchers libexpr
+
+libexprc_LDFLAGS += $(THREAD_LDFLAGS)
+
+$(eval $(call install-file-in, $(d)/nix-expr-c.pc, $(libdir)/pkgconfig, 0644))
+
+libexprc_FORCE_INSTALL := 1
+

src/libexpr-c/meson.build

@@ -0,0 +1,93 @@
+project('nix-expr-c', 'cpp',
+  version : files('.version'),
+  default_options : [
+    'cpp_std=c++2a',
+    # TODO(Qyriad): increase the warning level
+    'warning_level=1',
+    'debug=true',
+    'optimization=2',
+    'errorlogs=true', # Please print logs for tests that fail
+  ],
+  meson_version : '>= 1.1',
+  license : 'LGPL-2.1-or-later',
+)
+
+cxx = meson.get_compiler('cpp')
+
+subdir('build-utils-meson/deps-lists')
+
+configdata = configuration_data()
+
+deps_private_maybe_subproject = [
+  dependency('nix-util'),
+  dependency('nix-store'),
+  dependency('nix-expr'),
+]
+deps_public_maybe_subproject = [
+  dependency('nix-util-c'),
+  dependency('nix-store-c'),
+]
+subdir('build-utils-meson/subprojects')
+
+subdir('build-utils-meson/threads')
+
+# TODO rename, because it will conflict with downstream projects
+configdata.set_quoted('PACKAGE_VERSION', meson.project_version())
+
+config_h = configure_file(
+  configuration : configdata,
+  output : 'config-expr.h',
+)
+
+add_project_arguments(
+  # TODO(Qyriad): Yes this is how the autoconf+Make system did it.
+  # It would be nice for our headers to be idempotent instead.
+
+  # From C++ libraries, only for internals
+  '-include', 'config-util.hh',
+  '-include', 'config-store.hh',
+  '-include', 'config-expr.hh',
+
+  # From C libraries, for our public, installed headers too
+  '-include', 'config-util.h',
+  '-include', 'config-store.h',
+  '-include', 'config-expr.h',
+  language : 'cpp',
+)
+
+subdir('build-utils-meson/diagnostics')
+
+sources = files(
+  'nix_api_expr.cc',
+  'nix_api_external.cc',
+  'nix_api_value.cc',
+)
+
+include_dirs = [include_directories('.')]
+
+headers = [config_h] + files(
+  'nix_api_expr.h',
+  'nix_api_external.h',
+  'nix_api_value.h',
+)
+
+# TODO move this header to libexpr, maybe don't use it in tests?
+headers += files('nix_api_expr_internal.h')
+
+subdir('build-utils-meson/export-all-symbols')
+
+this_library = library(
+  'nixexprc',
+  sources,
+  dependencies : deps_public + deps_private + deps_other,
+  include_directories : include_dirs,
+  link_args: linker_export_flags,
+  prelink : true, # For C++ static initializers
+  install : true,
+)
+
+install_headers(headers, subdir : 'nix', preserve_path : true)
+
+libraries_private = []
+
+subdir('build-utils-meson/export')

src/libexpr-c/nix-expr-c.pc.in

@@ -0,0 +1,10 @@
+prefix=@prefix@
+libdir=@libdir@
+includedir=@includedir@
+
+Name: Nix
+Description: Nix Language Evaluator - C API
+Version: @PACKAGE_VERSION@
+Requires: nix-store-c
+Libs: -L${libdir} -lnixexprc
+Cflags: -I${includedir}/nix

src/libexpr-c/nix_api_expr.cc

@@ -0,0 +1,213 @@
+#include <cstring>
+#include <stdexcept>
+#include <string>
+
+#include "eval.hh"
+#include "eval-gc.hh"
+#include "globals.hh"
+#include "eval-settings.hh"
+
+#include "nix_api_expr.h"
+#include "nix_api_expr_internal.h"
+#include "nix_api_store.h"
+#include "nix_api_store_internal.h"
+#include "nix_api_util.h"
+#include "nix_api_util_internal.h"
+
+#if HAVE_BOEHMGC
+#  include <mutex>
+#  define GC_INCLUDE_NEW 1
+#  include "gc_cpp.h"
+#endif
+
+nix_err nix_libexpr_init(nix_c_context * context)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    {
+        auto ret = nix_libutil_init(context);
+        if (ret != NIX_OK)
+            return ret;
+    }
+    {
+        auto ret = nix_libstore_init(context);
+        if (ret != NIX_OK)
+            return ret;
+    }
+    try {
+        nix::initGC();
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_expr_eval_from_string(
+    nix_c_context * context, EvalState * state, const char * expr, const char * path, nix_value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        nix::Expr * parsedExpr = state->state.parseExprFromString(expr, state->state.rootPath(nix::CanonPath(path)));
+        state->state.eval(parsedExpr, value->value);
+        state->state.forceValue(value->value, nix::noPos);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, nix_value * arg, nix_value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        state->state.callFunction(fn->value, arg->value, value->value, nix::noPos);
+        state->state.forceValue(value->value, nix::noPos);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_value_call_multi(nix_c_context * context, EvalState * state, nix_value * fn, size_t nargs, nix_value ** args, nix_value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        state->state.callFunction(fn->value, nargs, (nix::Value * *)args, value->value, nix::noPos);
+        state->state.forceValue(value->value, nix::noPos);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_value_force(nix_c_context * context, EvalState * state, nix_value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        state->state.forceValue(value->value, nix::noPos);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        state->state.forceValueDeep(value->value);
+    }
+    NIXC_CATCH_ERRS
+}
+
+EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c, Store * store)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        nix::Strings lookupPath;
+        if (lookupPath_c != nullptr)
+            for (size_t i = 0; lookupPath_c[i] != nullptr; i++)
+                lookupPath.push_back(lookupPath_c[i]);
+
+        void * p = ::operator new(
+            sizeof(EvalState),
+            static_cast<std::align_val_t>(alignof(EvalState)));
+        auto * p2 = static_cast<EvalState *>(p);
+        new (p) EvalState {
+            .fetchSettings = nix::fetchers::Settings{},
+            .settings = nix::EvalSettings{
+                nix::settings.readOnlyMode,
+            },
+            .state = nix::EvalState(
+                nix::LookupPath::parse(lookupPath),
+                store->ptr,
+                p2->fetchSettings,
+                p2->settings),
+        };
+        loadConfFile(p2->settings);
+        return p2;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+void nix_state_free(EvalState * state)
+{
+    delete state;
+}
+
+#if HAVE_BOEHMGC
+std::unordered_map<
+    const void *,
+    unsigned int,
+    std::hash<const void *>,
+    std::equal_to<const void *>,
+    traceable_allocator<std::pair<const void * const, unsigned int>>>
+    nix_refcounts;
+
+std::mutex nix_refcount_lock;
+
+nix_err nix_gc_incref(nix_c_context * context, const void * p)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        std::scoped_lock lock(nix_refcount_lock);
+        auto f = nix_refcounts.find(p);
+        if (f != nix_refcounts.end()) {
+            f->second++;
+        } else {
+            nix_refcounts[p] = 1;
+        }
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_gc_decref(nix_c_context * context, const void * p)
+{
+
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        std::scoped_lock lock(nix_refcount_lock);
+        auto f = nix_refcounts.find(p);
+        if (f != nix_refcounts.end()) {
+            if (--f->second == 0)
+                nix_refcounts.erase(f);
+        } else
+            throw std::runtime_error("nix_gc_decref: object was not referenced");
+    }
+    NIXC_CATCH_ERRS
+}
+
+void nix_gc_now()
+{
+    GC_gcollect();
+}
+
+#else
+nix_err nix_gc_incref(nix_c_context * context, const void *)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    return NIX_OK;
+}
+nix_err nix_gc_decref(nix_c_context * context, const void *)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    return NIX_OK;
+}
+void nix_gc_now() {}
+#endif
+
+nix_err nix_value_incref(nix_c_context * context, nix_value *x)
+{
+    return nix_gc_incref(context, (const void *) x);
+}
+nix_err nix_value_decref(nix_c_context * context, nix_value *x)
+{
+    return nix_gc_decref(context, (const void *) x);
+}
+
+void nix_gc_register_finalizer(void * obj, void * cd, void (*finalizer)(void * obj, void * cd))
+{
+#if HAVE_BOEHMGC
+    GC_REGISTER_FINALIZER(obj, finalizer, cd, 0, 0);
+#endif
+}

src/libexpr-c/nix_api_expr.h

@@ -0,0 +1,247 @@
+#ifndef NIX_API_EXPR_H
+#define NIX_API_EXPR_H
+/** @defgroup libexpr libexpr
+ * @brief Bindings to the Nix language evaluator
+ *
+ * See *[Embedding the Nix Evaluator](@ref nix_evaluator_example)* for an example.
+ * @{
+ */
+/** @file
+ * @brief Main entry for the libexpr C bindings
+ */
+
+#include "nix_api_store.h"
+#include "nix_api_util.h"
+#include <stddef.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+// cffi start
+
+// Type definitions
+/**
+ * @brief Represents a state of the Nix language evaluator.
+ *
+ * Multiple states can be created for multi-threaded
+ * operation.
+ * @struct EvalState
+ * @see nix_state_create
+ */
+typedef struct EvalState EvalState; // nix::EvalState
+
+/** @brief A Nix language value, or thunk that may evaluate to a value.
+ *
+ * Values are the primary objects manipulated in the Nix language.
+ * They are considered to be immutable from a user's perspective, but the process of evaluating a value changes its
+ * ValueType if it was a thunk. After a value has been evaluated, its ValueType does not change.
+ *
+ * Evaluation in this context refers to the process of evaluating a single value object, also called "forcing" the
+ * value; see `nix_value_force`.
+ *
+ * The evaluator manages its own memory, but your use of the C API must follow the reference counting rules.
+ *
+ * @see value_manip
+ * @see nix_value_incref, nix_value_decref
+ */
+typedef struct nix_value nix_value;
+[[deprecated("use nix_value instead")]] typedef nix_value Value;
+
+// Function prototypes
+/**
+ * @brief Initialize the Nix language evaluator.
+ *
+ * This function must be called at least once,
+ * at some point before constructing a EvalState for the first time.
+ * This function can be called multiple times, and is idempotent.
+ *
+ * @param[out] context Optional, stores error information
+ * @return NIX_OK if the initialization was successful, an error code otherwise.
+ */
+nix_err nix_libexpr_init(nix_c_context * context);
+
+/**
+ * @brief Parses and evaluates a Nix expression from a string.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] state The state of the evaluation.
+ * @param[in] expr The Nix expression to parse.
+ * @param[in] path The file path to associate with the expression.
+ * This is required for expressions that contain relative paths (such as `./.`) that are resolved relative to the given
+ * directory.
+ * @param[out] value The result of the evaluation. You must allocate this
+ * yourself.
+ * @return NIX_OK if the evaluation was successful, an error code otherwise.
+ */
+nix_err nix_expr_eval_from_string(
+    nix_c_context * context, EvalState * state, const char * expr, const char * path, nix_value * value);
+
+/**
+ * @brief Calls a Nix function with an argument.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] state The state of the evaluation.
+ * @param[in] fn The Nix function to call.
+ * @param[in] arg The argument to pass to the function.
+ * @param[out] value The result of the function call.
+ * @return NIX_OK if the function call was successful, an error code otherwise.
+ * @see nix_init_apply() for a similar function that does not performs the call immediately, but stores it as a thunk.
+ *      Note the different argument order.
+ */
+nix_err nix_value_call(nix_c_context * context, EvalState * state, nix_value * fn, nix_value * arg, nix_value * value);
+
+/**
+ * @brief Calls a Nix function with multiple arguments.
+ *
+ * Technically these are functions that return functions. It is common for Nix
+ * functions to be curried, so this function is useful for calling them.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] state The state of the evaluation.
+ * @param[in] fn The Nix function to call.
+ * @param[in] nargs The number of arguments.
+ * @param[in] args The arguments to pass to the function.
+ * @param[out] value The result of the function call.
+ *
+ * @see nix_value_call     For the single argument primitive.
+ * @see NIX_VALUE_CALL           For a macro that wraps this function for convenience.
+ */
+nix_err nix_value_call_multi(
+    nix_c_context * context, EvalState * state, nix_value * fn, size_t nargs, nix_value ** args, nix_value * value);
+
+/**
+ * @brief Calls a Nix function with multiple arguments.
+ *
+ * Technically these are functions that return functions. It is common for Nix
+ * functions to be curried, so this function is useful for calling them.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] state The state of the evaluation.
+ * @param[out] value The result of the function call.
+ * @param[in] fn The Nix function to call.
+ * @param[in] args The arguments to pass to the function.
+ *
+ * @see nix_value_call_multi
+ */
+#define NIX_VALUE_CALL(context, state, value, fn, ...)                      \
+    do {                                                                    \
+        nix_value * args_array[] = {__VA_ARGS__};                           \
+        size_t nargs = sizeof(args_array) / sizeof(args_array[0]);          \
+        nix_value_call_multi(context, state, fn, nargs, args_array, value); \
+    } while (0)
+
+/**
+ * @brief Forces the evaluation of a Nix value.
+ *
+ * The Nix interpreter is lazy, and not-yet-evaluated values can be
+ * of type NIX_TYPE_THUNK instead of their actual value.
+ *
+ * This function mutates such a `nix_value`, so that, if successful, it has its final type.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] state The state of the evaluation.
+ * @param[in,out] value The Nix value to force.
+ * @post value is not of type NIX_TYPE_THUNK
+ * @return NIX_OK if the force operation was successful, an error code
+ * otherwise.
+ */
+nix_err nix_value_force(nix_c_context * context, EvalState * state, nix_value * value);
+
+/**
+ * @brief Forces the deep evaluation of a Nix value.
+ *
+ * Recursively calls nix_value_force
+ *
+ * @see nix_value_force
+ * @warning Calling this function on a recursive data structure will cause a
+ * stack overflow.
+ * @param[out] context Optional, stores error information
+ * @param[in] state The state of the evaluation.
+ * @param[in,out] value The Nix value to force.
+ * @return NIX_OK if the deep force operation was successful, an error code
+ * otherwise.
+ */
+nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_value * value);
+
+/**
+ * @brief Create a new Nix language evaluator state.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] lookupPath Null-terminated array of strings corresponding to entries in NIX_PATH.
+ * @param[in] store The Nix store to use.
+ * @return A new Nix state or NULL on failure.
+ */
+EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath, Store * store);
+
+/**
+ * @brief Frees a Nix state.
+ *
+ * Does not fail.
+ *
+ * @param[in] state The state to free.
+ */
+void nix_state_free(EvalState * state);
+
+/** @addtogroup GC
+ * @brief Reference counting and garbage collector operations
+ *
+ * The Nix language evaluator uses a garbage collector. To ease C interop, we implement
+ * a reference counting scheme, where objects will be deallocated
+ * when there are no references from the Nix side, and the reference count kept
+ * by the C API reaches `0`.
+ *
+ * Functions returning a garbage-collected object will automatically increase
+ * the refcount for you. You should make sure to call `nix_gc_decref` when
+ * you're done with a value returned by the evaluator.
+ * @{
+ */
+
+// TODO: Deprecate nix_gc_incref in favor of the type-specific reference counting functions?
+//       e.g. nix_value_incref.
+//       It gives implementors more flexibility, and adds safety, so that generated
+//       bindings can be used without fighting the host type system (where applicable).
+/**
+ * @brief Increment the garbage collector reference counter for the given object.
+ *
+ * The Nix language evaluator C API keeps track of alive objects by reference counting.
+ * When you're done with a refcounted pointer, call nix_gc_decref().
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] object The object to keep alive
+ */
+nix_err nix_gc_incref(nix_c_context * context, const void * object);
+/**
+ * @brief Decrement the garbage collector reference counter for the given object
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] object The object to stop referencing
+ */
+nix_err nix_gc_decref(nix_c_context * context, const void * object);
+
+/**
+ * @brief Trigger the garbage collector manually
+ *
+ * You should not need to do this, but it can be useful for debugging.
+ */
+void nix_gc_now();
+
+/**
+ * @brief Register a callback that gets called when the object is garbage
+ * collected.
+ * @note Objects can only have a single finalizer. This function overwrites existing values
+ * silently.
+ * @param[in] obj the object to watch
+ * @param[in] cd the data to pass to the finalizer
+ * @param[in] finalizer the callback function, called with obj and cd
+ */
+void nix_gc_register_finalizer(void * obj, void * cd, void (*finalizer)(void * obj, void * cd));
+
+/** @} */
+// cffi end
+#ifdef __cplusplus
+}
+#endif
+
+/** @} */
+
+#endif // NIX_API_EXPR_H

src/libexpr-c/nix_api_expr_internal.h

@@ -0,0 +1,53 @@
+#ifndef NIX_API_EXPR_INTERNAL_H
+#define NIX_API_EXPR_INTERNAL_H
+
+#include "fetch-settings.hh"
+#include "eval.hh"
+#include "eval-settings.hh"
+#include "attr-set.hh"
+#include "nix_api_value.h"
+
+struct EvalState
+{
+    nix::fetchers::Settings fetchSettings;
+    nix::EvalSettings settings;
+    nix::EvalState state;
+};
+
+struct BindingsBuilder
+{
+    nix::BindingsBuilder builder;
+};
+
+struct ListBuilder
+{
+    nix::ListBuilder builder;
+};
+
+struct nix_value
+{
+    nix::Value value;
+};
+
+struct nix_string_return
+{
+    std::string str;
+};
+
+struct nix_printer
+{
+    std::ostream & s;
+};
+
+struct nix_string_context
+{
+    nix::NixStringContext & ctx;
+};
+
+struct nix_realised_string
+{
+    std::string str;
+    std::vector<StorePath> storePaths;
+};
+
+#endif // NIX_API_EXPR_INTERNAL_H

src/libexpr-c/nix_api_external.cc

@@ -0,0 +1,198 @@
+#include "attr-set.hh"
+#include "config.hh"
+#include "eval.hh"
+#include "globals.hh"
+#include "value.hh"
+
+#include "nix_api_expr.h"
+#include "nix_api_expr_internal.h"
+#include "nix_api_external.h"
+#include "nix_api_util.h"
+#include "nix_api_util_internal.h"
+#include "nix_api_value.h"
+#include "value/context.hh"
+
+#include <nlohmann/json.hpp>
+
+#if HAVE_BOEHMGC
+# include "gc/gc.h"
+# define GC_INCLUDE_NEW 1
+# include "gc_cpp.h"
+#endif
+
+void nix_set_string_return(nix_string_return * str, const char * c)
+{
+    str->str = c;
+}
+
+nix_err nix_external_print(nix_c_context * context, nix_printer * printer, const char * c)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        printer->s << c;
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_external_add_string_context(nix_c_context * context, nix_string_context * ctx, const char * c)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto r = nix::NixStringContextElem::parse(c);
+        ctx->ctx.insert(r);
+    }
+    NIXC_CATCH_ERRS
+}
+
+class NixCExternalValue : public nix::ExternalValueBase
+{
+    NixCExternalValueDesc & desc;
+    void * v;
+
+public:
+    NixCExternalValue(NixCExternalValueDesc & desc, void * v)
+        : desc(desc)
+        , v(v){};
+    void * get_ptr()
+    {
+        return v;
+    }
+    /**
+     * Print out the value
+     */
+    virtual std::ostream & print(std::ostream & str) const override
+    {
+        nix_printer p{str};
+        desc.print(v, &p);
+        return str;
+    }
+
+    /**
+     * Return a simple string describing the type
+     */
+    virtual std::string showType() const override
+    {
+        nix_string_return res;
+        desc.showType(v, &res);
+        return std::move(res.str);
+    }
+
+    /**
+     * Return a string to be used in builtins.typeOf
+     */
+    virtual std::string typeOf() const override
+    {
+        nix_string_return res;
+        desc.typeOf(v, &res);
+        return std::move(res.str);
+    }
+
+    /**
+     * Coerce the value to a string.
+     */
+    virtual std::string coerceToString(
+        nix::EvalState & state,
+        const nix::PosIdx & pos,
+        nix::NixStringContext & context,
+        bool copyMore,
+        bool copyToStore) const override
+    {
+        if (!desc.coerceToString) {
+            return nix::ExternalValueBase::coerceToString(state, pos, context, copyMore, copyToStore);
+        }
+        nix_string_context ctx{context};
+        nix_string_return res{""};
+        // todo: pos, errors
+        desc.coerceToString(v, &ctx, copyMore, copyToStore, &res);
+        if (res.str.empty()) {
+            return nix::ExternalValueBase::coerceToString(state, pos, context, copyMore, copyToStore);
+        }
+        return std::move(res.str);
+    }
+
+    /**
+     * Compare to another value of the same type.
+     */
+    virtual bool operator==(const ExternalValueBase & b) const noexcept override
+    {
+        if (!desc.equal) {
+            return false;
+        }
+        auto r = dynamic_cast<const NixCExternalValue *>(&b);
+        if (!r)
+            return false;
+        return desc.equal(v, r->v);
+    }
+
+    /**
+     * Print the value as JSON.
+     */
+    virtual nlohmann::json printValueAsJSON(
+        nix::EvalState & state, bool strict, nix::NixStringContext & context, bool copyToStore = true) const override
+    {
+        if (!desc.printValueAsJSON) {
+            return nix::ExternalValueBase::printValueAsJSON(state, strict, context, copyToStore);
+        }
+        nix_string_context ctx{context};
+        nix_string_return res{""};
+        desc.printValueAsJSON(v, (EvalState *) &state, strict, &ctx, copyToStore, &res);
+        if (res.str.empty()) {
+            return nix::ExternalValueBase::printValueAsJSON(state, strict, context, copyToStore);
+        }
+        return nlohmann::json::parse(res.str);
+    }
+
+    /**
+     * Print the value as XML.
+     */
+    virtual void printValueAsXML(
+        nix::EvalState & state,
+        bool strict,
+        bool location,
+        nix::XMLWriter & doc,
+        nix::NixStringContext & context,
+        nix::PathSet & drvsSeen,
+        const nix::PosIdx pos) const override
+    {
+        if (!desc.printValueAsXML) {
+            return nix::ExternalValueBase::printValueAsXML(state, strict, location, doc, context, drvsSeen, pos);
+        }
+        nix_string_context ctx{context};
+        desc.printValueAsXML(
+            v, (EvalState *) &state, strict, location, &doc, &ctx, &drvsSeen,
+            *reinterpret_cast<const uint32_t *>(&pos));
+    }
+
+    virtual ~NixCExternalValue() override{};
+};
+
+ExternalValue * nix_create_external_value(nix_c_context * context, NixCExternalValueDesc * desc, void * v)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto ret = new
+#if HAVE_BOEHMGC
+            (GC)
+#endif
+                NixCExternalValue(*desc, v);
+        nix_gc_incref(nullptr, ret);
+        return (ExternalValue *) ret;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+void * nix_get_external_value_content(nix_c_context * context, ExternalValue * b)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto r = dynamic_cast<NixCExternalValue *>((nix::ExternalValueBase *) b);
+        if (r)
+            return r->get_ptr();
+        return nullptr;
+    }
+    NIXC_CATCH_ERRS_NULL
+}

src/libexpr-c/nix_api_external.h

@@ -0,0 +1,196 @@
+#ifndef NIX_API_EXTERNAL_H
+#define NIX_API_EXTERNAL_H
+/** @ingroup libexpr
+ * @addtogroup Externals
+ * @brief Deal with external values
+ * @{
+ */
+/** @file
+ * @brief libexpr C bindings dealing with external values
+ */
+
+#include "nix_api_expr.h"
+#include "nix_api_util.h"
+#include "nix_api_value.h"
+#include "stdbool.h"
+#include "stddef.h"
+#include "stdint.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+// cffi start
+
+/**
+ * @brief Represents a string owned by the Nix language evaluator.
+ * @see nix_set_owned_string
+ */
+typedef struct nix_string_return nix_string_return;
+/**
+ * @brief Wraps a stream that can output multiple string pieces.
+ */
+typedef struct nix_printer nix_printer;
+/**
+ * @brief A list of string context items
+ */
+typedef struct nix_string_context nix_string_context;
+
+/**
+ * @brief Sets the contents of a nix_string_return
+ *
+ * Copies the passed string.
+ * @param[out] str the nix_string_return to write to
+ * @param[in]  c   The string to copy
+ */
+void nix_set_string_return(nix_string_return * str, const char * c);
+
+/**
+ * Print to the nix_printer
+ *
+ * @param[out] context Optional, stores error information
+ * @param[out] printer The nix_printer to print to
+ * @param[in] str The string to print
+ * @returns NIX_OK if everything worked
+ */
+nix_err nix_external_print(nix_c_context * context, nix_printer * printer, const char * str);
+
+/**
+ * Add string context to the nix_string_context object
+ * @param[out] context Optional, stores error information
+ * @param[out] string_context The nix_string_context to add to
+ * @param[in] c The context string to add
+ * @returns NIX_OK if everything worked
+ */
+nix_err nix_external_add_string_context(nix_c_context * context, nix_string_context * string_context, const char * c);
+
+/**
+ * @brief Definition for a class of external values
+ *
+ * Create and implement one of these, then pass it to nix_create_external_value
+ * Make sure to keep it alive while the external value lives.
+ *
+ * Optional functions can be set to NULL
+ *
+ * @see nix_create_external_value
+ */
+typedef struct NixCExternalValueDesc
+{
+    /**
+     * @brief Called when printing the external value
+     *
+     * @param[in] self the void* passed to nix_create_external_value
+     * @param[out] printer The printer to print to, pass to nix_external_print
+     */
+    void (*print)(void * self, nix_printer * printer);
+    /**
+     * @brief Called on :t
+     * @param[in] self the void* passed to nix_create_external_value
+     * @param[out] res the return value
+     */
+    void (*showType)(void * self, nix_string_return * res);
+    /**
+     * @brief Called on `builtins.typeOf`
+     * @param self the void* passed to nix_create_external_value
+     * @param[out] res the return value
+     */
+    void (*typeOf)(void * self, nix_string_return * res);
+    /**
+     * @brief Called on "${str}" and builtins.toString.
+     *
+     * The latter with coerceMore=true
+     * Optional, the default is to throw an error.
+     * @param[in] self the void* passed to nix_create_external_value
+     * @param[out] c writable string context for the resulting string
+     * @param[in] coerceMore boolean, try to coerce to strings in more cases
+     * instead of throwing an error
+     * @param[in] copyToStore boolean, whether to copy referenced paths to store
+     * or keep them as-is
+     * @param[out] res the return value. Not touching this, or setting it to the
+     * empty string, will make the conversion throw an error.
+     */
+    void (*coerceToString)(
+        void * self, nix_string_context * c, int coerceMore, int copyToStore, nix_string_return * res);
+    /**
+     * @brief Try to compare two external values
+     *
+     * Optional, the default is always false.
+     * If the other object was not a Nix C external value, this comparison will
+     * also return false
+     * @param[in] self the void* passed to nix_create_external_value
+     * @param[in] other the void* passed to the other object's
+     * nix_create_external_value
+     * @returns true if the objects are deemed to be equal
+     */
+    int (*equal)(void * self, void * other);
+    /**
+     * @brief Convert the external value to json
+     *
+     * Optional, the default is to throw an error
+     * @param[in] self the void* passed to nix_create_external_value
+     * @param[in] state The evaluator state
+     * @param[in] strict boolean Whether to force the value before printing
+     * @param[out] c writable string context for the resulting string
+     * @param[in] copyToStore whether to copy referenced paths to store or keep
+     * them as-is
+     * @param[out] res the return value. Gets parsed as JSON. Not touching this,
+     * or setting it to the empty string, will make the conversion throw an error.
+     */
+    void (*printValueAsJSON)(
+        void * self, EvalState * state, bool strict, nix_string_context * c, bool copyToStore, nix_string_return * res);
+    /**
+     * @brief Convert the external value to XML
+     *
+     * Optional, the default is to throw an error
+     * @todo The mechanisms for this call are incomplete. There are no C
+     *       bindings to work with XML, pathsets and positions.
+     * @param[in] self the void* passed to nix_create_external_value
+     * @param[in] state The evaluator state
+     * @param[in] strict boolean Whether to force the value before printing
+     * @param[in] location boolean Whether to include position information in the
+     * xml
+     * @param[out] doc XML document to output to
+     * @param[out] c writable string context for the resulting string
+     * @param[in,out] drvsSeen a path set to avoid duplicating derivations
+     * @param[in] pos The position of the call.
+     */
+    void (*printValueAsXML)(
+        void * self,
+        EvalState * state,
+        int strict,
+        int location,
+        void * doc,
+        nix_string_context * c,
+        void * drvsSeen,
+        int pos);
+} NixCExternalValueDesc;
+
+/**
+ * @brief Create an external value, that can be given to nix_init_external
+ *
+ * Owned by the GC. Use nix_gc_decref when you're done with the pointer.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] desc a NixCExternalValueDesc, you should keep this alive as long
+ * as the ExternalValue lives
+ * @param[in] v the value to store
+ * @returns external value, owned by the garbage collector
+ * @see nix_init_external
+ */
+ExternalValue * nix_create_external_value(nix_c_context * context, NixCExternalValueDesc * desc, void * v);
+
+/**
+ * @brief Extract the pointer from a nix c external value.
+ * @param[out] context Optional, stores error information
+ * @param[in] b The external value
+ * @returns The pointer, or null if the external value was not from nix c.
+ * @see nix_get_external
+ */
+void * nix_get_external_value_content(nix_c_context * context, ExternalValue * b);
+
+// cffi end
+#ifdef __cplusplus
+}
+#endif
+/** @} */
+
+#endif // NIX_API_EXTERNAL_H

src/libexpr-c/nix_api_value.cc

@@ -0,0 +1,663 @@
+#include "attr-set.hh"
+#include "config.hh"
+#include "eval.hh"
+#include "globals.hh"
+#include "path.hh"
+#include "primops.hh"
+#include "value.hh"
+
+#include "nix_api_expr.h"
+#include "nix_api_expr_internal.h"
+#include "nix_api_util.h"
+#include "nix_api_util_internal.h"
+#include "nix_api_store_internal.h"
+#include "nix_api_value.h"
+#include "value/context.hh"
+
+#if HAVE_BOEHMGC
+#  include "gc/gc.h"
+#  define GC_INCLUDE_NEW 1
+#  include "gc_cpp.h"
+#endif
+
+// Internal helper functions to check [in] and [out] `Value *` parameters
+static const nix::Value & check_value_not_null(const nix_value * value)
+{
+    if (!value) {
+        throw std::runtime_error("nix_value is null");
+    }
+    return *((const nix::Value *) value);
+}
+
+static nix::Value & check_value_not_null(nix_value * value)
+{
+    if (!value) {
+        throw std::runtime_error("nix_value is null");
+    }
+    return value->value;
+}
+
+static const nix::Value & check_value_in(const nix_value * value)
+{
+    auto & v = check_value_not_null(value);
+    if (!v.isValid()) {
+        throw std::runtime_error("Uninitialized nix_value");
+    }
+    return v;
+}
+
+static nix::Value & check_value_in(nix_value * value)
+{
+    auto & v = check_value_not_null(value);
+    if (!v.isValid()) {
+        throw std::runtime_error("Uninitialized nix_value");
+    }
+    return v;
+}
+
+static nix::Value & check_value_out(nix_value * value)
+{
+    auto & v = check_value_not_null(value);
+    if (v.isValid()) {
+        throw std::runtime_error("nix_value already initialized. Variables are immutable");
+    }
+    return v;
+}
+
+static inline nix_value * as_nix_value_ptr(nix::Value * v)
+{
+    return reinterpret_cast<nix_value *>(v);
+}
+
+/**
+ * Helper function to convert calls from nix into C API.
+ *
+ * Deals with errors and converts arguments from C++ into C types.
+ */
+static void nix_c_primop_wrapper(
+    PrimOpFun f, void * userdata, nix::EvalState & state, const nix::PosIdx pos, nix::Value ** args, nix::Value & v)
+{
+    nix_c_context ctx;
+
+    // v currently has a thunk, but the C API initializers require an uninitialized value.
+    //
+    // We can't destroy the thunk, because that makes it impossible to retry,
+    // which is needed for tryEval and for evaluation drivers that evaluate more
+    // than one value (e.g. an attrset with two derivations, both of which
+    // reference v).
+    //
+    // Instead we create a temporary value, and then assign the result to v.
+    // This does not give the primop definition access to the thunk, but that's
+    // ok because we don't see a need for this yet (e.g. inspecting thunks,
+    // or maybe something to make blackholes work better; we don't know).
+    nix::Value vTmp;
+
+    f(userdata, &ctx, (EvalState *) &state, (nix_value **) args, (nix_value *) &vTmp);
+
+    if (ctx.last_err_code != NIX_OK) {
+        /* TODO: Throw different errors depending on the error code */
+        state.error<nix::EvalError>("Error from custom function: %s", *ctx.last_err).atPos(pos).debugThrow();
+    }
+
+    if (!vTmp.isValid()) {
+        state.error<nix::EvalError>("Implementation error in custom function: return value was not initialized")
+            .atPos(pos)
+            .debugThrow();
+    }
+
+    if (vTmp.type() == nix::nThunk) {
+        // We might allow this in the future if it makes sense for the evaluator
+        // e.g. implementing tail recursion by returning a thunk to the next
+        // "iteration". Until then, this is most likely a mistake or misunderstanding.
+        state.error<nix::EvalError>("Implementation error in custom function: return value must not be a thunk")
+            .atPos(pos)
+            .debugThrow();
+    }
+
+    v = vTmp;
+}
+
+PrimOp * nix_alloc_primop(
+    nix_c_context * context,
+    PrimOpFun fun,
+    int arity,
+    const char * name,
+    const char ** args,
+    const char * doc,
+    void * user_data)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        using namespace std::placeholders;
+        auto p = new
+#if HAVE_BOEHMGC
+            (GC)
+#endif
+                nix::PrimOp{
+                    .name = name,
+                    .args = {},
+                    .arity = (size_t) arity,
+                    .doc = doc,
+                    .fun = std::bind(nix_c_primop_wrapper, fun, user_data, _1, _2, _3, _4)};
+        if (args)
+            for (size_t i = 0; args[i]; i++)
+                p->args.emplace_back(*args);
+        nix_gc_incref(nullptr, p);
+        return (PrimOp *) p;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+nix_err nix_register_primop(nix_c_context * context, PrimOp * primOp)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        nix::RegisterPrimOp r(std::move(*((nix::PrimOp *) primOp)));
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_value * nix_alloc_value(nix_c_context * context, EvalState * state)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        nix_value * res = as_nix_value_ptr(state->state.allocValue());
+        nix_gc_incref(nullptr, res);
+        return res;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+ValueType nix_get_type(nix_c_context * context, const nix_value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_in(value);
+        using namespace nix;
+        switch (v.type()) {
+        case nThunk:
+            return NIX_TYPE_THUNK;
+        case nInt:
+            return NIX_TYPE_INT;
+        case nFloat:
+            return NIX_TYPE_FLOAT;
+        case nBool:
+            return NIX_TYPE_BOOL;
+        case nString:
+            return NIX_TYPE_STRING;
+        case nPath:
+            return NIX_TYPE_PATH;
+        case nNull:
+            return NIX_TYPE_NULL;
+        case nAttrs:
+            return NIX_TYPE_ATTRS;
+        case nList:
+            return NIX_TYPE_LIST;
+        case nFunction:
+            return NIX_TYPE_FUNCTION;
+        case nExternal:
+            return NIX_TYPE_EXTERNAL;
+        }
+        return NIX_TYPE_NULL;
+    }
+    NIXC_CATCH_ERRS_RES(NIX_TYPE_NULL);
+}
+
+const char * nix_get_typename(nix_c_context * context, const nix_value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_in(value);
+        auto s = nix::showType(v);
+        return strdup(s.c_str());
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+bool nix_get_bool(nix_c_context * context, const nix_value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_in(value);
+        assert(v.type() == nix::nBool);
+        return v.boolean();
+    }
+    NIXC_CATCH_ERRS_RES(false);
+}
+
+nix_err
+nix_get_string(nix_c_context * context, const nix_value * value, nix_get_string_callback callback, void * user_data)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_in(value);
+        assert(v.type() == nix::nString);
+        call_nix_get_string_callback(v.c_str(), callback, user_data);
+    }
+    NIXC_CATCH_ERRS
+}
+
+const char * nix_get_path_string(nix_c_context * context, const nix_value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_in(value);
+        assert(v.type() == nix::nPath);
+        // NOTE (from @yorickvP)
+        // v._path.path should work but may not be how Eelco intended it.
+        // Long-term this function should be rewritten to copy some data into a
+        // user-allocated string.
+        // We could use v.path().to_string().c_str(), but I'm concerned this
+        // crashes. Looks like .path() allocates a CanonPath with a copy of the
+        // string, then it gets the underlying data from that.
+        return v.payload.path.path;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+unsigned int nix_get_list_size(nix_c_context * context, const nix_value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_in(value);
+        assert(v.type() == nix::nList);
+        return v.listSize();
+    }
+    NIXC_CATCH_ERRS_RES(0);
+}
+
+unsigned int nix_get_attrs_size(nix_c_context * context, const nix_value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_in(value);
+        assert(v.type() == nix::nAttrs);
+        return v.attrs()->size();
+    }
+    NIXC_CATCH_ERRS_RES(0);
+}
+
+double nix_get_float(nix_c_context * context, const nix_value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_in(value);
+        assert(v.type() == nix::nFloat);
+        return v.fpoint();
+    }
+    NIXC_CATCH_ERRS_RES(0.0);
+}
+
+int64_t nix_get_int(nix_c_context * context, const nix_value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_in(value);
+        assert(v.type() == nix::nInt);
+        return v.integer();
+    }
+    NIXC_CATCH_ERRS_RES(0);
+}
+
+ExternalValue * nix_get_external(nix_c_context * context, nix_value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_out(value);
+        assert(v.type() == nix::nExternal);
+        return (ExternalValue *) v.external();
+    }
+    NIXC_CATCH_ERRS_NULL;
+}
+
+nix_value * nix_get_list_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int ix)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_in(value);
+        assert(v.type() == nix::nList);
+        auto * p = v.listElems()[ix];
+        nix_gc_incref(nullptr, p);
+        if (p != nullptr)
+            state->state.forceValue(*p, nix::noPos);
+        return as_nix_value_ptr(p);
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+nix_value * nix_get_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_in(value);
+        assert(v.type() == nix::nAttrs);
+        nix::Symbol s = state->state.symbols.create(name);
+        auto attr = v.attrs()->get(s);
+        if (attr) {
+            nix_gc_incref(nullptr, attr->value);
+            state->state.forceValue(*attr->value, nix::noPos);
+            return as_nix_value_ptr(attr->value);
+        }
+        nix_set_err_msg(context, NIX_ERR_KEY, "missing attribute");
+        return nullptr;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+bool nix_has_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_in(value);
+        assert(v.type() == nix::nAttrs);
+        nix::Symbol s = state->state.symbols.create(name);
+        auto attr = v.attrs()->get(s);
+        if (attr)
+            return true;
+        return false;
+    }
+    NIXC_CATCH_ERRS_RES(false);
+}
+
+nix_value * nix_get_attr_byidx(
+    nix_c_context * context, const nix_value * value, EvalState * state, unsigned int i, const char ** name)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_in(value);
+        const nix::Attr & a = (*v.attrs())[i];
+        *name = state->state.symbols[a.name].c_str();
+        nix_gc_incref(nullptr, a.value);
+        state->state.forceValue(*a.value, nix::noPos);
+        return as_nix_value_ptr(a.value);
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+const char *
+nix_get_attr_name_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int i)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_in(value);
+        const nix::Attr & a = (*v.attrs())[i];
+        return state->state.symbols[a.name].c_str();
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+nix_err nix_init_bool(nix_c_context * context, nix_value * value, bool b)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_out(value);
+        v.mkBool(b);
+    }
+    NIXC_CATCH_ERRS
+}
+
+// todo string context
+nix_err nix_init_string(nix_c_context * context, nix_value * value, const char * str)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_out(value);
+        v.mkString(std::string_view(str));
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_path_string(nix_c_context * context, EvalState * s, nix_value * value, const char * str)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_out(value);
+        v.mkPath(s->state.rootPath(nix::CanonPath(str)));
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_float(nix_c_context * context, nix_value * value, double d)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_out(value);
+        v.mkFloat(d);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_int(nix_c_context * context, nix_value * value, int64_t i)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_out(value);
+        v.mkInt(i);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_null(nix_c_context * context, nix_value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_out(value);
+        v.mkNull();
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_apply(nix_c_context * context, nix_value * value, nix_value * fn, nix_value * arg)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        auto & f = check_value_not_null(fn);
+        auto & a = check_value_not_null(arg);
+        v.mkApp(&f, &a);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_external(nix_c_context * context, nix_value * value, ExternalValue * val)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_out(value);
+        auto r = (nix::ExternalValueBase *) val;
+        v.mkExternal(r);
+    }
+    NIXC_CATCH_ERRS
+}
+
+ListBuilder * nix_make_list_builder(nix_c_context * context, EvalState * state, size_t capacity)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto builder = state->state.buildList(capacity);
+        return new
+#if HAVE_BOEHMGC
+            (NoGC)
+#endif
+                ListBuilder{std::move(builder)};
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+nix_err
+nix_list_builder_insert(nix_c_context * context, ListBuilder * list_builder, unsigned int index, nix_value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & e = check_value_not_null(value);
+        list_builder->builder[index] = &e;
+    }
+    NIXC_CATCH_ERRS
+}
+
+void nix_list_builder_free(ListBuilder * list_builder)
+{
+#if HAVE_BOEHMGC
+    GC_FREE(list_builder);
+#else
+    delete list_builder;
+#endif
+}
+
+nix_err nix_make_list(nix_c_context * context, ListBuilder * list_builder, nix_value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_out(value);
+        v.mkList(list_builder->builder);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_primop(nix_c_context * context, nix_value * value, PrimOp * p)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_out(value);
+        v.mkPrimOp((nix::PrimOp *) p);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_copy_value(nix_c_context * context, nix_value * value, const nix_value * source)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_out(value);
+        auto & s = check_value_in(source);
+        v = s;
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_make_attrs(nix_c_context * context, nix_value * value, BindingsBuilder * b)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_out(value);
+        v.mkAttrs(b->builder);
+    }
+    NIXC_CATCH_ERRS
+}
+
+BindingsBuilder * nix_make_bindings_builder(nix_c_context * context, EvalState * state, size_t capacity)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto bb = state->state.buildBindings(capacity);
+        return new
+#if HAVE_BOEHMGC
+            (NoGC)
+#endif
+                BindingsBuilder{std::move(bb)};
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+nix_err nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * bb, const char * name, nix_value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        nix::Symbol s = bb->builder.state.symbols.create(name);
+        bb->builder.insert(s, &v);
+    }
+    NIXC_CATCH_ERRS
+}
+
+void nix_bindings_builder_free(BindingsBuilder * bb)
+{
+#if HAVE_BOEHMGC
+    GC_FREE((nix::BindingsBuilder *) bb);
+#else
+    delete (nix::BindingsBuilder *) bb;
+#endif
+}
+
+nix_realised_string * nix_string_realise(nix_c_context * context, EvalState * state, nix_value * value, bool isIFD)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_in(value);
+        nix::NixStringContext stringContext;
+        auto rawStr = state->state.coerceToString(nix::noPos, v, stringContext, "while realising a string").toOwned();
+        nix::StorePathSet storePaths;
+        auto rewrites = state->state.realiseContext(stringContext, &storePaths);
+
+        auto s = nix::rewriteStrings(rawStr, rewrites);
+
+        // Convert to the C API StorePath type and convert to vector for index-based access
+        std::vector<StorePath> vec;
+        for (auto & sp : storePaths) {
+            vec.push_back(StorePath{sp});
+        }
+
+        return new nix_realised_string{.str = s, .storePaths = vec};
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+void nix_realised_string_free(nix_realised_string * s)
+{
+    delete s;
+}
+
+size_t nix_realised_string_get_buffer_size(nix_realised_string * s)
+{
+    return s->str.size();
+}
+
+const char * nix_realised_string_get_buffer_start(nix_realised_string * s)
+{
+    return s->str.data();
+}
+
+size_t nix_realised_string_get_store_path_count(nix_realised_string * s)
+{
+    return s->storePaths.size();
+}
+
+const StorePath * nix_realised_string_get_store_path(nix_realised_string * s, size_t i)
+{
+    return &s->storePaths[i];
+}

src/libexpr-c/nix_api_value.h

@@ -0,0 +1,537 @@
+#ifndef NIX_API_VALUE_H
+#define NIX_API_VALUE_H
+
+/** @addtogroup libexpr
+ * @{
+ */
+/** @file
+ * @brief libexpr C bindings dealing with values
+ */
+
+#include "nix_api_util.h"
+#include "nix_api_store.h"
+#include "stdbool.h"
+#include "stddef.h"
+#include "stdint.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+// cffi start
+
+// Type definitions
+typedef enum {
+    NIX_TYPE_THUNK,
+    NIX_TYPE_INT,
+    NIX_TYPE_FLOAT,
+    NIX_TYPE_BOOL,
+    NIX_TYPE_STRING,
+    NIX_TYPE_PATH,
+    NIX_TYPE_NULL,
+    NIX_TYPE_ATTRS,
+    NIX_TYPE_LIST,
+    NIX_TYPE_FUNCTION,
+    NIX_TYPE_EXTERNAL
+} ValueType;
+
+// forward declarations
+typedef struct nix_value nix_value;
+typedef struct EvalState EvalState;
+
+[[deprecated("use nix_value instead")]] typedef nix_value Value;
+
+// type defs
+/** @brief Stores an under-construction set of bindings
+ * @ingroup value_manip
+ *
+ * Do not reuse.
+ * @see nix_make_bindings_builder, nix_bindings_builder_free, nix_make_attrs
+ * @see nix_bindings_builder_insert
+ */
+typedef struct BindingsBuilder BindingsBuilder;
+
+/** @brief Stores an under-construction list
+ * @ingroup value_manip
+ *
+ * Do not reuse.
+ * @see nix_make_list_builder, nix_list_builder_free, nix_make_list
+ * @see nix_list_builder_insert
+ */
+typedef struct ListBuilder ListBuilder;
+
+/** @brief PrimOp function
+ * @ingroup primops
+ *
+ * Owned by the GC
+ * @see nix_alloc_primop, nix_init_primop
+ */
+typedef struct PrimOp PrimOp;
+/** @brief External Value
+ * @ingroup Externals
+ *
+ * Owned by the GC
+ */
+typedef struct ExternalValue ExternalValue;
+
+/** @brief String without placeholders, and realised store paths
+ */
+typedef struct nix_realised_string nix_realised_string;
+
+/** @defgroup primops
+ * @brief Create your own primops
+ * @{
+ */
+/** @brief Function pointer for primops
+ *
+ * When you want to return an error, call nix_set_err_msg(context, NIX_ERR_UNKNOWN, "your error message here").
+ *
+ * @param[in] user_data Arbitrary data that was initially supplied to nix_alloc_primop
+ * @param[out] context Stores error information.
+ * @param[in] state Evaluator state
+ * @param[in] args list of arguments. Note that these can be thunks and should be forced using nix_value_force before
+ * use.
+ * @param[out] ret return value
+ * @see nix_alloc_primop, nix_init_primop
+ */
+typedef void (*PrimOpFun)(
+    void * user_data, nix_c_context * context, EvalState * state, nix_value ** args, nix_value * ret);
+
+/** @brief Allocate a PrimOp
+ *
+ * Owned by the garbage collector.
+ * Use nix_gc_decref() when you're done with the returned PrimOp.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] fun callback
+ * @param[in] arity expected number of function arguments
+ * @param[in] name function name
+ * @param[in] args array of argument names, NULL-terminated
+ * @param[in] doc optional, documentation for this primop
+ * @param[in] user_data optional, arbitrary data, passed to the callback when it's called
+ * @return primop, or null in case of errors
+ * @see nix_init_primop
+ */
+PrimOp * nix_alloc_primop(
+    nix_c_context * context,
+    PrimOpFun fun,
+    int arity,
+    const char * name,
+    const char ** args,
+    const char * doc,
+    void * user_data);
+
+/** @brief add a primop to the `builtins` attribute set
+ *
+ * Only applies to States created after this call.
+ *
+ * Moves your PrimOp content into the global evaluator
+ * registry, meaning your input PrimOp pointer is no longer usable.
+ * You are free to remove your references to it,
+ * after which it will be garbage collected.
+ *
+ * @param[out] context Optional, stores error information
+ * @return primop, or null in case of errors
+ *
+ */
+nix_err nix_register_primop(nix_c_context * context, PrimOp * primOp);
+/** @} */
+
+// Function prototypes
+
+/** @brief Allocate a Nix value
+ *
+ * Owned by the GC. Use nix_gc_decref() when you're done with the pointer
+ * @param[out] context Optional, stores error information
+ * @param[in] state nix evaluator state
+ * @return value, or null in case of errors
+ *
+ */
+nix_value * nix_alloc_value(nix_c_context * context, EvalState * state);
+
+/**
+ * @brief Increment the garbage collector reference counter for the given `nix_value`.
+ *
+ * The Nix language evaluator C API keeps track of alive objects by reference counting.
+ * When you're done with a refcounted pointer, call nix_value_decref().
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] value The object to keep alive
+ */
+nix_err nix_value_incref(nix_c_context * context, nix_value * value);
+
+/**
+ * @brief Decrement the garbage collector reference counter for the given object
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] value The object to stop referencing
+ */
+nix_err nix_value_decref(nix_c_context * context, nix_value * value);
+
+/** @addtogroup value_manip Manipulating values
+ * @brief Functions to inspect and change Nix language values, represented by nix_value.
+ * @{
+ */
+/** @anchor getters
+ * @name Getters
+ */
+/**@{*/
+/** @brief Get value type
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return type of nix value
+ */
+ValueType nix_get_type(nix_c_context * context, const nix_value * value);
+
+/** @brief Get type name of value as defined in the evaluator
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return type name, owned string
+ * @todo way to free the result
+ */
+const char * nix_get_typename(nix_c_context * context, const nix_value * value);
+
+/** @brief Get boolean value
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return true or false, error info via context
+ */
+bool nix_get_bool(nix_c_context * context, const nix_value * value);
+
+/** @brief Get the raw string
+ *
+ * This may contain placeholders.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @param[in] callback Called with the string value.
+ * @param[in] user_data optional, arbitrary data, passed to the callback when it's called.
+ * @return string
+ * @return error code, NIX_OK on success.
+ */
+nix_err
+nix_get_string(nix_c_context * context, const nix_value * value, nix_get_string_callback callback, void * user_data);
+
+/** @brief Get path as string
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return string
+ * @return NULL in case of error.
+ */
+const char * nix_get_path_string(nix_c_context * context, const nix_value * value);
+
+/** @brief Get the length of a list
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return length of list, error info via context
+ */
+unsigned int nix_get_list_size(nix_c_context * context, const nix_value * value);
+
+/** @brief Get the element count of an attrset
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return attrset element count, error info via context
+ */
+unsigned int nix_get_attrs_size(nix_c_context * context, const nix_value * value);
+
+/** @brief Get float value in 64 bits
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return float contents, error info via context
+ */
+double nix_get_float(nix_c_context * context, const nix_value * value);
+
+/** @brief Get int value
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return int contents, error info via context
+ */
+int64_t nix_get_int(nix_c_context * context, const nix_value * value);
+
+/** @brief Get external reference
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return reference to external, NULL in case of error
+ */
+ExternalValue * nix_get_external(nix_c_context * context, nix_value *);
+
+/** @brief Get the ix'th element of a list
+ *
+ * Owned by the GC. Use nix_gc_decref when you're done with the pointer
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @param[in] state nix evaluator state
+ * @param[in] ix list element to get
+ * @return value, NULL in case of errors
+ */
+nix_value * nix_get_list_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int ix);
+
+/** @brief Get an attr by name
+ *
+ * Owned by the GC. Use nix_gc_decref when you're done with the pointer
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @param[in] state nix evaluator state
+ * @param[in] name attribute name
+ * @return value, NULL in case of errors
+ */
+nix_value * nix_get_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name);
+
+/** @brief Check if an attribute name exists on a value
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @param[in] state nix evaluator state
+ * @param[in] name attribute name
+ * @return value, error info via context
+ */
+bool nix_has_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name);
+
+/** @brief Get an attribute by index in the sorted bindings
+ *
+ * Also gives you the name.
+ *
+ * Owned by the GC. Use nix_gc_decref when you're done with the pointer
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @param[in] state nix evaluator state
+ * @param[in] i attribute index
+ * @param[out] name will store a pointer to the attribute name
+ * @return value, NULL in case of errors
+ */
+nix_value * nix_get_attr_byidx(
+    nix_c_context * context, const nix_value * value, EvalState * state, unsigned int i, const char ** name);
+
+/** @brief Get an attribute name by index in the sorted bindings
+ *
+ * Useful when you want the name but want to avoid evaluation.
+ *
+ * Owned by the nix EvalState
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @param[in] state nix evaluator state
+ * @param[in] i attribute index
+ * @return name, NULL in case of errors
+ */
+const char *
+nix_get_attr_name_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int i);
+
+/**@}*/
+/** @name Initializers
+ *
+ * Values are typically "returned" by initializing already allocated memory that serves as the return value.
+ * For this reason, the construction of values is not tied their allocation.
+ * Nix is a language with immutable values. Respect this property by only initializing Values once; and only initialize
+ * Values that are meant to be initialized by you. Failing to adhere to these rules may lead to undefined behavior.
+ */
+/**@{*/
+/** @brief Set boolean value
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] b the boolean value
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_init_bool(nix_c_context * context, nix_value * value, bool b);
+
+/** @brief Set a string
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] str the string, copied
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_init_string(nix_c_context * context, nix_value * value, const char * str);
+
+/** @brief Set a path
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] str the path string, copied
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_init_path_string(nix_c_context * context, EvalState * s, nix_value * value, const char * str);
+
+/** @brief Set a float
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] d the float, 64-bits
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_init_float(nix_c_context * context, nix_value * value, double d);
+
+/** @brief Set an int
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] i the int
+ * @return error code, NIX_OK on success.
+ */
+
+nix_err nix_init_int(nix_c_context * context, nix_value * value, int64_t i);
+/** @brief Set null
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_init_null(nix_c_context * context, nix_value * value);
+
+/** @brief Set the value to a thunk that will perform a function application when needed.
+ *
+ * Thunks may be put into attribute sets and lists to perform some computation lazily; on demand.
+ * However, note that in some places, a thunk must not be returned, such as in the return value of a PrimOp.
+ * In such cases, you may use nix_value_call() instead (but note the different argument order).
+ *
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] fn function to call
+ * @param[in] arg argument to pass
+ * @return error code, NIX_OK on successful initialization.
+ * @see nix_value_call() for a similar function that performs the call immediately and only stores the return value.
+ *      Note the different argument order.
+ */
+nix_err nix_init_apply(nix_c_context * context, nix_value * value, nix_value * fn, nix_value * arg);
+
+/** @brief Set an external value
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] val the external value to set. Will be GC-referenced by the value.
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_init_external(nix_c_context * context, nix_value * value, ExternalValue * val);
+
+/** @brief Create a list from a list builder
+ * @param[out] context Optional, stores error information
+ * @param[in] list_builder list builder to use. Make sure to unref this afterwards.
+ * @param[out] value Nix value to modify
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_make_list(nix_c_context * context, ListBuilder * list_builder, nix_value * value);
+
+/** @brief Create a list builder
+ * @param[out] context Optional, stores error information
+ * @param[in] state nix evaluator state
+ * @param[in] capacity how many bindings you'll add. Don't exceed.
+ * @return owned reference to a list builder. Make sure to unref when you're done.
+ */
+ListBuilder * nix_make_list_builder(nix_c_context * context, EvalState * state, size_t capacity);
+
+/** @brief Insert bindings into a builder
+ * @param[out] context Optional, stores error information
+ * @param[in] list_builder ListBuilder to insert into
+ * @param[in] index index to manipulate
+ * @param[in] value value to insert
+ * @return error code, NIX_OK on success.
+ */
+nix_err
+nix_list_builder_insert(nix_c_context * context, ListBuilder * list_builder, unsigned int index, nix_value * value);
+
+/** @brief Free a list builder
+ *
+ * Does not fail.
+ * @param[in] builder the builder to free
+ */
+void nix_list_builder_free(ListBuilder * list_builder);
+
+/** @brief Create an attribute set from a bindings builder
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] b bindings builder to use. Make sure to unref this afterwards.
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_make_attrs(nix_c_context * context, nix_value * value, BindingsBuilder * b);
+
+/** @brief Set primop
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] op primop, will be gc-referenced by the value
+ * @see nix_alloc_primop
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_init_primop(nix_c_context * context, nix_value * value, PrimOp * op);
+/** @brief Copy from another value
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] source value to copy from
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_copy_value(nix_c_context * context, nix_value * value, const nix_value * source);
+/**@}*/
+
+/** @brief Create a bindings builder
+* @param[out] context Optional, stores error information
+* @param[in] state nix evaluator state
+* @param[in] capacity how many bindings you'll add. Don't exceed.
+* @return owned reference to a bindings builder. Make sure to unref when you're
+done.
+*/
+BindingsBuilder * nix_make_bindings_builder(nix_c_context * context, EvalState * state, size_t capacity);
+
+/** @brief Insert bindings into a builder
+ * @param[out] context Optional, stores error information
+ * @param[in] builder BindingsBuilder to insert into
+ * @param[in] name attribute name, only used for the duration of the call.
+ * @param[in] value value to give the binding
+ * @return error code, NIX_OK on success.
+ */
+nix_err
+nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * builder, const char * name, nix_value * value);
+
+/** @brief Free a bindings builder
+ *
+ * Does not fail.
+ * @param[in] builder the builder to free
+ */
+void nix_bindings_builder_free(BindingsBuilder * builder);
+/**@}*/
+
+/** @brief Realise a string context.
+ *
+ * This will
+ *  - realise the store paths referenced by the string's context, and
+ *  - perform the replacement of placeholders.
+ *  - create temporary garbage collection roots for the store paths, for
+ *    the lifetime of the current process.
+ *  - log to stderr
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value, which must be a string
+ * @param[in] state Nix evaluator state
+ * @param[in] isIFD If true, disallow derivation outputs if setting `allow-import-from-derivation` is false.
+                    You should set this to true when this call is part of a primop.
+                    You should set this to false when building for your application's purpose.
+ * @return NULL if failed, are a new nix_realised_string, which must be freed with nix_realised_string_free
+ */
+nix_realised_string * nix_string_realise(nix_c_context * context, EvalState * state, nix_value * value, bool isIFD);
+
+/** @brief Start of the string
+ * @param[in] realised_string
+ * @return pointer to the start of the string. It may not be null-terminated.
+ */
+const char * nix_realised_string_get_buffer_start(nix_realised_string * realised_string);
+
+/** @brief Length of the string
+ * @param[in] realised_string
+ * @return length of the string in bytes
+ */
+size_t nix_realised_string_get_buffer_size(nix_realised_string * realised_string);
+
+/** @brief Number of realised store paths
+ * @param[in] realised_string
+ * @return number of realised store paths that were referenced by the string via its context
+ */
+size_t nix_realised_string_get_store_path_count(nix_realised_string * realised_string);
+
+/** @brief Get a store path. The store paths are stored in an arbitrary order.
+ * @param[in] realised_string
+ * @param[in] index index of the store path, must be less than the count
+ * @return store path
+ */
+const StorePath * nix_realised_string_get_store_path(nix_realised_string * realised_string, size_t index);
+
+/** @brief Free a realised string
+ * @param[in] realised_string
+ */
+void nix_realised_string_free(nix_realised_string * realised_string);
+
+// cffi end
+#ifdef __cplusplus
+}
+#endif
+
+/** @} */
+#endif // NIX_API_VALUE_H

src/libexpr-c/package.nix

@@ -0,0 +1,74 @@
+{ lib
+, stdenv
+, mkMesonDerivation
+
+, meson
+, ninja
+, pkg-config
+
+, nix-store-c
+, nix-expr
+
+# Configuration Options
+
+, version
+}:
+
+let
+  inherit (lib) fileset;
+in
+
+mkMesonDerivation (finalAttrs: {
+  pname = "nix-expr-c";
+  inherit version;
+
+  workDir = ./.;
+  fileset = fileset.unions [
+    ../../build-utils-meson
+    ./build-utils-meson
+    ../../.version
+    ./.version
+    ./meson.build
+    # ./meson.options
+    (fileset.fileFilter (file: file.hasExt "cc") ./.)
+    (fileset.fileFilter (file: file.hasExt "hh") ./.)
+    (fileset.fileFilter (file: file.hasExt "h") ./.)
+  ];
+
+  outputs = [ "out" "dev" ];
+
+  nativeBuildInputs = [
+    meson
+    ninja
+    pkg-config
+  ];
+
+  propagatedBuildInputs = [
+    nix-store-c
+    nix-expr
+  ];
+
+  preConfigure =
+    # "Inline" .version so it's not a symlink, and includes the suffix.
+    # Do the meson utils, without modification.
+    ''
+      chmod u+w ./.version
+      echo ${version} > ../../.version
+    '';
+
+  mesonFlags = [
+  ];
+
+  env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) {
+    LDFLAGS = "-fuse-ld=gold";
+  };
+
+  separateDebugInfo = !stdenv.hostPlatform.isStatic;
+
+  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
+
+  meta = {
+    platforms = lib.platforms.unix ++ lib.platforms.windows;
+  };
+
+})

src/libexpr/.version

@@ -0,0 +1 @@
+../../.version

src/libexpr/attr-path.cc

@@ -72,11 +72,11 @@ std::pair<Value *, PosIdx> findAlongAttrPath(EvalState & state, const std::strin
             if (attr.empty())
                 throw Error("empty attribute name in selection path '%1%'", attrPath);
 
-            Bindings::iterator a = v->attrs->find(state.symbols.create(attr));
-            if (a == v->attrs->end()) {
+            auto a = v->attrs()->get(state.symbols.create(attr));
+            if (!a) {
                 std::set<std::string> attrNames;
-                for (auto & attr : *v->attrs)
-                    attrNames.insert(state.symbols[attr.name]);
+                for (auto & attr : *v->attrs())
+                    attrNames.insert(std::string(state.symbols[attr.name]));
 
                 auto suggestions = Suggestions::bestMatches(attrNames, attr);
                 throw AttrPathNotFound(suggestions, "attribute '%1%' in selection path '%2%' not found", attr, attrPath);
@@ -134,7 +134,7 @@ std::pair<SourcePath, uint32_t> findPackageFilename(EvalState & state, Value & v
         return {SourcePath{path.accessor, CanonPath(fn.substr(0, colon))}, lineno};
     } catch (std::invalid_argument & e) {
         fail();
-        abort();
+        unreachable();
     }
 }
 

src/libexpr/attr-set.cc

@@ -23,23 +23,6 @@ Bindings * EvalState::allocBindings(size_t capacity)
 }
 
 
-/* Create a new attribute named 'name' on an existing attribute set stored
-   in 'vAttrs' and return the newly allocated Value which is associated with
-   this attribute. */
-Value * EvalState::allocAttr(Value & vAttrs, Symbol name)
-{
-    Value * v = allocValue();
-    vAttrs.attrs->push_back(Attr(name, v));
-    return v;
-}
-
-
-Value * EvalState::allocAttr(Value & vAttrs, std::string_view name)
-{
-    return allocAttr(vAttrs, symbols.create(name));
-}
-
-
 Value & BindingsBuilder::alloc(Symbol name, PosIdx pos)
 {
     auto value = state.allocValue();

src/libexpr/attr-set.hh

@@ -5,7 +5,6 @@
 #include "symbol-table.hh"
 
 #include <algorithm>
-#include <optional>
 
 namespace nix {
 
@@ -28,9 +27,9 @@ struct Attr
     Attr(Symbol name, Value * value, PosIdx pos = noPos)
         : name(name), pos(pos), value(value) { };
     Attr() { };
-    bool operator < (const Attr & a) const
+    auto operator <=> (const Attr & a) const
     {
-        return name < a.name;
+        return name <=> a.name;
     }
 };
 
@@ -65,24 +64,26 @@ public:
 
     typedef Attr * iterator;
 
+    typedef const Attr * const_iterator;
+
     void push_back(const Attr & attr)
     {
         assert(size_ < capacity_);
         attrs[size_++] = attr;
     }
 
-    iterator find(Symbol name)
+    const_iterator find(Symbol name) const
     {
         Attr key(name, 0);
-        iterator i = std::lower_bound(begin(), end(), key);
+        const_iterator i = std::lower_bound(begin(), end(), key);
         if (i != end() && i->name == name) return i;
         return end();
     }
 
-    Attr * get(Symbol name)
+    const Attr * get(Symbol name) const
     {
         Attr key(name, 0);
-        iterator i = std::lower_bound(begin(), end(), key);
+        const_iterator i = std::lower_bound(begin(), end(), key);
         if (i != end() && i->name == name) return &*i;
         return nullptr;
     }
@@ -90,14 +91,22 @@ public:
     iterator begin() { return &attrs[0]; }
     iterator end() { return &attrs[size_]; }
 
+    const_iterator begin() const { return &attrs[0]; }
+    const_iterator end() const { return &attrs[size_]; }
+
     Attr & operator[](size_t pos)
     {
         return attrs[pos];
     }
 
+    const Attr & operator[](size_t pos) const
+    {
+        return attrs[pos];
+    }
+
     void sort();
 
-    size_t capacity() { return capacity_; }
+    size_t capacity() const { return capacity_; }
 
     /**
      * Returns the attributes in lexicographically sorted order.
@@ -166,6 +175,20 @@ public:
     {
         return bindings;
     }
+
+    size_t capacity()
+    {
+        return bindings->capacity();
+    }
+
+    void grow(Bindings * newBindings)
+    {
+        for (auto & i : *bindings)
+            newBindings->push_back(i);
+        bindings = newBindings;
+    }
+
+    friend struct ExprAttrs;
 };
 
 }

src/libexpr/build-utils-meson

@@ -0,0 +1 @@
+../../build-utils-meson

src/libexpr/call-flake.nix

@@ -4,7 +4,7 @@
 lockFileStr:
 
 # A mapping of lock file node IDs to { sourceInfo, subdir } attrsets,
-# with sourceInfo.outPath providing an InputAccessor to a previously
+# with sourceInfo.outPath providing an SourceAccessor to a previously
 # fetched tree. This is necessary for possibly unlocked inputs, in
 # particular the root input, but also --override-inputs pointing to
 # unlocked trees.

src/libexpr/eval-cache.cc

@@ -7,6 +7,25 @@
 
 namespace nix::eval_cache {
 
+CachedEvalError::CachedEvalError(ref<AttrCursor> cursor, Symbol attr)
+    : EvalError(cursor->root->state, "cached failure of attribute '%s'", cursor->getAttrPathStr(attr))
+    , cursor(cursor), attr(attr)
+{ }
+
+void CachedEvalError::force()
+{
+    auto & v = cursor->forceValue();
+
+    if (v.type() == nAttrs) {
+        auto a = v.attrs()->get(this->attr);
+
+        state.forceValue(*a->value, a->pos);
+    }
+
+    // Shouldn't happen.
+    throw EvalError(state, "evaluation of cached failed attribute '%s' unexpectedly succeeded", cursor->getAttrPathStr(attr));
+}
+
 static const char * schema = R"sql(
 create table if not exists Attributes (
     parent      integer not null,
@@ -76,7 +95,7 @@ struct AttrDb
     {
         try {
             auto state(_state->lock());
-            if (!failed)
+            if (!failed && state->txn->active)
                 state->txn->commit();
             state->txn.reset();
         } catch (...) {
@@ -206,7 +225,7 @@ struct AttrDb
                 (key.first)
                 (symbols[key.second])
                 (AttrType::ListOfStrings)
-                (concatStringsSep("\t", l)).exec();
+                (dropEmptyInitThenConcatStringsSep("\t", l)).exec();
 
             return state->db.getLastInsertedRowId();
         });
@@ -387,7 +406,7 @@ Value & AttrCursor::getValue()
         if (parent) {
             auto & vParent = parent->first->getValue();
             root->state.forceAttrs(vParent, noPos, "while searching for an attribute");
-            auto attr = vParent.attrs->get(parent->second);
+            auto attr = vParent.attrs()->get(parent->second);
             if (!attr)
                 throw Error("attribute '%s' is unexpectedly missing", getAttrPathStr());
             _value = allocRootValue(attr->value);
@@ -416,12 +435,12 @@ std::vector<Symbol> AttrCursor::getAttrPath(Symbol name) const
 
 std::string AttrCursor::getAttrPathStr() const
 {
-    return concatStringsSep(".", root->state.symbols.resolve(getAttrPath()));
+    return dropEmptyInitThenConcatStringsSep(".", root->state.symbols.resolve(getAttrPath()));
 }
 
 std::string AttrCursor::getAttrPathStr(Symbol name) const
 {
-    return concatStringsSep(".", root->state.symbols.resolve(getAttrPath(name)));
+    return dropEmptyInitThenConcatStringsSep(".", root->state.symbols.resolve(getAttrPath(name)));
 }
 
 Value & AttrCursor::forceValue()
@@ -448,9 +467,9 @@ Value & AttrCursor::forceValue()
             cachedValue = {root->db->setString(getKey(), path.abs()), string_t{path.abs(), {}}};
         }
         else if (v.type() == nBool)
-            cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean};
+            cachedValue = {root->db->setBool(getKey(), v.boolean()), v.boolean()};
         else if (v.type() == nInt)
-            cachedValue = {root->db->setInt(getKey(), v.integer), int_t{v.integer}};
+            cachedValue = {root->db->setInt(getKey(), v.integer()), int_t{v.integer()}};
         else if (v.type() == nAttrs)
             ; // FIXME: do something?
         else
@@ -465,12 +484,12 @@ Suggestions AttrCursor::getSuggestionsForAttr(Symbol name)
     auto attrNames = getAttrs();
     std::set<std::string> strAttrNames;
     for (auto & name : attrNames)
-        strAttrNames.insert(root->state.symbols[name]);
+        strAttrNames.insert(std::string(root->state.symbols[name]));
 
     return Suggestions::bestMatches(strAttrNames, root->state.symbols[name]);
 }
 
-std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErrors)
+std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name)
 {
     if (root->db) {
         if (!cachedValue)
@@ -487,12 +506,9 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErro
                 if (attr) {
                     if (std::get_if<missing_t>(&attr->second))
                         return nullptr;
-                    else if (std::get_if<failed_t>(&attr->second)) {
-                        if (forceErrors)
-                            debug("reevaluating failed cached attribute '%s'", getAttrPathStr(name));
-                        else
-                            throw CachedEvalError(root->state, "cached failure of attribute '%s'", getAttrPathStr(name));
-                    } else
+                    else if (std::get_if<failed_t>(&attr->second))
+                        throw CachedEvalError(ref(shared_from_this()), name);
+                    else
                         return std::make_shared<AttrCursor>(root,
                             std::make_pair(shared_from_this(), name), nullptr, std::move(attr));
                 }
@@ -510,7 +526,7 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErro
         return nullptr;
         //error<TypeError>("'%s' is not an attribute set", getAttrPathStr()).debugThrow();
 
-    auto attr = v.attrs->get(name);
+    auto attr = v.attrs()->get(name);
 
     if (!attr) {
         if (root->db) {
@@ -537,9 +553,9 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(std::string_view name)
     return maybeGetAttr(root->state.symbols.create(name));
 }
 
-ref<AttrCursor> AttrCursor::getAttr(Symbol name, bool forceErrors)
+ref<AttrCursor> AttrCursor::getAttr(Symbol name)
 {
-    auto p = maybeGetAttr(name, forceErrors);
+    auto p = maybeGetAttr(name);
     if (!p)
         throw Error("attribute '%s' does not exist", getAttrPathStr(name));
     return ref(p);
@@ -550,11 +566,11 @@ ref<AttrCursor> AttrCursor::getAttr(std::string_view name)
     return getAttr(root->state.symbols.create(name));
 }
 
-OrSuggestions<ref<AttrCursor>> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force)
+OrSuggestions<ref<AttrCursor>> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath)
 {
     auto res = shared_from_this();
     for (auto & attr : attrPath) {
-        auto child = res->maybeGetAttr(attr, force);
+        auto child = res->maybeGetAttr(attr);
         if (!child) {
             auto suggestions = res->getSuggestionsForAttr(attr);
             return OrSuggestions<ref<AttrCursor>>::failed(suggestions);
@@ -581,7 +597,7 @@ std::string AttrCursor::getString()
     auto & v = forceValue();
 
     if (v.type() != nString && v.type() != nPath)
-        root->state.error<TypeError>("'%s' is not a string but %s", getAttrPathStr()).debugThrow();
+        root->state.error<TypeError>("'%s' is not a string but %s", getAttrPathStr(), showType(v)).debugThrow();
 
     return v.type() == nString ? v.c_str() : v.path().to_string();
 }
@@ -630,7 +646,7 @@ string_t AttrCursor::getStringWithContext()
     else if (v.type() == nPath)
         return {v.path().to_string(), {}};
     else
-        root->state.error<TypeError>("'%s' is not a string but %s", getAttrPathStr()).debugThrow();
+        root->state.error<TypeError>("'%s' is not a string but %s", getAttrPathStr(), showType(v)).debugThrow();
 }
 
 bool AttrCursor::getBool()
@@ -652,7 +668,7 @@ bool AttrCursor::getBool()
     if (v.type() != nBool)
         root->state.error<TypeError>("'%s' is not a Boolean", getAttrPathStr()).debugThrow();
 
-    return v.boolean;
+    return v.boolean();
 }
 
 NixInt AttrCursor::getInt()
@@ -674,7 +690,7 @@ NixInt AttrCursor::getInt()
     if (v.type() != nInt)
         root->state.error<TypeError>("'%s' is not an integer", getAttrPathStr()).debugThrow();
 
-    return v.integer;
+    return v.integer();
 }
 
 std::vector<std::string> AttrCursor::getListOfStrings()
@@ -730,7 +746,7 @@ std::vector<Symbol> AttrCursor::getAttrs()
         root->state.error<TypeError>("'%s' is not an attribute set", getAttrPathStr()).debugThrow();
 
     std::vector<Symbol> attrs;
-    for (auto & attr : *getValue().attrs)
+    for (auto & attr : *getValue().attrs())
         attrs.push_back(attr.name);
     std::sort(attrs.begin(), attrs.end(), [&](Symbol a, Symbol b) {
         std::string_view sa = root->state.symbols[a], sb = root->state.symbols[b];
@@ -751,8 +767,9 @@ bool AttrCursor::isDerivation()
 
 StorePath AttrCursor::forceDerivation()
 {
-    auto aDrvPath = getAttr(root->state.sDrvPath, true);
+    auto aDrvPath = getAttr(root->state.sDrvPath);
     auto drvPath = root->state.store->parseStorePath(aDrvPath->getString());
+    drvPath.requireDerivation();
     if (!root->state.store->isValidPath(drvPath) && !settings.readOnlyMode) {
         /* The eval cache contains 'drvPath', but the actual path has
            been garbage-collected. So force it to be regenerated. */

src/libexpr/eval-cache.hh

@@ -10,14 +10,28 @@
 
 namespace nix::eval_cache {
 
-MakeError(CachedEvalError, EvalError);
-
 struct AttrDb;
 class AttrCursor;
 
+struct CachedEvalError : EvalError
+{
+    const ref<AttrCursor> cursor;
+    const Symbol attr;
+
+    CachedEvalError(ref<AttrCursor> cursor, Symbol attr);
+
+    /**
+     * Evaluate this attribute, which should result in a regular
+     * `EvalError` exception being thrown.
+     */
+    [[noreturn]]
+    void force();
+};
+
 class EvalCache : public std::enable_shared_from_this<EvalCache>
 {
     friend class AttrCursor;
+    friend struct CachedEvalError;
 
     std::shared_ptr<AttrDb> db;
     EvalState & state;
@@ -73,6 +87,7 @@ typedef std::variant<
 class AttrCursor : public std::enable_shared_from_this<AttrCursor>
 {
     friend class EvalCache;
+    friend struct CachedEvalError;
 
     ref<EvalCache> root;
     typedef std::optional<std::pair<std::shared_ptr<AttrCursor>, Symbol>> Parent;
@@ -102,11 +117,11 @@ public:
 
     Suggestions getSuggestionsForAttr(Symbol name);
 
-    std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name, bool forceErrors = false);
+    std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name);
 
     std::shared_ptr<AttrCursor> maybeGetAttr(std::string_view name);
 
-    ref<AttrCursor> getAttr(Symbol name, bool forceErrors = false);
+    ref<AttrCursor> getAttr(Symbol name);
 
     ref<AttrCursor> getAttr(std::string_view name);
 
@@ -114,7 +129,7 @@ public:
      * Get an attribute along a chain of attrsets. Note that this does
      * not auto-call functors or functions.
      */
-    OrSuggestions<ref<AttrCursor>> findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force = false);
+    OrSuggestions<ref<AttrCursor>> findAlongAttrPath(const std::vector<Symbol> & attrPath);
 
     std::string getString();
 

src/libexpr/eval-error.cc

@@ -27,8 +27,7 @@ EvalErrorBuilder<T> & EvalErrorBuilder<T>::atPos(Value & value, PosIdx fallback)
 template<class T>
 EvalErrorBuilder<T> & EvalErrorBuilder<T>::withTrace(PosIdx pos, const std::string_view text)
 {
-    error.err.traces.push_front(
-        Trace{.pos = error.state.positions[pos], .hint = HintFmt(std::string(text))});
+    error.addTrace(error.state.positions[pos], text);
     return *this;
 }
 
@@ -71,15 +70,17 @@ EvalErrorBuilder<T>::addTrace(PosIdx pos, std::string_view formatString, const A
     return *this;
 }
 
+template<class T>
+EvalErrorBuilder<T> & EvalErrorBuilder<T>::setIsFromExpr()
+{
+    error.err.isFromExpr = true;
+    return *this;
+}
+
 template<class T>
 void EvalErrorBuilder<T>::debugThrow()
 {
-    if (error.state.debugRepl && !error.state.debugTraces.empty()) {
-        const DebugTrace & last = error.state.debugTraces.front();
-        const Env * env = &last.env;
-        const Expr * expr = &last.expr;
-        error.state.runDebugRepl(&error, *env, *expr);
-    }
+    error.state.runDebugRepl(&error);
 
     // `EvalState` is the only class that can construct an `EvalErrorBuilder`,
     // and it does so in dynamic storage. This is the final method called on
@@ -91,6 +92,15 @@ void EvalErrorBuilder<T>::debugThrow()
     throw error;
 }
 
+template<class T>
+void EvalErrorBuilder<T>::panic()
+{
+    logError(error.info());
+    printError("This is a bug! An unexpected condition occurred, causing the Nix evaluator to have to stop. If you could share a reproducible example or a core dump, please open an issue at https://github.com/NixOS/nix/issues");
+    abort();
+}
+
+template class EvalErrorBuilder<EvalBaseError>;
 template class EvalErrorBuilder<EvalError>;
 template class EvalErrorBuilder<AssertionError>;
 template class EvalErrorBuilder<ThrownError>;
@@ -99,7 +109,6 @@ template class EvalErrorBuilder<TypeError>;
 template class EvalErrorBuilder<UndefinedVarError>;
 template class EvalErrorBuilder<MissingArgumentError>;
 template class EvalErrorBuilder<InfiniteRecursionError>;
-template class EvalErrorBuilder<CachedEvalError>;
 template class EvalErrorBuilder<InvalidPathError>;
 
 }

src/libexpr/eval-error.hh

@@ -1,7 +1,5 @@
 #pragma once
 
-#include <algorithm>
-
 #include "error.hh"
 #include "pos-idx.hh"
 
@@ -15,27 +13,39 @@ class EvalState;
 template<class T>
 class EvalErrorBuilder;
 
-class EvalError : public Error
+/**
+ * Base class for all errors that occur during evaluation.
+ *
+ * Most subclasses should inherit from `EvalError` instead of this class.
+ */
+class EvalBaseError : public Error
 {
     template<class T>
     friend class EvalErrorBuilder;
 public:
     EvalState & state;
 
-    EvalError(EvalState & state, ErrorInfo && errorInfo)
+    EvalBaseError(EvalState & state, ErrorInfo && errorInfo)
         : Error(errorInfo)
         , state(state)
     {
     }
 
     template<typename... Args>
-    explicit EvalError(EvalState & state, const std::string & formatString, const Args &... formatArgs)
+    explicit EvalBaseError(EvalState & state, const std::string & formatString, const Args &... formatArgs)
         : Error(formatString, formatArgs...)
         , state(state)
     {
     }
 };
 
+/**
+ * `EvalError` is the base class for almost all errors that occur during evaluation.
+ *
+ * All instances of `EvalError` should show a degree of purity that allows them to be
+ * cached in pure mode. This means that they should not depend on the configuration or the overall environment.
+ */
+MakeError(EvalError, EvalBaseError);
 MakeError(ParseError, Error);
 MakeError(AssertionError, EvalError);
 MakeError(ThrownError, AssertionError);
@@ -43,7 +53,6 @@ MakeError(Abort, EvalError);
 MakeError(TypeError, EvalError);
 MakeError(UndefinedVarError, EvalError);
 MakeError(MissingArgumentError, EvalError);
-MakeError(CachedEvalError, EvalError);
 MakeError(InfiniteRecursionError, EvalError);
 
 struct InvalidPathError : public EvalError
@@ -91,6 +100,8 @@ public:
 
     [[nodiscard, gnu::noinline]] EvalErrorBuilder<T> & addTrace(PosIdx pos, HintFmt hint);
 
+    [[nodiscard, gnu::noinline]] EvalErrorBuilder<T> & setIsFromExpr();
+
     template<typename... Args>
     [[nodiscard, gnu::noinline]] EvalErrorBuilder<T> &
     addTrace(PosIdx pos, std::string_view formatString, const Args &... formatArgs);
@@ -99,6 +110,12 @@ public:
      * Delete the `EvalErrorBuilder` and throw the underlying exception.
      */
     [[gnu::noinline, gnu::noreturn]] void debugThrow();
+
+    /**
+     * A programming error or fatal condition occurred. Abort the process for core dump and debugging.
+     * This does not print a proper backtrace, because unwinding the stack is destructive.
+     */
+    [[gnu::noinline, gnu::noreturn]] void panic();
 };
 
 }

src/libexpr/eval-gc.cc

@@ -0,0 +1,251 @@
+#include "error.hh"
+#include "environment-variables.hh"
+#include "eval-settings.hh"
+#include "config-global.hh"
+#include "serialise.hh"
+#include "eval-gc.hh"
+
+#if HAVE_BOEHMGC
+
+#  include <pthread.h>
+#  if __FreeBSD__
+#    include <pthread_np.h>
+#  endif
+
+#  include <gc/gc.h>
+#  include <gc/gc_cpp.h>
+#  include <gc/gc_allocator.h>
+
+#  include <boost/coroutine2/coroutine.hpp>
+#  include <boost/coroutine2/protected_fixedsize_stack.hpp>
+#  include <boost/context/stack_context.hpp>
+
+#endif
+
+namespace nix {
+
+#if HAVE_BOEHMGC
+/* Called when the Boehm GC runs out of memory. */
+static void * oomHandler(size_t requested)
+{
+    /* Convert this to a proper C++ exception. */
+    throw std::bad_alloc();
+}
+
+class BoehmGCStackAllocator : public StackAllocator
+{
+    boost::coroutines2::protected_fixedsize_stack stack{
+        // We allocate 8 MB, the default max stack size on NixOS.
+        // A smaller stack might be quicker to allocate but reduces the stack
+        // depth available for source filter expressions etc.
+        std::max(boost::context::stack_traits::default_size(), static_cast<std::size_t>(8 * 1024 * 1024))};
+
+    // This is specific to boost::coroutines2::protected_fixedsize_stack.
+    // The stack protection page is included in sctx.size, so we have to
+    // subtract one page size from the stack size.
+    std::size_t pfss_usable_stack_size(boost::context::stack_context & sctx)
+    {
+        return sctx.size - boost::context::stack_traits::page_size();
+    }
+
+public:
+    boost::context::stack_context allocate() override
+    {
+        auto sctx = stack.allocate();
+
+        // Stacks generally start at a high address and grow to lower addresses.
+        // Architectures that do the opposite are rare; in fact so rare that
+        // boost_routine does not implement it.
+        // So we subtract the stack size.
+        GC_add_roots(static_cast<char *>(sctx.sp) - pfss_usable_stack_size(sctx), sctx.sp);
+        return sctx;
+    }
+
+    void deallocate(boost::context::stack_context sctx) override
+    {
+        GC_remove_roots(static_cast<char *>(sctx.sp) - pfss_usable_stack_size(sctx), sctx.sp);
+        stack.deallocate(sctx);
+    }
+};
+
+static BoehmGCStackAllocator boehmGCStackAllocator;
+
+/**
+ * When a thread goes into a coroutine, we lose its original sp until
+ * control flow returns to the thread.
+ * While in the coroutine, the sp points outside the thread stack,
+ * so we can detect this and push the entire thread stack instead,
+ * as an approximation.
+ * The coroutine's stack is covered by `BoehmGCStackAllocator`.
+ * This is not an optimal solution, because the garbage is scanned when a
+ * coroutine is active, for both the coroutine and the original thread stack.
+ * However, the implementation is quite lean, and usually we don't have active
+ * coroutines during evaluation, so this is acceptable.
+ */
+void fixupBoehmStackPointer(void ** sp_ptr, void * _pthread_id)
+{
+    void *& sp = *sp_ptr;
+    auto pthread_id = reinterpret_cast<pthread_t>(_pthread_id);
+#  ifndef __APPLE__
+    pthread_attr_t pattr;
+#  endif
+    size_t osStackSize;
+    // The low address of the stack, which grows down.
+    void * osStackLimit;
+    void * osStackBase;
+
+#  ifdef __APPLE__
+    osStackSize = pthread_get_stacksize_np(pthread_id);
+    osStackLimit = pthread_get_stackaddr_np(pthread_id);
+#  else
+    if (pthread_attr_init(&pattr)) {
+        throw Error("fixupBoehmStackPointer: pthread_attr_init failed");
+    }
+#    ifdef HAVE_PTHREAD_GETATTR_NP
+    if (pthread_getattr_np(pthread_id, &pattr)) {
+        throw Error("fixupBoehmStackPointer: pthread_getattr_np failed");
+    }
+#    elif HAVE_PTHREAD_ATTR_GET_NP
+    if (!pthread_attr_init(&pattr)) {
+        throw Error("fixupBoehmStackPointer: pthread_attr_init failed");
+    }
+    if (!pthread_attr_get_np(pthread_id, &pattr)) {
+        throw Error("fixupBoehmStackPointer: pthread_attr_get_np failed");
+    }
+#    else
+#      error "Need one of `pthread_attr_get_np` or `pthread_getattr_np`"
+#    endif
+    if (pthread_attr_getstack(&pattr, &osStackLimit, &osStackSize)) {
+        throw Error("fixupBoehmStackPointer: pthread_attr_getstack failed");
+    }
+    if (pthread_attr_destroy(&pattr)) {
+        throw Error("fixupBoehmStackPointer: pthread_attr_destroy failed");
+    }
+#  endif
+    osStackBase = (char *) osStackLimit + osStackSize;
+    // NOTE: We assume the stack grows down, as it does on all architectures we support.
+    //       Architectures that grow the stack up are rare.
+    if (sp >= osStackBase || sp < osStackLimit) { // sp is outside the os stack
+        sp = osStackLimit;
+    }
+}
+
+/* Disable GC while this object lives. Used by CoroutineContext.
+ *
+ * Boehm keeps a count of GC_disable() and GC_enable() calls,
+ * and only enables GC when the count matches.
+ */
+class BoehmDisableGC
+{
+public:
+    BoehmDisableGC()
+    {
+        GC_disable();
+    };
+    ~BoehmDisableGC()
+    {
+        GC_enable();
+    };
+};
+
+static inline void initGCReal()
+{
+    /* Initialise the Boehm garbage collector. */
+
+    /* Don't look for interior pointers. This reduces the odds of
+       misdetection a bit. */
+    GC_set_all_interior_pointers(0);
+
+    /* We don't have any roots in data segments, so don't scan from
+       there. */
+    GC_set_no_dls(1);
+
+    /* Enable perf measurements. This is just a setting; not much of a
+       start of something. */
+    GC_start_performance_measurement();
+
+    GC_INIT();
+
+    GC_set_oom_fn(oomHandler);
+
+    StackAllocator::defaultAllocator = &boehmGCStackAllocator;
+
+// TODO: Remove __APPLE__ condition.
+//       Comment suggests an implementation that works on darwin and windows
+//       https://github.com/ivmai/bdwgc/issues/362#issuecomment-1936672196
+#  if GC_VERSION_MAJOR >= 8 && GC_VERSION_MINOR >= 2 && GC_VERSION_MICRO >= 4 && !defined(__APPLE__)
+    GC_set_sp_corrector(&fixupBoehmStackPointer);
+
+    if (!GC_get_sp_corrector()) {
+        printTalkative("BoehmGC on this platform does not support sp_corrector; will disable GC inside coroutines");
+        /* Used to disable GC when entering coroutines on macOS */
+        create_coro_gc_hook = []() -> std::shared_ptr<void> { return std::make_shared<BoehmDisableGC>(); };
+    }
+#  else
+#    warning \
+        "BoehmGC version does not support GC while coroutine exists. GC will be disabled inside coroutines. Consider updating bdw-gc to 8.2.4 or later."
+#  endif
+
+    /* Set the initial heap size to something fairly big (25% of
+       physical RAM, up to a maximum of 384 MiB) so that in most cases
+       we don't need to garbage collect at all.  (Collection has a
+       fairly significant overhead.)  The heap size can be overridden
+       through libgc's GC_INITIAL_HEAP_SIZE environment variable.  We
+       should probably also provide a nix.conf setting for this.  Note
+       that GC_expand_hp() causes a lot of virtual, but not physical
+       (resident) memory to be allocated.  This might be a problem on
+       systems that don't overcommit. */
+    if (!getEnv("GC_INITIAL_HEAP_SIZE")) {
+        size_t size = 32 * 1024 * 1024;
+#  if HAVE_SYSCONF && defined(_SC_PAGESIZE) && defined(_SC_PHYS_PAGES)
+        size_t maxSize = 384 * 1024 * 1024;
+        long pageSize = sysconf(_SC_PAGESIZE);
+        long pages = sysconf(_SC_PHYS_PAGES);
+        if (pageSize != -1)
+            size = (pageSize * pages) / 4; // 25% of RAM
+        if (size > maxSize)
+            size = maxSize;
+#  endif
+        debug("setting initial heap size to %1% bytes", size);
+        GC_expand_hp(size);
+    }
+}
+
+static size_t gcCyclesAfterInit = 0;
+
+size_t getGCCycles()
+{
+    assertGCInitialized();
+    return static_cast<size_t>(GC_get_gc_no()) - gcCyclesAfterInit;
+}
+
+#endif
+
+static bool gcInitialised = false;
+
+void initGC()
+{
+    if (gcInitialised)
+        return;
+
+#if HAVE_BOEHMGC
+    initGCReal();
+
+    gcCyclesAfterInit = GC_get_gc_no();
+#endif
+
+    // NIX_PATH must override the regular setting
+    // See the comment in applyConfig
+    if (auto nixPathEnv = getEnv("NIX_PATH")) {
+        globalConfig.set("nix-path", concatStringsSep(" ", EvalSettings::parseNixPath(nixPathEnv.value())));
+    }
+
+    gcInitialised = true;
+}
+
+void assertGCInitialized()
+{
+    assert(gcInitialised);
+}
+
+} // namespace nix

src/libexpr/eval-gc.hh

@@ -0,0 +1,25 @@
+#pragma once
+///@file
+
+#include <cstddef>
+
+namespace nix {
+
+/**
+ * Initialise the Boehm GC, if applicable.
+ */
+void initGC();
+
+/**
+ * Make sure `initGC` has already been called.
+ */
+void assertGCInitialized();
+
+#ifdef HAVE_BOEHMGC
+/**
+ * The number of GC cycles since initGC().
+ */
+size_t getGCCycles();
+#endif
+
+} // namespace nix

src/libexpr/eval-inline.hh

@@ -85,8 +85,8 @@ Env & EvalState::allocEnv(size_t size)
 void EvalState::forceValue(Value & v, const PosIdx pos)
 {
     if (v.isThunk()) {
-        Env * env = v.thunk.env;
-        Expr * expr = v.thunk.expr;
+        Env * env = v.payload.thunk.env;
+        Expr * expr = v.payload.thunk.expr;
         try {
             v.mkBlackhole();
             //checkInterrupt();
@@ -98,7 +98,7 @@ void EvalState::forceValue(Value & v, const PosIdx pos)
         }
     }
     else if (v.isApp())
-        callFunction(*v.app.left, *v.app.right, v, pos);
+        callFunction(*v.payload.app.left, *v.payload.app.right, v, pos);
 }
 
 

src/libexpr/eval-settings.cc

@@ -8,7 +8,7 @@ namespace nix {
 
 /* Very hacky way to parse $NIX_PATH, which is colon-separated, but
    can contain URLs (e.g. "nixpkgs=https://bla...:foo=https://"). */
-static Strings parseNixPath(const std::string & s)
+Strings EvalSettings::parseNixPath(const std::string & s)
 {
     Strings res;
 
@@ -44,10 +44,13 @@ static Strings parseNixPath(const std::string & s)
     return res;
 }
 
-EvalSettings::EvalSettings()
+EvalSettings::EvalSettings(bool & readOnlyMode, EvalSettings::LookupPathHooks lookupPathHooks)
+    : readOnlyMode{readOnlyMode}
+    , lookupPathHooks{lookupPathHooks}
 {
-    auto var = getEnv("NIX_PATH");
-    if (var) nixPath = parseNixPath(*var);
+    auto var = getEnv("NIX_ABORT_ON_WARN");
+    if (var && (var == "1" || var == "yes" || var == "true"))
+        builtinsAbortOnWarn = true;
 }
 
 Strings EvalSettings::getDefaultNixPath()
@@ -63,11 +66,9 @@ Strings EvalSettings::getDefaultNixPath()
         }
     };
 
-    if (!evalSettings.restrictEval && !evalSettings.pureEval) {
-        add(getNixDefExpr() + "/channels");
-        add(rootChannelsDir() + "/nixpkgs", "nixpkgs");
-        add(rootChannelsDir());
-    }
+    add(getNixDefExpr() + "/channels");
+    add(rootChannelsDir() + "/nixpkgs", "nixpkgs");
+    add(rootChannelsDir());
 
     return res;
 }
@@ -89,16 +90,12 @@ std::string EvalSettings::resolvePseudoUrl(std::string_view url)
         return std::string(url);
 }
 
-const std::string & EvalSettings::getCurrentSystem()
+const std::string & EvalSettings::getCurrentSystem() const
 {
     const auto & evalSystem = currentSystem.get();
     return evalSystem != "" ? evalSystem : settings.thisSystem.get();
 }
 
-EvalSettings evalSettings;
-
-static GlobalConfig::Register rEvalSettings(&evalSettings);
-
 Path getNixDefExpr()
 {
     return settings.useXDGBaseDirectories

src/libexpr/eval-settings.hh

@@ -2,36 +2,108 @@
 ///@file
 
 #include "config.hh"
+#include "ref.hh"
 
 namespace nix {
 
+class Store;
+
 struct EvalSettings : Config
 {
-    EvalSettings();
+    /**
+     * Function used to interpet look path entries of a given scheme.
+     *
+     * The argument is the non-scheme part of the lookup path entry (see
+     * `LookupPathHooks` below).
+     *
+     * The return value is (a) whether the entry was valid, and, if so,
+     * what does it map to.
+     *
+     * @todo Return (`std::optional` of) `SourceAccssor` or something
+     * more structured instead of mere `std::string`?
+     */
+    using LookupPathHook = std::optional<std::string>(ref<Store> store, std::string_view);
+
+    /**
+     * Map from "scheme" to a `LookupPathHook`.
+     *
+     * Given a lookup path value (i.e. either the whole thing, or after
+     * the `<key>=`) in the form of:
+     *
+     * ```
+     * <scheme>:<arbitrary string>
+     * ```
+     *
+     * if `<scheme>` is a key in this map, then `<arbitrary string>` is
+     * passed to the hook that is the value in this map.
+     */
+    using LookupPathHooks = std::map<std::string, std::function<LookupPathHook>>;
+
+    EvalSettings(bool & readOnlyMode, LookupPathHooks lookupPathHooks = {});
+
+    bool & readOnlyMode;
 
     static Strings getDefaultNixPath();
 
     static bool isPseudoUrl(std::string_view s);
 
+    static Strings parseNixPath(const std::string & s);
+
     static std::string resolvePseudoUrl(std::string_view url);
 
-    Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation",
-        "Whether builtin functions that allow executing native code should be enabled."};
+    LookupPathHooks lookupPathHooks;
+
+    Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation", R"(
+        Enable built-in functions that allow executing native code.
+
+        In particular, this adds:
+        - `builtins.importNative` *path* *symbol*
+
+          Opens dynamic shared object (DSO) at *path*, loads the function with the symbol name *symbol* from it and runs it.
+          The loaded function must have the following signature:
+          ```cpp
+          extern "C" typedef void (*ValueInitialiser) (EvalState & state, Value & v);
+          ```
+
+          The [Nix C++ API documentation](@docroot@/development/documentation.md#api-documentation) has more details on evaluator internals.
+
+        - `builtins.exec` *arguments*
+
+          Execute a program, where *arguments* are specified as a list of strings, and parse its output as a Nix expression.
+    )"};
 
     Setting<Strings> nixPath{
-        this, getDefaultNixPath(), "nix-path",
+        this, {}, "nix-path",
         R"(
-          List of directories to be searched for `<...>` file references
+          List of search paths to use for [lookup path](@docroot@/language/constructs/lookup-path.md) resolution.
+          This setting determines the value of
+          [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath) and can be used with [`builtins.findFile`](@docroot@/language/builtins.md#builtins-findFile).
 
-          In particular, outside of [pure evaluation mode](#conf-pure-eval), this determines the value of
-          [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath).
-        )"};
+          - The configuration setting is overridden by the [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH)
+          environment variable.
+          - `NIX_PATH` is overridden by [specifying the setting as the command line flag](@docroot@/command-ref/conf-file.md#command-line-flags) `--nix-path`.
+          - Any current value is extended by the [`-I` option](@docroot@/command-ref/opt-common.md#opt-I) or `--extra-nix-path`.
+
+          If the respective paths are accessible, the default values are:
+
+          - `$HOME/.nix-defexpr/channels`
+          - `nixpkgs=$NIX_STATE_DIR/profiles/per-user/root/channels/nixpkgs`
+          - `$NIX_STATE_DIR/profiles/per-user/root/channels`
+
+          See [`NIX_STATE_DIR`](@docroot@/command-ref/env-common.md#env-NIX_STATE_DIR) for details.
+
+          > **Note**
+          >
+          > If [restricted evaluation](@docroot@/command-ref/conf-file.md#conf-restrict-eval) is enabled, the default value is empty.
+          >
+          > If [pure evaluation](#conf-pure-eval) is enabled, `builtins.nixPath` *always* evaluates to the empty list `[ ]`.
+        )", {}, false};
 
     Setting<std::string> currentSystem{
         this, "", "eval-system",
         R"(
           This option defines
-          [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem)
+          [`builtins.currentSystem`](@docroot@/language/builtins.md#builtins-currentSystem)
           in the Nix language if it is set as a non-empty string.
           Otherwise, if it is defined as the empty string (the default), the value of the
           [`system` ](#conf-system)
@@ -45,18 +117,16 @@ struct EvalSettings : Config
      * Implements the `eval-system` vs `system` defaulting logic
      * described for `eval-system`.
      */
-    const std::string & getCurrentSystem();
+    const std::string & getCurrentSystem() const;
 
     Setting<bool> restrictEval{
         this, false, "restrict-eval",
         R"(
           If set to `true`, the Nix evaluator will not allow access to any
           files outside of
-          [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath),
+          [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath),
           or to URIs outside of
           [`allowed-uris`](@docroot@/command-ref/conf-file.md#conf-allowed-uris).
-
-          Also the default value for [`nix-path`](#conf-nix-path) is ignored, such that only explicitly set search path entries are taken into account.
         )"};
 
     Setting<bool> pureEval{this, false, "pure-eval",
@@ -65,9 +135,10 @@ struct EvalSettings : Config
 
           - Restrict file system and network access to files specified by cryptographic hash
           - Disable impure constants:
-            - [`bultins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem)
-            - [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime)
-            - [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath)
+            - [`builtins.currentSystem`](@docroot@/language/builtins.md#builtins-currentSystem)
+            - [`builtins.currentTime`](@docroot@/language/builtins.md#builtins-currentTime)
+            - [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath)
+            - [`builtins.storePath`](@docroot@/language/builtins.md#builtins-storePath)
         )"
         };
 
@@ -130,16 +201,40 @@ struct EvalSettings : Config
 
     Setting<bool> builtinsTraceDebugger{this, false, "debugger-on-trace",
         R"(
-          If set to true and the `--debugger` flag is given,
-          [`builtins.trace`](@docroot@/language/builtins.md#builtins-trace) will
-          enter the debugger like
-          [`builtins.break`](@docroot@/language/builtins.md#builtins-break).
+          If set to true and the `--debugger` flag is given, the following functions
+          will enter the debugger like [`builtins.break`](@docroot@/language/builtins.md#builtins-break).
+
+          * [`builtins.trace`](@docroot@/language/builtins.md#builtins-trace)
+          * [`builtins.traceVerbose`](@docroot@/language/builtins.md#builtins-traceVerbose)
+            if [`trace-verbose`](#conf-trace-verbose) is set to true.
+          * [`builtins.warn`](@docroot@/language/builtins.md#builtins-warn)
 
           This is useful for debugging warnings in third-party Nix code.
         )"};
-};
 
-extern EvalSettings evalSettings;
+    Setting<bool> builtinsDebuggerOnWarn{this, false, "debugger-on-warn",
+        R"(
+          If set to true and the `--debugger` flag is given, [`builtins.warn`](@docroot@/language/builtins.md#builtins-warn)
+          will enter the debugger like [`builtins.break`](@docroot@/language/builtins.md#builtins-break).
+
+          This is useful for debugging warnings in third-party Nix code.
+
+          Use [`debugger-on-trace`](#conf-debugger-on-trace) to also enter the debugger on legacy warnings that are logged with [`builtins.trace`](@docroot@/language/builtins.md#builtins-trace).
+        )"};
+
+    Setting<bool> builtinsAbortOnWarn{this, false, "abort-on-warn",
+        R"(
+          If set to true, [`builtins.warn`](@docroot@/language/builtins.md#builtins-warn) will throw an error when logging a warning.
+
+          This will give you a stack trace that leads to the location of the warning.
+
+          This is useful for finding information about warnings in third-party Nix code when you can not start the interactive debugger, such as when Nix is called from a non-interactive script. See [`debugger-on-warn`](#conf-debugger-on-warn).
+
+          Currently, a stack trace can only be produced when the debugger is enabled, or when evaluation is aborted.
+
+          This option can be enabled by setting `NIX_ABORT_ON_WARN=1` in the environment.
+        )"};
+};
 
 /**
  * Conventionally part of the default nix path in impure mode.

src/libexpr/eval.hh

@@ -9,14 +9,16 @@
 #include "symbol-table.hh"
 #include "config.hh"
 #include "experimental-features.hh"
-#include "input-accessor.hh"
+#include "position.hh"
+#include "pos-table.hh"
+#include "source-accessor.hh"
 #include "search-path.hh"
 #include "repl-exit-status.hh"
+#include "ref.hh"
 
 #include <map>
 #include <optional>
-#include <unordered_map>
-#include <mutex>
+#include <functional>
 
 namespace nix {
 
@@ -28,17 +30,21 @@ namespace nix {
 constexpr size_t maxPrimOpArity = 8;
 
 class Store;
+namespace fetchers { struct Settings; }
+struct EvalSettings;
 class EvalState;
 class StorePath;
 struct SingleDerivedPath;
 enum RepairFlag : bool;
-struct MemoryInputAccessor;
-
+struct MemorySourceAccessor;
+namespace eval_cache {
+    class EvalCache;
+}
 
 /**
  * Function that implements a primop.
  */
-typedef void (* PrimOpFun) (EvalState & state, const PosIdx pos, Value * * args, Value & v);
+using PrimOpFun = void(EvalState & state, const PosIdx pos, Value * * args, Value & v);
 
 /**
  * Info about a primitive operation, and its implementation
@@ -69,10 +75,17 @@ struct PrimOp
      */
     const char * doc = nullptr;
 
+    /**
+     * Add a trace item, `while calling the '<name>' builtin`
+     *
+     * This is used to remove the redundant item for `builtins.addErrorContext`.
+     */
+    bool addTrace = true;
+
     /**
      * Implementation of the primop.
      */
-    PrimOpFun fun;
+    std::function<PrimOpFun> fun;
 
     /**
      * Optional experimental for this to be gated on.
@@ -86,7 +99,7 @@ struct PrimOp
     void check();
 };
 
-std::ostream & operator<<(std::ostream & output, PrimOp & primOp);
+std::ostream & operator<<(std::ostream & output, const PrimOp & primOp);
 
 /**
  * Info about a constant
@@ -117,6 +130,8 @@ struct Constant
     typedef std::map<std::string, Value *> ValMap;
 #endif
 
+typedef std::unordered_map<PosIdx, DocComment> DocCommentMap;
+
 struct Env
 {
     Env * up;
@@ -135,12 +150,6 @@ std::string printValue(EvalState & state, Value & v);
 std::ostream & operator << (std::ostream & os, const ValueType t);
 
 
-/**
- * Initialise the Boehm GC, if applicable.
- */
-void initGC();
-
-
 struct RegexCache;
 
 std::shared_ptr<RegexCache> makeRegexCache();
@@ -156,13 +165,18 @@ struct DebugTrace {
 class EvalState : public std::enable_shared_from_this<EvalState>
 {
 public:
+    const fetchers::Settings & fetchSettings;
+    const EvalSettings & settings;
     SymbolTable symbols;
     PosTable positions;
 
     const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue,
         sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls,
         sFile, sLine, sColumn, sFunctor, sToString,
-        sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
+        sRight, sWrong, sStructuredAttrs,
+        sAllowedReferences, sAllowedRequisites, sDisallowedReferences, sDisallowedRequisites,
+        sMaxSize, sMaxClosureSize,
+        sBuilder, sArgs,
         sContentAddressed, sImpure,
         sOutputHash, sOutputHashAlgo, sOutputHashMode,
         sRecurseForDerivations,
@@ -185,21 +199,51 @@ public:
      */
     Value vEmptyList;
 
+    /**
+     * `null` constant.
+     *
+     * This is _not_ a singleton. Pointer equality is _not_ sufficient.
+     */
+    Value vNull;
+
+    /**
+     * `true` constant.
+     *
+     * This is _not_ a singleton. Pointer equality is _not_ sufficient.
+     */
+    Value vTrue;
+
+    /**
+     * `true` constant.
+     *
+     * This is _not_ a singleton. Pointer equality is _not_ sufficient.
+     */
+    Value vFalse;
+
+    /** `"regular"` */
+    Value vStringRegular;
+    /** `"directory"` */
+    Value vStringDirectory;
+    /** `"symlink"` */
+    Value vStringSymlink;
+    /** `"unknown"` */
+    Value vStringUnknown;
+
     /**
      * The accessor for the root filesystem.
      */
-    const ref<InputAccessor> rootFS;
+    const ref<SourceAccessor> rootFS;
 
     /**
      * The in-memory filesystem for <nix/...> paths.
      */
-    const ref<MemoryInputAccessor> corepkgsFS;
+    const ref<MemorySourceAccessor> corepkgsFS;
 
     /**
      * In-memory filesystem for internal, non-user-callable Nix
      * expressions like call-flake.nix.
      */
-    const ref<MemoryInputAccessor> internalFS;
+    const ref<MemorySourceAccessor> internalFS;
 
     const SourcePath derivationInternal;
 
@@ -222,6 +266,7 @@ public:
      */
     ReplExitStatus (* debugRepl)(ref<EvalState> es, const ValMap & extraEnv);
     bool debugStop;
+    bool inDebugger = false;
     int trylevel;
     std::list<DebugTrace> debugTraces;
     std::map<const Expr*, const std::shared_ptr<const StaticEnv>> exprEnvs;
@@ -234,6 +279,18 @@ public:
             return std::shared_ptr<const StaticEnv>();;
     }
 
+    /** Whether a debug repl can be started. If `false`, `runDebugRepl(error)` will return without starting a repl. */
+    bool canDebug();
+
+    /** Use front of `debugTraces`; see `runDebugRepl(error,env,expr)` */
+    void runDebugRepl(const Error * error);
+
+    /**
+     * Run a debug repl with the given error, environment and expression.
+     * @param error The error to debug, may be nullptr.
+     * @param env The environment to debug, matching the expression.
+     * @param expr The expression to debug, matching the environment.
+     */
     void runDebugRepl(const Error * error, const Env & env, const Expr & expr);
 
     template<class T, typename... Args>
@@ -243,19 +300,24 @@ public:
         return *new EvalErrorBuilder<T>(*this, args...);
     }
 
+    /**
+     * A cache for evaluation caches, so as to reuse the same root value if possible
+     */
+    std::map<const Hash, ref<eval_cache::EvalCache>> evalCaches;
+
 private:
 
     /* Cache for calls to addToStore(); maps source paths to the store
        paths. */
-    std::map<SourcePath, StorePath> srcToStore;
+    Sync<std::unordered_map<SourcePath, StorePath>> srcToStore;
 
     /**
      * A cache from path names to parse trees.
      */
 #if HAVE_BOEHMGC
-    typedef std::map<SourcePath, Expr *, std::less<SourcePath>, traceable_allocator<std::pair<const SourcePath, Expr *>>> FileParseCache;
+    typedef std::unordered_map<SourcePath, Expr *, std::hash<SourcePath>, std::equal_to<SourcePath>, traceable_allocator<std::pair<const SourcePath, Expr *>>> FileParseCache;
 #else
-    typedef std::map<SourcePath, Expr *> FileParseCache;
+    typedef std::unordered_map<SourcePath, Expr *> FileParseCache;
 #endif
     FileParseCache fileParseCache;
 
@@ -263,15 +325,21 @@ private:
      * A cache from path names to values.
      */
 #if HAVE_BOEHMGC
-    typedef std::map<SourcePath, Value, std::less<SourcePath>, traceable_allocator<std::pair<const SourcePath, Value>>> FileEvalCache;
+    typedef std::unordered_map<SourcePath, Value, std::hash<SourcePath>, std::equal_to<SourcePath>, traceable_allocator<std::pair<const SourcePath, Value>>> FileEvalCache;
 #else
-    typedef std::map<SourcePath, Value> FileEvalCache;
+    typedef std::unordered_map<SourcePath, Value> FileEvalCache;
 #endif
     FileEvalCache fileEvalCache;
 
-    SearchPath searchPath;
+    /**
+     * Associate source positions of certain AST nodes with their preceding doc comment, if they have one.
+     * Grouped by file.
+     */
+    std::unordered_map<SourcePath, DocCommentMap> positionToDocComment;
 
-    std::map<std::string, std::optional<std::string>> searchPathResolved;
+    LookupPath lookupPath;
+
+    std::map<std::string, std::optional<std::string>> lookupPathResolved;
 
     /**
      * Cache used by prim_match().
@@ -293,12 +361,14 @@ private:
 public:
 
     EvalState(
-        const SearchPath & _searchPath,
+        const LookupPath & _lookupPath,
         ref<Store> store,
+        const fetchers::Settings & fetchSettings,
+        const EvalSettings & settings,
         std::shared_ptr<Store> buildStore = nullptr);
     ~EvalState();
 
-    SearchPath getSearchPath() { return searchPath; }
+    LookupPath getLookupPath() { return lookupPath; }
 
     /**
      * Return a `SourcePath` that refers to `path` in the root
@@ -367,7 +437,7 @@ public:
      * Look up a file in the search path.
      */
     SourcePath findFile(const std::string_view path);
-    SourcePath findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos);
+    SourcePath findFile(const LookupPath & lookupPath, const std::string_view path, const PosIdx pos = noPos);
 
     /**
      * Try to resolve a search path value (not the optional key part).
@@ -376,8 +446,8 @@ public:
      *
      * If it is not found, return `std::nullopt`
      */
-    std::optional<std::string> resolveSearchPathPath(
-        const SearchPath::Path & elem,
+    std::optional<std::string> resolveLookupPathPath(
+        const LookupPath::Path & elem,
         bool initAccessControl = false);
 
     /**
@@ -500,6 +570,11 @@ public:
      */
     SingleDerivedPath coerceToSingleDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx);
 
+#if HAVE_BOEHMGC
+    /** A GC root for the baseEnv reference. */
+    std::shared_ptr<Env *> baseEnvP;
+#endif
+
 public:
 
     /**
@@ -580,6 +655,15 @@ public:
      */
     bool eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx);
 
+    /**
+     * Like `eqValues`, but throws an `AssertionError` if not equal.
+     *
+     * WARNING:
+     * Callers should call `eqValues` first and report if `assertEqValues` behaves
+     * incorrectly. (e.g. if it doesn't throw if eqValues returns false or vice versa)
+     */
+    void assertEqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx);
+
     bool isFunctor(Value & fun);
 
     // FIXME: use std::span
@@ -603,9 +687,6 @@ public:
     inline Value * allocValue();
     inline Env & allocEnv(size_t size);
 
-    Value * allocAttr(Value & vAttrs, Symbol name);
-    Value * allocAttr(Value & vAttrs, std::string_view name);
-
     Bindings * allocBindings(size_t capacity);
 
     BindingsBuilder buildBindings(size_t capacity)
@@ -613,7 +694,16 @@ public:
         return BindingsBuilder(*this, allocBindings(capacity));
     }
 
-    void mkList(Value & v, size_t length);
+    ListBuilder buildList(size_t size)
+    {
+        return ListBuilder(*this, size);
+    }
+
+    /**
+     * Return a boolean `Value *` without allocating.
+     */
+    Value *getBool(bool b);
+
     void mkThunk_(Value & v, Expr * expr);
     void mkPos(Value & v, PosIdx pos);
 
@@ -660,7 +750,7 @@ public:
         const SingleDerivedPath & p,
         Value & v);
 
-    void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx);
+    void concatLists(Value & v, size_t nrLists, Value * const * lists, const PosIdx pos, std::string_view errorCtx);
 
     /**
      * Print statistics, if enabled.
@@ -684,10 +774,12 @@ public:
     bool fullGC();
 
     /**
-     * Realise the given context, and return a mapping from the placeholders
-     * used to construct the associated value to their final store path
+     * Realise the given context
+     * @param[in] context the context to realise
+     * @param[out] maybePaths if not nullptr, all built or referenced store paths will be added to this set
+     * @return a mapping from the placeholders used to construct the associated value to their final store path.
      */
-    [[nodiscard]] StringMap realiseContext(const NixStringContext & context);
+    [[nodiscard]] StringMap realiseContext(const NixStringContext & context, StorePathSet * maybePaths = nullptr, bool isIFD = true);
 
     /* Call the binary path filter predicate used builtins.path etc. */
     bool callPathFilter(
@@ -696,6 +788,8 @@ public:
         std::string_view pathArg,
         PosIdx pos);
 
+    DocComment getDocCommentForPos(PosIdx pos);
+
 private:
 
     /**
@@ -754,6 +848,7 @@ private:
     friend void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v);
 
     friend struct Value;
+    friend class ListBuilder;
 };
 
 struct DebugTraceStacker {
@@ -777,8 +872,10 @@ std::string showType(const Value & v);
 
 /**
  * If `path` refers to a directory, then append "/default.nix".
+ *
+ * @param addDefaultNix Whether to append "/default.nix" after resolving symlinks.
  */
-SourcePath resolveExprPath(SourcePath path);
+SourcePath resolveExprPath(SourcePath path, bool addDefaultNix = true);
 
 /**
  * Whether a URI is allowed, assuming restrictEval is enabled

src/libexpr/fetchurl.nix

@@ -28,11 +28,8 @@ derivation ({
   # No need to double the amount of network traffic
   preferLocalBuild = true;
 
+  # This attribute does nothing; it's here to avoid changing evaluation results.
   impureEnvVars = [
-    # We borrow these environment variables from the caller to allow
-    # easy proxy configuration.  This is impure, but a fixed-output
-    # derivation like fetchurl is allowed to do so since its result is
-    # by definition pure.
     "http_proxy" "https_proxy" "ftp_proxy" "all_proxy" "no_proxy"
   ];
 

src/libexpr/get-drvs.cc

@@ -11,7 +11,7 @@
 namespace nix {
 
 
-PackageInfo::PackageInfo(EvalState & state, std::string attrPath, Bindings * attrs)
+PackageInfo::PackageInfo(EvalState & state, std::string attrPath, const Bindings * attrs)
     : state(&state), attrs(attrs), attrPath(std::move(attrPath))
 {
 }
@@ -69,12 +69,18 @@ std::string PackageInfo::querySystem() const
 std::optional<StorePath> PackageInfo::queryDrvPath() const
 {
     if (!drvPath && attrs) {
-        Bindings::iterator i = attrs->find(state->sDrvPath);
-        NixStringContext context;
-        if (i == attrs->end())
+        if (auto i = attrs->get(state->sDrvPath)) {
+            NixStringContext context;
+            auto found = state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the 'drvPath' attribute of a derivation");
+            try {
+                found.requireDerivation();
+            } catch (Error & e) {
+                e.addTrace(state->positions[i->pos], "while evaluating the 'drvPath' attribute of a derivation");
+                throw;
+            }
+            drvPath = {std::move(found)};
+        } else
             drvPath = {std::nullopt};
-        else
-            drvPath = {state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the 'drvPath' attribute of a derivation")};
     }
     return drvPath.value_or(std::nullopt);
 }
@@ -91,7 +97,7 @@ StorePath PackageInfo::requireDrvPath() const
 StorePath PackageInfo::queryOutPath() const
 {
     if (!outPath && attrs) {
-        Bindings::iterator i = attrs->find(state->sOutPath);
+        auto i = attrs->find(state->sOutPath);
         NixStringContext context;
         if (i != attrs->end())
             outPath = state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the output path of a derivation");
@@ -106,8 +112,8 @@ PackageInfo::Outputs PackageInfo::queryOutputs(bool withPaths, bool onlyOutputsT
 {
     if (outputs.empty()) {
         /* Get the ‘outputs’ list. */
-        Bindings::iterator i;
-        if (attrs && (i = attrs->find(state->sOutputs)) != attrs->end()) {
+        const Attr * i;
+        if (attrs && (i = attrs->get(state->sOutputs))) {
             state->forceList(*i->value, i->pos, "while evaluating the 'outputs' attribute of a derivation");
 
             /* For each output... */
@@ -116,13 +122,13 @@ PackageInfo::Outputs PackageInfo::queryOutputs(bool withPaths, bool onlyOutputsT
 
                 if (withPaths) {
                     /* Evaluate the corresponding set. */
-                    Bindings::iterator out = attrs->find(state->symbols.create(output));
-                    if (out == attrs->end()) continue; // FIXME: throw error?
+                    auto out = attrs->get(state->symbols.create(output));
+                    if (!out) continue; // FIXME: throw error?
                     state->forceAttrs(*out->value, i->pos, "while evaluating an output of a derivation");
 
                     /* And evaluate its ‘outPath’ attribute. */
-                    Bindings::iterator outPath = out->value->attrs->find(state->sOutPath);
-                    if (outPath == out->value->attrs->end()) continue; // FIXME: throw error?
+                    auto outPath = out->value->attrs()->get(state->sOutPath);
+                    if (!outPath) continue; // FIXME: throw error?
                     NixStringContext context;
                     outputs.emplace(output, state->coerceToStorePath(outPath->pos, *outPath->value, context, "while evaluating an output path of a derivation"));
                 } else
@@ -135,8 +141,8 @@ PackageInfo::Outputs PackageInfo::queryOutputs(bool withPaths, bool onlyOutputsT
     if (!onlyOutputsToInstall || !attrs)
         return outputs;
 
-    Bindings::iterator i;
-    if (attrs && (i = attrs->find(state->sOutputSpecified)) != attrs->end() && state->forceBool(*i->value, i->pos, "while evaluating the 'outputSpecified' attribute of a derivation")) {
+    const Attr * i;
+    if (attrs && (i = attrs->get(state->sOutputSpecified)) && state->forceBool(*i->value, i->pos, "while evaluating the 'outputSpecified' attribute of a derivation")) {
         Outputs result;
         auto out = outputs.find(queryOutputName());
         if (out == outputs.end())
@@ -167,21 +173,21 @@ PackageInfo::Outputs PackageInfo::queryOutputs(bool withPaths, bool onlyOutputsT
 std::string PackageInfo::queryOutputName() const
 {
     if (outputName == "" && attrs) {
-        Bindings::iterator i = attrs->find(state->sOutputName);
-        outputName = i != attrs->end() ? state->forceStringNoCtx(*i->value, noPos, "while evaluating the output name of a derivation") : "";
+        auto i = attrs->get(state->sOutputName);
+        outputName = i ? state->forceStringNoCtx(*i->value, noPos, "while evaluating the output name of a derivation") : "";
     }
     return outputName;
 }
 
 
-Bindings * PackageInfo::getMeta()
+const Bindings * PackageInfo::getMeta()
 {
     if (meta) return meta;
     if (!attrs) return 0;
-    Bindings::iterator a = attrs->find(state->sMeta);
-    if (a == attrs->end()) return 0;
+    auto a = attrs->get(state->sMeta);
+    if (!a) return 0;
     state->forceAttrs(*a->value, a->pos, "while evaluating the 'meta' attribute of a derivation");
-    meta = a->value->attrs;
+    meta = a->value->attrs();
     return meta;
 }
 
@@ -205,9 +211,8 @@ bool PackageInfo::checkMeta(Value & v)
         return true;
     }
     else if (v.type() == nAttrs) {
-        Bindings::iterator i = v.attrs->find(state->sOutPath);
-        if (i != v.attrs->end()) return false;
-        for (auto & i : *v.attrs)
+        if (v.attrs()->get(state->sOutPath)) return false;
+        for (auto & i : *v.attrs())
             if (!checkMeta(*i.value)) return false;
         return true;
     }
@@ -219,8 +224,8 @@ bool PackageInfo::checkMeta(Value & v)
 Value * PackageInfo::queryMeta(const std::string & name)
 {
     if (!getMeta()) return 0;
-    Bindings::iterator a = meta->find(state->symbols.create(name));
-    if (a == meta->end() || !checkMeta(*a->value)) return 0;
+    auto a = meta->get(state->symbols.create(name));
+    if (!a || !checkMeta(*a->value)) return 0;
     return a->value;
 }
 
@@ -237,7 +242,7 @@ NixInt PackageInfo::queryMetaInt(const std::string & name, NixInt def)
 {
     Value * v = queryMeta(name);
     if (!v) return def;
-    if (v->type() == nInt) return v->integer;
+    if (v->type() == nInt) return v->integer();
     if (v->type() == nString) {
         /* Backwards compatibility with before we had support for
            integer meta fields. */
@@ -251,7 +256,7 @@ NixFloat PackageInfo::queryMetaFloat(const std::string & name, NixFloat def)
 {
     Value * v = queryMeta(name);
     if (!v) return def;
-    if (v->type() == nFloat) return v->fpoint;
+    if (v->type() == nFloat) return v->fpoint();
     if (v->type() == nString) {
         /* Backwards compatibility with before we had support for
            float meta fields. */
@@ -266,7 +271,7 @@ bool PackageInfo::queryMetaBool(const std::string & name, bool def)
 {
     Value * v = queryMeta(name);
     if (!v) return def;
-    if (v->type() == nBool) return v->boolean;
+    if (v->type() == nBool) return v->boolean();
     if (v->type() == nString) {
         /* Backwards compatibility with before we had support for
            Boolean meta fields. */
@@ -292,7 +297,7 @@ void PackageInfo::setMeta(const std::string & name, Value * v)
 
 
 /* Cache for already considered attrsets. */
-typedef std::set<Bindings *> Done;
+typedef std::set<const Bindings *> Done;
 
 
 /* Evaluate value `v'.  If it evaluates to a set of type `derivation',
@@ -309,9 +314,9 @@ static bool getDerivation(EvalState & state, Value & v,
 
         /* Remove spurious duplicates (e.g., a set like `rec { x =
            derivation {...}; y = x;}'. */
-        if (!done.insert(v.attrs).second) return false;
+        if (!done.insert(v.attrs()).second) return false;
 
-        PackageInfo drv(state, attrPath, v.attrs);
+        PackageInfo drv(state, attrPath, v.attrs());
 
         drv.queryName();
 
@@ -337,9 +342,9 @@ std::optional<PackageInfo> getDerivation(EvalState & state, Value & v,
 }
 
 
-static std::string addToPath(const std::string & s1, const std::string & s2)
+static std::string addToPath(const std::string & s1, std::string_view s2)
 {
-    return s1.empty() ? s2 : s1 + "." + s2;
+    return s1.empty() ? std::string(s2) : s1 + "." + s2;
 }
 
 
@@ -361,29 +366,34 @@ static void getDerivations(EvalState & state, Value & vIn,
 
         /* !!! undocumented hackery to support combining channels in
            nix-env.cc. */
-        bool combineChannels = v.attrs->find(state.symbols.create("_combineChannels")) != v.attrs->end();
+        bool combineChannels = v.attrs()->get(state.symbols.create("_combineChannels"));
 
         /* Consider the attributes in sorted order to get more
            deterministic behaviour in nix-env operations (e.g. when
            there are names clashes between derivations, the derivation
            bound to the attribute with the "lower" name should take
            precedence). */
-        for (auto & i : v.attrs->lexicographicOrder(state.symbols)) {
-            debug("evaluating attribute '%1%'", state.symbols[i->name]);
-            if (!std::regex_match(std::string(state.symbols[i->name]), attrRegex))
-                continue;
-            std::string pathPrefix2 = addToPath(pathPrefix, state.symbols[i->name]);
-            if (combineChannels)
-                getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
-            else if (getDerivation(state, *i->value, pathPrefix2, drvs, done, ignoreAssertionFailures)) {
-                /* If the value of this attribute is itself a set,
-                   should we recurse into it?  => Only if it has a
-                   `recurseForDerivations = true' attribute. */
-                if (i->value->type() == nAttrs) {
-                    Bindings::iterator j = i->value->attrs->find(state.sRecurseForDerivations);
-                    if (j != i->value->attrs->end() && state.forceBool(*j->value, j->pos, "while evaluating the attribute `recurseForDerivations`"))
-                        getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
+        for (auto & i : v.attrs()->lexicographicOrder(state.symbols)) {
+            try {
+                debug("evaluating attribute '%1%'", state.symbols[i->name]);
+                if (!std::regex_match(std::string(state.symbols[i->name]), attrRegex))
+                    continue;
+                std::string pathPrefix2 = addToPath(pathPrefix, state.symbols[i->name]);
+                if (combineChannels)
+                    getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
+                else if (getDerivation(state, *i->value, pathPrefix2, drvs, done, ignoreAssertionFailures)) {
+                    /* If the value of this attribute is itself a set,
+                    should we recurse into it?  => Only if it has a
+                    `recurseForDerivations = true' attribute. */
+                    if (i->value->type() == nAttrs) {
+                        auto j = i->value->attrs()->get(state.sRecurseForDerivations);
+                        if (j && state.forceBool(*j->value, j->pos, "while evaluating the attribute `recurseForDerivations`"))
+                            getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
+                    }
                 }
+            } catch (Error & e) {
+                e.addTrace(state.positions[i->pos], "while evaluating the attribute '%s'", state.symbols[i->name]);
+                throw;
             }
         }
     }

src/libexpr/get-drvs.hh

@@ -33,9 +33,9 @@ private:
      */
     bool failed = false;
 
-    Bindings * attrs = nullptr, * meta = nullptr;
+    const Bindings * attrs = nullptr, * meta = nullptr;
 
-    Bindings * getMeta();
+    const Bindings * getMeta();
 
     bool checkMeta(Value & v);
 
@@ -46,7 +46,7 @@ public:
     std::string attrPath;
 
     PackageInfo(EvalState & state) : state(&state) { };
-    PackageInfo(EvalState & state, std::string attrPath, Bindings * attrs);
+    PackageInfo(EvalState & state, std::string attrPath, const Bindings * attrs);
     PackageInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs);
 
     std::string queryName() const;

src/libexpr/json-to-value.cc

@@ -42,7 +42,7 @@ class JSONSax : nlohmann::json_sax<json> {
             auto attrs2 = state.buildBindings(attrs.size());
             for (auto & i : attrs)
                 attrs2.insert(i.first, i.second);
-            parent->value(state).mkAttrs(attrs2.alreadySorted());
+            parent->value(state).mkAttrs(attrs2);
             return std::move(parent);
         }
         void add() override { v = nullptr; }
@@ -57,11 +57,10 @@ class JSONSax : nlohmann::json_sax<json> {
         ValueVector values;
         std::unique_ptr<JSONState> resolve(EvalState & state) override
         {
-            Value & v = parent->value(state);
-            state.mkList(v, values.size());
-            for (size_t n = 0; n < values.size(); ++n) {
-                v.listElems()[n] = values[n];
-            }
+            auto list = state.buildList(values.size());
+            for (const auto & [n, v2] : enumerate(list))
+                v2 = values[n];
+            parent->value(state).mkList(list);
             return std::move(parent);
         }
         void add() override {
@@ -81,42 +80,42 @@ class JSONSax : nlohmann::json_sax<json> {
 public:
     JSONSax(EvalState & state, Value & v) : state(state), rs(new JSONState(&v)) {};
 
-    bool null()
+    bool null() override
     {
         rs->value(state).mkNull();
         rs->add();
         return true;
     }
 
-    bool boolean(bool val)
+    bool boolean(bool val) override
     {
         rs->value(state).mkBool(val);
         rs->add();
         return true;
     }
 
-    bool number_integer(number_integer_t val)
+    bool number_integer(number_integer_t val) override
     {
         rs->value(state).mkInt(val);
         rs->add();
         return true;
     }
 
-    bool number_unsigned(number_unsigned_t val)
+    bool number_unsigned(number_unsigned_t val) override
     {
         rs->value(state).mkInt(val);
         rs->add();
         return true;
     }
 
-    bool number_float(number_float_t val, const string_t & s)
+    bool number_float(number_float_t val, const string_t & s) override
     {
         rs->value(state).mkFloat(val);
         rs->add();
         return true;
     }
 
-    bool string(string_t & val)
+    bool string(string_t & val) override
     {
         rs->value(state).mkString(val);
         rs->add();
@@ -124,7 +123,7 @@ public:
     }
 
 #if NLOHMANN_JSON_VERSION_MAJOR >= 3 && NLOHMANN_JSON_VERSION_MINOR >= 8
-    bool binary(binary_t&)
+    bool binary(binary_t&) override
     {
         // This function ought to be unreachable
         assert(false);
@@ -132,35 +131,35 @@ public:
     }
 #endif
 
-    bool start_object(std::size_t len)
+    bool start_object(std::size_t len) override
     {
         rs = std::make_unique<JSONObjectState>(std::move(rs));
         return true;
     }
 
-    bool key(string_t & name)
+    bool key(string_t & name) override
     {
         dynamic_cast<JSONObjectState*>(rs.get())->key(name, state);
         return true;
     }
 
-    bool end_object() {
+    bool end_object() override {
         rs = rs->resolve(state);
         rs->add();
         return true;
     }
 
-    bool end_array() {
+    bool end_array() override {
         return end_object();
     }
 
-    bool start_array(size_t len) {
+    bool start_array(size_t len) override {
         rs = std::make_unique<JSONListState>(std::move(rs),
             len != std::numeric_limits<size_t>::max() ? len : 128);
         return true;
     }
 
-    bool parse_error(std::size_t, const std::string&, const nlohmann::detail::exception& ex) {
+    bool parse_error(std::size_t, const std::string&, const nlohmann::detail::exception& ex) override {
         throw JSONParseError("%s", ex.what());
     }
 };

src/libexpr/lexer-helpers.cc

@@ -0,0 +1,30 @@
+#include "lexer-tab.hh"
+#include "lexer-helpers.hh"
+#include "parser-tab.hh"
+
+void nix::lexer::internal::initLoc(YYLTYPE * loc)
+{
+    loc->beginOffset = loc->endOffset = 0;
+}
+
+void nix::lexer::internal::adjustLoc(yyscan_t yyscanner, YYLTYPE * loc, const char * s, size_t len)
+{
+    loc->stash();
+
+    LexerState & lexerState = *yyget_extra(yyscanner);
+
+    if (lexerState.docCommentDistance == 1) {
+        // Preceding token was a doc comment.
+        ParserLocation doc;
+        doc.beginOffset = lexerState.lastDocCommentLoc.beginOffset;
+        ParserLocation docEnd;
+        docEnd.beginOffset = lexerState.lastDocCommentLoc.endOffset;
+        DocComment docComment{lexerState.at(doc), lexerState.at(docEnd)};
+        PosIdx locPos = lexerState.at(*loc);
+        lexerState.positionToDocComment.emplace(locPos, docComment);
+    }
+    lexerState.docCommentDistance++;
+
+    loc->beginOffset = loc->endOffset;
+    loc->endOffset += len;
+}

src/libexpr/lexer-helpers.hh

@@ -0,0 +1,9 @@
+#pragma once
+
+namespace nix::lexer::internal {
+
+void initLoc(YYLTYPE * loc);
+
+void adjustLoc(yyscan_t yyscanner, YYLTYPE * loc, const char * s, size_t len);
+
+} // namespace nix::lexer

src/libexpr/lexer.l

@@ -5,7 +5,7 @@
 %option stack
 %option nodefault
 %option nounput noyy_top_state
-
+%option extra-type="::nix::LexerState *"
 
 %s DEFAULT
 %x STRING
@@ -14,55 +14,31 @@
 %x INPATH_SLASH
 %x PATH_START
 
+%top {
+#include "parser-tab.hh" // YYSTYPE
+#include "parser-state.hh"
+}
 
 %{
 #ifdef __clang__
 #pragma clang diagnostic ignored "-Wunneeded-internal-declaration"
 #endif
 
-#include <boost/lexical_cast.hpp>
-
 #include "nixexpr.hh"
 #include "parser-tab.hh"
+#include "lexer-helpers.hh"
+
+namespace nix {
+  struct LexerState;
+}
 
 using namespace nix;
+using namespace nix::lexer::internal;
 
 namespace nix {
 
 #define CUR_POS state->at(*yylloc)
 
-static void initLoc(YYLTYPE * loc)
-{
-    loc->first_line = loc->last_line = 1;
-    loc->first_column = loc->last_column = 1;
-}
-
-static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
-{
-    loc->stash();
-
-    loc->first_line = loc->last_line;
-    loc->first_column = loc->last_column;
-
-    for (size_t i = 0; i < len; i++) {
-       switch (*s++) {
-       case '\r':
-           if (*s == '\n') { /* cr/lf */
-               i++;
-               s++;
-           }
-           /* fall through */
-       case '\n':
-           ++loc->last_line;
-           loc->last_column = 1;
-           break;
-       default:
-           ++loc->last_column;
-       }
-    }
-}
-
-
 // we make use of the fact that the parser receives a private copy of the input
 // string and can munge around in it.
 static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length)
@@ -91,6 +67,14 @@ static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length)
     return {result, size_t(t - result)};
 }
 
+static void requireExperimentalFeature(const ExperimentalFeature & feature, const Pos & pos)
+{
+    if (!experimentalFeatureSettings.isEnabled(feature))
+        throw ParseError(ErrorInfo{
+            .msg = HintFmt("experimental Nix feature '%1%' is disabled; add '--extra-experimental-features %1%' to enable it", showExperimentalFeature(feature)),
+            .pos = pos,
+        });
+}
 
 }
 
@@ -98,7 +82,7 @@ static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length)
 #pragma GCC diagnostic ignored "-Wimplicit-fallthrough"
 
 #define YY_USER_INIT initLoc(yylloc)
-#define YY_USER_ACTION adjustLoc(yylloc, yytext, yyleng);
+#define YY_USER_ACTION adjustLoc(yyscanner, yylloc, yytext, yyleng);
 
 #define PUSH_STATE(state) yy_push_state(state, yyscanner)
 #define POP_STATE() yy_pop_state(yyscanner)
@@ -143,12 +127,19 @@ or          { return OR_KW; }
 \-\>        { return IMPL; }
 \/\/        { return UPDATE; }
 \+\+        { return CONCAT; }
+\<\|        { requireExperimentalFeature(Xp::PipeOperators, state->positions[CUR_POS]);
+              return PIPE_FROM;
+            }
+\|\>        { requireExperimentalFeature(Xp::PipeOperators, state->positions[CUR_POS]);
+              return PIPE_INTO;
+            }
 
 {ID}        { yylval->id = {yytext, (size_t) yyleng}; return ID; }
 {INT}       { errno = 0;
-              try {
-                  yylval->n = boost::lexical_cast<int64_t>(yytext);
-              } catch (const boost::bad_lexical_cast &) {
+              std::optional<int64_t> numMay = string2Int<int64_t>(yytext);
+              if (numMay.has_value()) {
+                  yylval->n = *numMay;
+              } else {
                   throw ParseError(ErrorInfo{
                       .msg = HintFmt("invalid integer '%1%'", yytext),
                       .pos = state->positions[CUR_POS],
@@ -297,9 +288,32 @@ or          { return OR_KW; }
 {SPATH}     { yylval->path = {yytext, (size_t) yyleng}; return SPATH; }
 {URI}       { yylval->uri = {yytext, (size_t) yyleng}; return URI; }
 
-[ \t\r\n]+    /* eat up whitespace */
-\#[^\r\n]*    /* single-line comments */
-\/\*([^*]|\*+[^*/])*\*+\/  /* long comments */
+%{
+// Doc comment rule
+//
+//   \/\*\*                            /**
+//         [^/*]                       reject /**/ (empty comment) and /***
+//              ([^*]|\*+[^*/])*\*+\/  same as the long comment rule
+//              (             )*       zero or more non-ending sequences
+//                              \*     end(1)
+//                                 \/  end(2)
+%}
+\/\*\*[^/*]([^*]|\*+[^*/])*\*+\/  /* doc comments */ {
+    LexerState & lexerState = *yyget_extra(yyscanner);
+    lexerState.docCommentDistance = 0;
+    lexerState.lastDocCommentLoc.beginOffset = yylloc->beginOffset;
+    lexerState.lastDocCommentLoc.endOffset = yylloc->endOffset;
+}
+
+
+%{
+// The following rules have docCommentDistance--
+// This compensates for the docCommentDistance++ which happens by default to
+// make all the other rules invalidate the doc comment.
+%}
+[ \t\r\n]+    /* eat up whitespace */ { yyget_extra(yyscanner)->docCommentDistance--; }
+\#[^\r\n]*    /* single-line comments */ { yyget_extra(yyscanner)->docCommentDistance--; }
+\/\*([^*]|\*+[^*/])*\*+\/  /* long comments */ { yyget_extra(yyscanner)->docCommentDistance--; }
 
 {ANY}       {
               /* Don't return a negative number, as this will cause

src/libexpr/local.mk

@@ -8,11 +8,15 @@ libexpr_SOURCES := \
   $(wildcard $(d)/*.cc) \
   $(wildcard $(d)/value/*.cc) \
   $(wildcard $(d)/primops/*.cc) \
-  $(wildcard $(d)/flake/*.cc) \
   $(d)/lexer-tab.cc \
   $(d)/parser-tab.cc
+# Not just for this library itself, but also for downstream libraries using this library
 
-libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libmain -I src/libexpr
+INCLUDE_libexpr := -I $(d)
+
+libexpr_CXXFLAGS += \
+	$(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) \
+	-DGC_THREADS
 
 libexpr_LIBS = libutil libstore libfetchers
 
@@ -40,11 +44,7 @@ $(eval $(call install-file-in, $(buildprefix)$(d)/nix-expr.pc, $(libdir)/pkgconf
 
 $(foreach i, $(wildcard src/libexpr/value/*.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix/value, 0644)))
-$(foreach i, $(wildcard src/libexpr/flake/*.hh), \
-  $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644)))
 
 $(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh
 
-$(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh $(d)/flake/call-flake.nix.gen.hh
-
-$(buildprefix)src/libexpr/primops/fromTOML.o:	ERROR_SWITCH_ENUM =
+$(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh $(d)/call-flake.nix.gen.hh

src/libexpr/meson.build

@@ -0,0 +1,210 @@
+project('nix-expr', 'cpp',
+  version : files('.version'),
+  default_options : [
+    'cpp_std=c++2a',
+    # TODO(Qyriad): increase the warning level
+    'warning_level=1',
+    'debug=true',
+    'optimization=2',
+    'errorlogs=true', # Please print logs for tests that fail
+  ],
+  meson_version : '>= 1.1',
+  license : 'LGPL-2.1-or-later',
+)
+
+cxx = meson.get_compiler('cpp')
+
+subdir('build-utils-meson/deps-lists')
+
+configdata = configuration_data()
+
+deps_private_maybe_subproject = [
+]
+deps_public_maybe_subproject = [
+  dependency('nix-util'),
+  dependency('nix-store'),
+  dependency('nix-fetchers'),
+]
+subdir('build-utils-meson/subprojects')
+
+subdir('build-utils-meson/threads')
+
+boost = dependency(
+  'boost',
+  modules : ['container', 'context'],
+  include_type: 'system',
+)
+# boost is a public dependency, but not a pkg-config dependency unfortunately, so we
+# put in `deps_other`.
+deps_other += boost
+
+nlohmann_json = dependency('nlohmann_json', version : '>= 3.9')
+deps_public += nlohmann_json
+
+bdw_gc = dependency('bdw-gc', required : get_option('gc'))
+if bdw_gc.found()
+  deps_public += bdw_gc
+  foreach funcspec : [
+    'pthread_attr_get_np',
+    'pthread_getattr_np',
+  ]
+    define_name = 'HAVE_' + funcspec.underscorify().to_upper()
+    define_value = cxx.has_function(funcspec).to_int()
+    configdata.set(define_name, define_value)
+  endforeach
+  configdata.set('GC_THREADS', 1)
+endif
+configdata.set('HAVE_BOEHMGC', bdw_gc.found().to_int())
+
+toml11 = dependency(
+  'toml11',
+  version : '>=3.7.0',
+  method : 'cmake',
+  include_type: 'system',
+)
+deps_other += toml11
+
+config_h = configure_file(
+  configuration : configdata,
+  output : 'config-expr.hh',
+)
+
+add_project_arguments(
+  # TODO(Qyriad): Yes this is how the autoconf+Make system did it.
+  # It would be nice for our headers to be idempotent instead.
+  '-include', 'config-util.hh',
+  '-include', 'config-store.hh',
+  # '-include', 'config-fetchers.h',
+  '-include', 'config-expr.hh',
+  language : 'cpp',
+)
+
+subdir('build-utils-meson/diagnostics')
+
+parser_tab = custom_target(
+  input : 'parser.y',
+  output : [
+    'parser-tab.cc',
+    'parser-tab.hh',
+  ],
+  command : [
+    'bison',
+    '-v',
+    '-o',
+    '@OUTPUT0@',
+    '@INPUT@',
+    '-d',
+  ],
+  # NOTE(Qyriad): Meson doesn't support installing only part of a custom target, so we add
+  # an install script below which removes parser-tab.cc.
+  install : true,
+  install_dir : get_option('includedir') / 'nix',
+)
+
+lexer_tab = custom_target(
+  input : [
+    'lexer.l',
+    parser_tab,
+  ],
+  output : [
+    'lexer-tab.cc',
+    'lexer-tab.hh',
+  ],
+  command : [
+    'flex',
+    '--outfile',
+    '@OUTPUT0@',
+    '--header-file=' + '@OUTPUT1@',
+    '@INPUT0@',
+  ],
+  # NOTE(Qyriad): Meson doesn't support installing only part of a custom target, so we add
+  # an install script below which removes lexer-tab.cc.
+  install : true,
+  install_dir : get_option('includedir') / 'nix',
+)
+
+subdir('build-utils-meson/generate-header')
+
+generated_headers = []
+foreach header : [
+  'imported-drv-to-derivation.nix',
+  'fetchurl.nix',
+  'call-flake.nix',
+]
+  generated_headers += gen_header.process(header)
+endforeach
+
+sources = files(
+  'attr-path.cc',
+  'attr-set.cc',
+  'eval-cache.cc',
+  'eval-error.cc',
+  'eval-gc.cc',
+  'eval-settings.cc',
+  'eval.cc',
+  'function-trace.cc',
+  'get-drvs.cc',
+  'json-to-value.cc',
+  'lexer-helpers.cc',
+  'nixexpr.cc',
+  'paths.cc',
+  'primops.cc',
+  'print-ambiguous.cc',
+  'print.cc',
+  'search-path.cc',
+  'value-to-json.cc',
+  'value-to-xml.cc',
+  'value/context.cc',
+)
+
+include_dirs = [include_directories('.')]
+
+headers = [config_h] + files(
+  'attr-path.hh',
+  'attr-set.hh',
+  'eval-cache.hh',
+  'eval-error.hh',
+  'eval-gc.hh',
+  'eval-inline.hh',
+  'eval-settings.hh',
+  'eval.hh',
+  'function-trace.hh',
+  'gc-small-vector.hh',
+  'get-drvs.hh',
+  'json-to-value.hh',
+  # internal: 'lexer-helpers.hh',
+  'nixexpr.hh',
+  'parser-state.hh',
+  'pos-idx.hh',
+  'pos-table.hh',
+  'primops.hh',
+  'print-ambiguous.hh',
+  'print-options.hh',
+  'print.hh',
+  'repl-exit-status.hh',
+  'search-path.hh',
+  'symbol-table.hh',
+  'value-to-json.hh',
+  'value-to-xml.hh',
+  'value.hh',
+  'value/context.hh',
+)
+
+subdir('primops')
+
+this_library = library(
+  'nixexpr',
+  sources,
+  parser_tab,
+  lexer_tab,
+  generated_headers,
+  dependencies : deps_public + deps_private + deps_other,
+  prelink : true, # For C++ static initializers
+  install : true,
+)
+
+install_headers(headers, subdir : 'nix', preserve_path : true)
+
+libraries_private = []
+
+subdir('build-utils-meson/export')

src/libexpr/meson.options

@@ -0,0 +1,3 @@
+option('gc', type : 'feature',
+  description : 'enable garbage collection in the Nix expression evaluator (requires Boehm GC)',
+)

src/libexpr/nixexpr.cc

@@ -1,11 +1,13 @@
 #include "nixexpr.hh"
-#include "derivations.hh"
 #include "eval.hh"
 #include "symbol-table.hh"
 #include "util.hh"
 #include "print.hh"
 
 #include <cstdlib>
+#include <sstream>
+
+#include "strings-inline.hh"
 
 namespace nix {
 
@@ -23,17 +25,17 @@ std::ostream & operator <<(std::ostream & str, const SymbolStr & symbol)
 
 void Expr::show(const SymbolTable & symbols, std::ostream & str) const
 {
-    abort();
+    unreachable();
 }
 
 void ExprInt::show(const SymbolTable & symbols, std::ostream & str) const
 {
-    str << v.integer;
+    str << v.integer();
 }
 
 void ExprFloat::show(const SymbolTable & symbols, std::ostream & str) const
 {
-    str << v.fpoint;
+    str << v.fpoint();
 }
 
 void ExprString::show(const SymbolTable & symbols, std::ostream & str) const
@@ -80,7 +82,9 @@ void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) co
         return sa < sb;
     });
     std::vector<Symbol> inherits;
-    std::map<ExprInheritFrom *, std::vector<Symbol>> inheritsFrom;
+    // We can use the displacement as a proxy for the order in which the symbols were parsed.
+    // The assignment of displacements should be deterministic, so that showBindings is deterministic.
+    std::map<Displacement, std::vector<Symbol>> inheritsFrom;
     for (auto & i : sorted) {
         switch (i->second.kind) {
         case AttrDef::Kind::Plain:
@@ -91,7 +95,7 @@ void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) co
         case AttrDef::Kind::InheritedFrom: {
             auto & select = dynamic_cast<ExprSelect &>(*i->second.e);
             auto & from = dynamic_cast<ExprInheritFrom &>(*select.e);
-            inheritsFrom[&from].push_back(i->first);
+            inheritsFrom[from.displ].push_back(i->first);
             break;
         }
         }
@@ -103,7 +107,7 @@ void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) co
     }
     for (const auto & [from, syms] : inheritsFrom) {
         str << "inherit (";
-        (*inheritFromExprs)[from->displ]->show(symbols, str);
+        (*inheritFromExprs)[from]->show(symbols, str);
         str << ")";
         for (auto sym : syms) str << " " << symbols[sym];
         str << "; ";
@@ -149,7 +153,10 @@ void ExprLambda::show(const SymbolTable & symbols, std::ostream & str) const
     if (hasFormals()) {
         str << "{ ";
         bool first = true;
-        for (auto & i : formals->formals) {
+        // the natural Symbol ordering is by creation time, which can lead to the
+        // same expression being printed in two different ways depending on its
+        // context. always use lexicographic ordering to avoid this.
+        for (auto & i : formals->lexicographicOrder(symbols)) {
             if (first) first = false; else str << ", ";
             str << symbols[i.name];
             if (i.def) {
@@ -264,7 +271,7 @@ std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath)
 
 void Expr::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
 {
-    abort();
+    unreachable();
 }
 
 void ExprInt::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
@@ -578,6 +585,55 @@ std::string ExprLambda::showNamePos(const EvalState & state) const
     return fmt("%1% at %2%", id, state.positions[pos]);
 }
 
+void ExprLambda::setDocComment(DocComment docComment) {
+    // RFC 145 specifies that the innermost doc comment wins.
+    // See https://github.com/NixOS/rfcs/blob/master/rfcs/0145-doc-strings.md#ambiguous-placement
+    if (!this->docComment) {
+        this->docComment = docComment;
+
+        // Curried functions are defined by putting a function directly
+        // in the body of another function. To render docs for those, we
+        // need to propagate the doc comment to the innermost function.
+        //
+        // If we have our own comment, we've already propagated it, so this
+        // belongs in the same conditional.
+        body->setDocComment(docComment);
+    }
+};
+
+
+
+/* Position table. */
+
+Pos PosTable::operator[](PosIdx p) const
+{
+    auto origin = resolve(p);
+    if (!origin)
+        return {};
+
+    const auto offset = origin->offsetOf(p);
+
+    Pos result{0, 0, origin->origin};
+    auto lines = this->lines.lock();
+    auto linesForInput = (*lines)[origin->offset];
+
+    if (linesForInput.empty()) {
+        auto source = result.getSource().value_or("");
+        const char * begin = source.data();
+        for (Pos::LinesIterator it(source), end; it != end; it++)
+            linesForInput.push_back(it->data() - begin);
+        if (linesForInput.empty())
+            linesForInput.push_back(0);
+    }
+    // as above: the first line starts at byte 0 and is always present
+    auto lineStartOffset = std::prev(
+        std::upper_bound(linesForInput.begin(), linesForInput.end(), offset));
+
+    result.line = 1 + (lineStartOffset - linesForInput.begin());
+    result.column = 1 + (offset - *lineStartOffset);
+    return result;
+}
+
 
 
 /* Symbol table. */
@@ -589,4 +645,22 @@ size_t SymbolTable::totalSize() const
     return n;
 }
 
+std::string DocComment::getInnerText(const PosTable & positions) const {
+    auto beginPos = positions[begin];
+    auto endPos = positions[end];
+    auto docCommentStr = beginPos.getSnippetUpTo(endPos).value_or("");
+
+    // Strip "/**" and "*/"
+    constexpr size_t prefixLen = 3;
+    constexpr size_t suffixLen = 2;
+    std::string docStr = docCommentStr.substr(prefixLen, docCommentStr.size() - prefixLen - suffixLen);
+    if (docStr.empty())
+        return {};
+    // Turn the now missing "/**" into indentation
+    docStr = "   " + docStr;
+    // Strip indentation (for the whole, potentially multi-line string)
+    docStr = stripIndentation(docStr);
+    return docStr;
+}
+
 }

src/libexpr/nixexpr.hh

@@ -6,22 +6,58 @@
 
 #include "value.hh"
 #include "symbol-table.hh"
-#include "error.hh"
-#include "chunked-vector.hh"
-#include "position.hh"
 #include "eval-error.hh"
 #include "pos-idx.hh"
-#include "pos-table.hh"
 
 namespace nix {
 
-
-struct Env;
-struct Value;
 class EvalState;
+class PosTable;
+struct Env;
 struct ExprWith;
 struct StaticEnv;
+struct Value;
 
+/**
+ * A documentation comment, in the sense of [RFC 145](https://github.com/NixOS/rfcs/blob/master/rfcs/0145-doc-strings.md)
+ *
+ * Note that this does not implement the following:
+ *  - argument attribute names ("formals"): TBD
+ *  - argument names: these are internal to the function and their names may not be optimal for documentation
+ *  - function arity (degree of currying or number of ':'s):
+ *      - Functions returning partially applied functions have a higher arity
+ *        than can be determined locally and without evaluation.
+ *        We do not want to present false data.
+ *      - Some functions should be thought of as transformations of other
+ *        functions. For instance `overlay -> overlay -> overlay` is the simplest
+ *        way to understand `composeExtensions`, but its implementation looks like
+ *        `f: g: final: prev: <...>`. The parameters `final` and `prev` are part
+ *        of the overlay concept, while distracting from the function's purpose.
+ */
+struct DocComment {
+
+    /**
+     * Start of the comment, including the opening, ie `/` and `**`.
+     */
+    PosIdx begin;
+
+    /**
+     * Position right after the final asterisk and `/` that terminate the comment.
+     */
+    PosIdx end;
+
+    /**
+     * Whether the comment is set.
+     *
+     * A `DocComment` is small enough that it makes sense to pass by value, and
+     * therefore baking optionality into it is also useful, to avoiding the memory
+     * overhead of `std::optional`.
+     */
+    operator bool() const { return static_cast<bool>(begin); }
+
+    std::string getInnerText(const PosTable & positions) const;
+
+};
 
 /**
  * An attribute path is a sequence of attribute names.
@@ -58,6 +94,7 @@ struct Expr
     virtual void eval(EvalState & state, Env & env, Value & v);
     virtual Value * maybeThunk(EvalState & state, Env & env);
     virtual void setName(Symbol name);
+    virtual void setDocComment(DocComment docComment) { };
     virtual PosIdx getPos() const { return noPos; }
 };
 
@@ -93,10 +130,10 @@ struct ExprString : Expr
 
 struct ExprPath : Expr
 {
-    ref<InputAccessor> accessor;
+    ref<SourceAccessor> accessor;
     std::string s;
     Value v;
-    ExprPath(ref<InputAccessor> accessor, std::string s) : accessor(accessor), s(std::move(s))
+    ExprPath(ref<SourceAccessor> accessor, std::string s) : accessor(accessor), s(std::move(s))
     {
         v.mkPath(&*accessor, this->s.c_str());
     }
@@ -149,7 +186,7 @@ struct ExprInheritFrom : ExprVar
         this->fromWith = nullptr;
     }
 
-    void bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env);
+    void bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env) override;
 };
 
 struct ExprSelect : Expr
@@ -160,6 +197,17 @@ struct ExprSelect : Expr
     ExprSelect(const PosIdx & pos, Expr * e, AttrPath attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(std::move(attrPath)) { };
     ExprSelect(const PosIdx & pos, Expr * e, Symbol name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); };
     PosIdx getPos() const override { return pos; }
+
+    /**
+     * Evaluate the `a.b.c` part of `a.b.c.d`. This exists mostly for the purpose of :doc in the repl.
+     *
+     * @param[out] v The attribute set that should contain the last attribute name (if it exists).
+     * @return The last attribute name in `attrPath`
+     *
+     * @note This does *not* evaluate the final attribute, and does not fail if that's the only attribute that does not exist.
+     */
+    Symbol evalExceptFinalSelect(EvalState & state, Env & env, Value & attrs);
+
     COMMON_METHODS
 };
 
@@ -282,6 +330,8 @@ struct ExprLambda : Expr
     Symbol arg;
     Formals * formals;
     Expr * body;
+    DocComment docComment;
+
     ExprLambda(PosIdx pos, Symbol arg, Formals * formals, Expr * body)
         : pos(pos), arg(arg), formals(formals), body(body)
     {
@@ -294,6 +344,7 @@ struct ExprLambda : Expr
     std::string showNamePos(const EvalState & state) const;
     inline bool hasFormals() const { return formals != nullptr; }
     PosIdx getPos() const override { return pos; }
+    virtual void setDocComment(DocComment docComment) override;
     COMMON_METHODS
 };
 

src/libexpr/package.nix

@@ -0,0 +1,113 @@
+{ lib
+, stdenv
+, mkMesonDerivation
+, releaseTools
+
+, meson
+, ninja
+, pkg-config
+, bison
+, flex
+, cmake # for resolving toml11 dep
+
+, nix-util
+, nix-store
+, nix-fetchers
+, boost
+, boehmgc
+, nlohmann_json
+, toml11
+
+# Configuration Options
+
+, version
+
+# Whether to use garbage collection for the Nix language evaluator.
+#
+# If it is disabled, we just leak memory, but this is not as bad as it
+# sounds so long as evaluation just takes places within short-lived
+# processes. (When the process exits, the memory is reclaimed; it is
+# only leaked *within* the process.)
+#
+# Temporarily disabled on Windows because the `GC_throw_bad_alloc`
+# symbol is missing during linking.
+, enableGC ? !stdenv.hostPlatform.isWindows
+}:
+
+let
+  inherit (lib) fileset;
+in
+
+mkMesonDerivation (finalAttrs: {
+  pname = "nix-expr";
+  inherit version;
+
+  workDir = ./.;
+  fileset = fileset.unions [
+    ../../build-utils-meson
+    ./build-utils-meson
+    ../../.version
+    ./.version
+    ./meson.build
+    ./meson.options
+    ./primops/meson.build
+    (fileset.fileFilter (file: file.hasExt "cc") ./.)
+    (fileset.fileFilter (file: file.hasExt "hh") ./.)
+    ./lexer.l
+    ./parser.y
+    (fileset.fileFilter (file: file.hasExt "nix") ./.)
+  ];
+
+  outputs = [ "out" "dev" ];
+
+  nativeBuildInputs = [
+    meson
+    ninja
+    pkg-config
+    bison
+    flex
+    cmake
+  ];
+
+  buildInputs = [
+    toml11
+  ];
+
+  propagatedBuildInputs = [
+    nix-util
+    nix-store
+    nix-fetchers
+    boost
+    nlohmann_json
+  ] ++ lib.optional enableGC boehmgc;
+
+  preConfigure =
+    # "Inline" .version so it's not a symlink, and includes the suffix.
+    # Do the meson utils, without modification.
+    ''
+      chmod u+w ./.version
+      echo ${version} > ../../.version
+    '';
+
+  mesonFlags = [
+    (lib.mesonEnable "gc" enableGC)
+  ];
+
+  env = {
+    # Needed for Meson to find Boost.
+    # https://github.com/NixOS/nixpkgs/issues/86131.
+    BOOST_INCLUDEDIR = "${lib.getDev boost}/include";
+    BOOST_LIBRARYDIR = "${lib.getLib boost}/lib";
+  } // lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) {
+    LDFLAGS = "-fuse-ld=gold";
+  };
+
+  separateDebugInfo = !stdenv.hostPlatform.isStatic;
+
+  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
+
+  meta = {
+    platforms = lib.platforms.unix ++ lib.platforms.windows;
+  };
+
+})

src/libexpr/parser-state.hh

@@ -1,6 +1,8 @@
 #pragma once
 ///@file
 
+#include <limits>
+
 #include "eval.hh"
 
 namespace nix {
@@ -20,37 +22,67 @@ struct StringToken
 
 struct ParserLocation
 {
-    int first_line, first_column;
-    int last_line, last_column;
+    int beginOffset;
+    int endOffset;
 
     // backup to recover from yyless(0)
-    int stashed_first_line, stashed_first_column;
-    int stashed_last_line, stashed_last_column;
+    int stashedBeginOffset, stashedEndOffset;
 
     void stash() {
-        stashed_first_line = first_line;
-        stashed_first_column = first_column;
-        stashed_last_line = last_line;
-        stashed_last_column = last_column;
+        stashedBeginOffset = beginOffset;
+        stashedEndOffset = endOffset;
     }
 
     void unstash() {
-        first_line = stashed_first_line;
-        first_column = stashed_first_column;
-        last_line = stashed_last_line;
-        last_column = stashed_last_column;
+        beginOffset = stashedBeginOffset;
+        endOffset = stashedEndOffset;
     }
+
+    /** Latest doc comment position, or 0. */
+    int doc_comment_first_column, doc_comment_last_column;
+};
+
+struct LexerState
+{
+    /**
+     * Tracks the distance to the last doc comment, in terms of lexer tokens.
+     *
+     * The lexer sets this to 0 when reading a doc comment, and increments it
+     * for every matched rule; see `lexer-helpers.cc`.
+     * Whitespace and comment rules decrement the distance, so that they result
+     * in a net 0 change in distance.
+     */
+    int docCommentDistance = std::numeric_limits<int>::max();
+
+    /**
+     * The location of the last doc comment.
+     *
+     * (stashing fields are not used)
+     */
+    ParserLocation lastDocCommentLoc;
+
+    /**
+     * @brief Maps some positions to a DocComment, where the comment is relevant to the location.
+     */
+    std::unordered_map<PosIdx, DocComment> & positionToDocComment;
+
+    PosTable & positions;
+    PosTable::Origin origin;
+
+    PosIdx at(const ParserLocation & loc);
 };
 
 struct ParserState
 {
+    const LexerState & lexerState;
     SymbolTable & symbols;
     PosTable & positions;
     Expr * result;
     SourcePath basePath;
     PosTable::Origin origin;
-    const ref<InputAccessor> rootFS;
+    const ref<SourceAccessor> rootFS;
     const Expr::AstSymbols & s;
+    const EvalSettings & settings;
 
     void dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos);
     void dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos);
@@ -259,12 +291,23 @@ inline Expr * ParserState::stripIndentation(const PosIdx pos,
                 s2 = std::string(s2, 0, p + 1);
         }
 
-        es2->emplace_back(i->first, new ExprString(std::move(s2)));
+        // Ignore empty strings for a minor optimisation and AST simplification
+        if (s2 != "") {
+            es2->emplace_back(i->first, new ExprString(std::move(s2)));
+        }
     };
     for (; i != es.end(); ++i, --n) {
         std::visit(overloaded { trimExpr, trimString }, i->second);
     }
 
+    // If there is nothing at all, return the empty string directly.
+    // This also ensures that equivalent empty strings result in the same ast, which is helpful when testing formatters.
+    if (es2->size() == 0) {
+        auto *const result = new ExprString("");
+        delete es2;
+        return result;
+    }
+
     /* If this is a single string, then don't do a concatenation. */
     if (es2->size() == 1 && dynamic_cast<ExprString *>((*es2)[0].second)) {
         auto *const result = (*es2)[0].second;
@@ -274,9 +317,14 @@ inline Expr * ParserState::stripIndentation(const PosIdx pos,
     return new ExprConcatStrings(pos, true, es2);
 }
 
+inline PosIdx LexerState::at(const ParserLocation & loc)
+{
+    return positions.add(origin, loc.beginOffset);
+}
+
 inline PosIdx ParserState::at(const ParserLocation & loc)
 {
-    return positions.add(origin, loc.first_line, loc.first_column);
+    return positions.add(origin, loc.beginOffset);
 }
 
 }

src/libexpr/parser.y

@@ -25,23 +25,41 @@
 #include "nixexpr.hh"
 #include "eval.hh"
 #include "eval-settings.hh"
-#include "globals.hh"
 #include "parser-state.hh"
 
 #define YYLTYPE ::nix::ParserLocation
 #define YY_DECL int yylex \
     (YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParserState * state)
 
+// For efficiency, we only track offsets; not line,column coordinates
+# define YYLLOC_DEFAULT(Current, Rhs, N)                                \
+    do                                                                  \
+      if (N)                                                            \
+        {                                                               \
+          (Current).beginOffset = YYRHSLOC (Rhs, 1).beginOffset;        \
+          (Current).endOffset  = YYRHSLOC (Rhs, N).endOffset;           \
+        }                                                               \
+      else                                                              \
+        {                                                               \
+          (Current).beginOffset = (Current).endOffset =                 \
+            YYRHSLOC (Rhs, 0).endOffset;                                \
+        }                                                               \
+    while (0)
+
 namespace nix {
 
+typedef std::unordered_map<PosIdx, DocComment> DocCommentMap;
+
 Expr * parseExprFromBuf(
     char * text,
     size_t length,
     Pos::Origin origin,
     const SourcePath & basePath,
     SymbolTable & symbols,
+    const EvalSettings & settings,
     PosTable & positions,
-    const ref<InputAccessor> rootFS,
+    DocCommentMap & docComments,
+    const ref<SourceAccessor> rootFS,
     const Expr::AstSymbols & astSymbols);
 
 }
@@ -64,12 +82,31 @@ using namespace nix;
 
 void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * error)
 {
+    if (std::string_view(error).starts_with("syntax error, unexpected end of file")) {
+        loc->beginOffset = loc->endOffset;
+    }
     throw ParseError({
         .msg = HintFmt(error),
         .pos = state->positions[state->at(*loc)]
     });
 }
 
+#define SET_DOC_POS(lambda, pos) setDocPosition(state->lexerState, lambda, state->at(pos))
+static void setDocPosition(const LexerState & lexerState, ExprLambda * lambda, PosIdx start) {
+    auto it = lexerState.positionToDocComment.find(start);
+    if (it != lexerState.positionToDocComment.end()) {
+        lambda->setDocComment(it->second);
+    }
+}
+
+static Expr * makeCall(PosIdx pos, Expr * fn, Expr * arg) {
+    if (auto e2 = dynamic_cast<ExprCall *>(fn)) {
+        e2->args.push_back(arg);
+        return fn;
+    }
+    return new ExprCall(pos, fn, {arg});
+}
+
 
 %}
 
@@ -87,17 +124,20 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char *
   nix::StringToken uri;
   nix::StringToken str;
   std::vector<nix::AttrName> * attrNames;
+  std::vector<std::pair<nix::AttrName, nix::PosIdx>> * inheritAttrs;
   std::vector<std::pair<nix::PosIdx, nix::Expr *>> * string_parts;
   std::vector<std::pair<nix::PosIdx, std::variant<nix::Expr *, nix::StringToken>>> * ind_string_parts;
 }
 
 %type <e> start expr expr_function expr_if expr_op
 %type <e> expr_select expr_simple expr_app
+%type <e> expr_pipe_from expr_pipe_into
 %type <list> expr_list
 %type <attrs> binds
 %type <formals> formals
 %type <formal> formal
-%type <attrNames> attrs attrpath
+%type <attrNames> attrpath
+%type <inheritAttrs> attrs
 %type <string_parts> string_parts_interpolated
 %type <ind_string_parts> ind_string_parts
 %type <e> path_start string_parts string_attr
@@ -109,10 +149,12 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char *
 %token <path> PATH HPATH SPATH PATH_END
 %token <uri> URI
 %token IF THEN ELSE ASSERT WITH LET IN_KW REC INHERIT EQ NEQ AND OR IMPL OR_KW
+%token PIPE_FROM PIPE_INTO /* <| and |> */
 %token DOLLAR_CURLY /* == ${ */
 %token IND_STRING_OPEN IND_STRING_CLOSE
 %token ELLIPSIS
 
+
 %right IMPL
 %left OR
 %left AND
@@ -134,18 +176,28 @@ expr: expr_function;
 
 expr_function
   : ID ':' expr_function
-    { $$ = new ExprLambda(CUR_POS, state->symbols.create($1), 0, $3); }
+    { auto me = new ExprLambda(CUR_POS, state->symbols.create($1), 0, $3);
+      $$ = me;
+      SET_DOC_POS(me, @1);
+    }
   | '{' formals '}' ':' expr_function
-    { $$ = new ExprLambda(CUR_POS, state->validateFormals($2), $5); }
+    { auto me = new ExprLambda(CUR_POS, state->validateFormals($2), $5);
+      $$ = me;
+      SET_DOC_POS(me, @1);
+    }
   | '{' formals '}' '@' ID ':' expr_function
     {
       auto arg = state->symbols.create($5);
-      $$ = new ExprLambda(CUR_POS, arg, state->validateFormals($2, CUR_POS, arg), $7);
+      auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($2, CUR_POS, arg), $7);
+      $$ = me;
+      SET_DOC_POS(me, @1);
     }
   | ID '@' '{' formals '}' ':' expr_function
     {
       auto arg = state->symbols.create($1);
-      $$ = new ExprLambda(CUR_POS, arg, state->validateFormals($4, CUR_POS, arg), $7);
+      auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($4, CUR_POS, arg), $7);
+      $$ = me;
+      SET_DOC_POS(me, @1);
     }
   | ASSERT expr ';' expr_function
     { $$ = new ExprAssert(CUR_POS, $2, $4); }
@@ -164,9 +216,21 @@ expr_function
 
 expr_if
   : IF expr THEN expr ELSE expr { $$ = new ExprIf(CUR_POS, $2, $4, $6); }
+  | expr_pipe_from
+  | expr_pipe_into
   | expr_op
   ;
 
+expr_pipe_from
+  : expr_op PIPE_FROM expr_pipe_from { $$ = makeCall(state->at(@2), $1, $3); }
+  | expr_op PIPE_FROM expr_op        { $$ = makeCall(state->at(@2), $1, $3); }
+  ;
+
+expr_pipe_into
+  : expr_pipe_into PIPE_INTO expr_op { $$ = makeCall(state->at(@2), $3, $1); }
+  | expr_op        PIPE_INTO expr_op { $$ = makeCall(state->at(@2), $3, $1); }
+  ;
+
 expr_op
   : '!' expr_op %prec NOT { $$ = new ExprOpNot($2); }
   | '-' expr_op %prec NEGATE { $$ = new ExprCall(CUR_POS, new ExprVar(state->s.sub), {new ExprInt(0), $2}); }
@@ -191,13 +255,7 @@ expr_op
   ;
 
 expr_app
-  : expr_app expr_select {
-      if (auto e2 = dynamic_cast<ExprCall *>($1)) {
-          e2->args.push_back($2);
-          $$ = $1;
-      } else
-          $$ = new ExprCall(CUR_POS, $1, {$2});
-  }
+  : expr_app expr_select { $$ = makeCall(CUR_POS, $1, $2); }
   | expr_select
   ;
 
@@ -285,17 +343,17 @@ path_start
     /* add back in the trailing '/' to the first segment */
     if ($1.p[$1.l-1] == '/' && $1.l > 1)
       path += "/";
-    $$ = new ExprPath(ref<InputAccessor>(state->rootFS), std::move(path));
+    $$ = new ExprPath(ref<SourceAccessor>(state->rootFS), std::move(path));
   }
   | HPATH {
-    if (evalSettings.pureEval) {
+    if (state->settings.pureEval) {
         throw Error(
             "the path '%s' can not be resolved in pure mode",
             std::string_view($1.p, $1.l)
         );
     }
     Path path(getHome() + std::string($1.p + 1, $1.l - 1));
-    $$ = new ExprPath(ref<InputAccessor>(state->rootFS), std::move(path));
+    $$ = new ExprPath(ref<SourceAccessor>(state->rootFS), std::move(path));
   }
   ;
 
@@ -306,16 +364,30 @@ ind_string_parts
   ;
 
 binds
-  : binds attrpath '=' expr ';' { $$ = $1; state->addAttr($$, std::move(*$2), $4, state->at(@2)); delete $2; }
+  : binds attrpath '=' expr ';' {
+      $$ = $1;
+
+      auto pos = state->at(@2);
+      auto exprPos = state->at(@4);
+      {
+        auto it = state->lexerState.positionToDocComment.find(pos);
+        if (it != state->lexerState.positionToDocComment.end()) {
+          $4->setDocComment(it->second);
+          state->lexerState.positionToDocComment.emplace(exprPos, it->second);
+        }
+      }
+
+      state->addAttr($$, std::move(*$2), $4, pos);
+      delete $2;
+    }
   | binds INHERIT attrs ';'
     { $$ = $1;
-      for (auto & i : *$3) {
+      for (auto & [i, iPos] : *$3) {
           if ($$->attrs.find(i.symbol) != $$->attrs.end())
-              state->dupAttr(i.symbol, state->at(@3), $$->attrs[i.symbol].pos);
-          auto pos = state->at(@3);
+              state->dupAttr(i.symbol, iPos, $$->attrs[i.symbol].pos);
           $$->attrs.emplace(
               i.symbol,
-              ExprAttrs::AttrDef(new ExprVar(CUR_POS, i.symbol), pos, ExprAttrs::AttrDef::Kind::Inherited));
+              ExprAttrs::AttrDef(new ExprVar(iPos, i.symbol), iPos, ExprAttrs::AttrDef::Kind::Inherited));
       }
       delete $3;
     }
@@ -325,14 +397,14 @@ binds
           $$->inheritFromExprs = std::make_unique<std::vector<Expr *>>();
       $$->inheritFromExprs->push_back($4);
       auto from = new nix::ExprInheritFrom(state->at(@4), $$->inheritFromExprs->size() - 1);
-      for (auto & i : *$6) {
+      for (auto & [i, iPos] : *$6) {
           if ($$->attrs.find(i.symbol) != $$->attrs.end())
-              state->dupAttr(i.symbol, state->at(@6), $$->attrs[i.symbol].pos);
+              state->dupAttr(i.symbol, iPos, $$->attrs[i.symbol].pos);
           $$->attrs.emplace(
               i.symbol,
               ExprAttrs::AttrDef(
-                  new ExprSelect(CUR_POS, from, i.symbol),
-                  state->at(@6),
+                  new ExprSelect(iPos, from, i.symbol),
+                  iPos,
                   ExprAttrs::AttrDef::Kind::InheritedFrom));
       }
       delete $6;
@@ -341,12 +413,12 @@ binds
   ;
 
 attrs
-  : attrs attr { $$ = $1; $1->push_back(AttrName(state->symbols.create($2))); }
+  : attrs attr { $$ = $1; $1->emplace_back(AttrName(state->symbols.create($2)), state->at(@2)); }
   | attrs string_attr
     { $$ = $1;
       ExprString * str = dynamic_cast<ExprString *>($2);
       if (str) {
-          $$->push_back(AttrName(state->symbols.create(str->s)));
+          $$->emplace_back(AttrName(state->symbols.create(str->s)), state->at(@2));
           delete str;
       } else
           throw ParseError({
@@ -354,7 +426,7 @@ attrs
               .pos = state->positions[state->at(@2)]
           });
     }
-  | { $$ = new AttrPath; }
+  | { $$ = new std::vector<std::pair<AttrName, PosIdx>>; }
   ;
 
 attrpath
@@ -424,21 +496,30 @@ Expr * parseExprFromBuf(
     Pos::Origin origin,
     const SourcePath & basePath,
     SymbolTable & symbols,
+    const EvalSettings & settings,
     PosTable & positions,
-    const ref<InputAccessor> rootFS,
+    DocCommentMap & docComments,
+    const ref<SourceAccessor> rootFS,
     const Expr::AstSymbols & astSymbols)
 {
     yyscan_t scanner;
+    LexerState lexerState {
+        .positionToDocComment = docComments,
+        .positions = positions,
+        .origin = positions.addOrigin(origin, length),
+    };
     ParserState state {
+        .lexerState = lexerState,
         .symbols = symbols,
         .positions = positions,
         .basePath = basePath,
-        .origin = {origin},
+        .origin = lexerState.origin,
         .rootFS = rootFS,
         .s = astSymbols,
+        .settings = settings,
     };
 
-    yylex_init(&scanner);
+    yylex_init_extra(&lexerState, &scanner);
     Finally _destroy([&] { yylex_destroy(scanner); });
 
     yy_scan_buffer(text, length, scanner);

src/libexpr/pos-idx.hh

@@ -1,12 +1,15 @@
 #pragma once
 
 #include <cinttypes>
+#include <functional>
 
 namespace nix {
 
 class PosIdx
 {
+    friend struct LazyPosAcessors;
     friend class PosTable;
+    friend class std::hash<PosIdx>;
 
 private:
     uint32_t id;
@@ -27,9 +30,9 @@ public:
         return id > 0;
     }
 
-    bool operator<(const PosIdx other) const
+    auto operator<=>(const PosIdx other) const
     {
-        return id < other.id;
+        return id <=> other.id;
     }
 
     bool operator==(const PosIdx other) const
@@ -37,12 +40,25 @@ public:
         return id == other.id;
     }
 
-    bool operator!=(const PosIdx other) const
+    size_t hash() const noexcept
     {
-        return id != other.id;
+        return std::hash<uint32_t>{}(id);
     }
 };
 
 inline PosIdx noPos = {};
 
 }
+
+namespace std {
+
+template<>
+struct hash<nix::PosIdx>
+{
+    std::size_t operator()(nix::PosIdx pos) const noexcept
+    {
+        return pos.hash();
+    }
+};
+
+} // namespace std

src/libexpr/pos-table.hh

@@ -1,12 +1,11 @@
 #pragma once
 
-#include <cinttypes>
-#include <numeric>
+#include <cstdint>
 #include <vector>
 
-#include "chunked-vector.hh"
 #include "pos-idx.hh"
 #include "position.hh"
+#include "sync.hh"
 
 namespace nix {
 
@@ -17,66 +16,69 @@ public:
     {
         friend PosTable;
     private:
-        // must always be invalid by default, add() replaces this with the actual value.
-        // subsequent add() calls use this index as a token to quickly check whether the
-        // current origins.back() can be reused or not.
-        mutable uint32_t idx = std::numeric_limits<uint32_t>::max();
+        uint32_t offset;
 
-        // Used for searching in PosTable::[].
-        explicit Origin(uint32_t idx)
-            : idx(idx)
-            , origin{std::monostate()}
-        {
-        }
+        Origin(Pos::Origin origin, uint32_t offset, size_t size):
+            offset(offset), origin(origin), size(size)
+        {}
 
     public:
         const Pos::Origin origin;
+        const size_t size;
 
-        Origin(Pos::Origin origin)
-            : origin(origin)
+        uint32_t offsetOf(PosIdx p) const
         {
+            return p.id - 1 - offset;
         }
     };
 
-    struct Offset
-    {
-        uint32_t line, column;
-    };
-
 private:
-    std::vector<Origin> origins;
-    ChunkedVector<Offset, 8192> offsets;
+    using Lines = std::vector<uint32_t>;
 
-public:
-    PosTable()
-        : offsets(1024)
-    {
-        origins.reserve(1024);
-    }
+    std::map<uint32_t, Origin> origins;
+    mutable Sync<std::map<uint32_t, Lines>> lines;
 
-    PosIdx add(const Origin & origin, uint32_t line, uint32_t column)
+    const Origin * resolve(PosIdx p) const
     {
-        const auto idx = offsets.add({line, column}).second;
-        if (origins.empty() || origins.back().idx != origin.idx) {
-            origin.idx = idx;
-            origins.push_back(origin);
-        }
-        return PosIdx(idx + 1);
-    }
+        if (p.id == 0)
+            return nullptr;
 
-    Pos operator[](PosIdx p) const
-    {
-        if (p.id == 0 || p.id > offsets.size())
-            return {};
         const auto idx = p.id - 1;
         /* we want the last key <= idx, so we'll take prev(first key > idx).
-           this is guaranteed to never rewind origin.begin because the first
-           key is always 0. */
-        const auto pastOrigin = std::upper_bound(
-            origins.begin(), origins.end(), Origin(idx), [](const auto & a, const auto & b) { return a.idx < b.idx; });
-        const auto origin = *std::prev(pastOrigin);
-        const auto offset = offsets[idx];
-        return {offset.line, offset.column, origin.origin};
+            this is guaranteed to never rewind origin.begin because the first
+            key is always 0. */
+        const auto pastOrigin = origins.upper_bound(idx);
+        return &std::prev(pastOrigin)->second;
+    }
+
+public:
+    Origin addOrigin(Pos::Origin origin, size_t size)
+    {
+        uint32_t offset = 0;
+        if (auto it = origins.rbegin(); it != origins.rend())
+            offset = it->first + it->second.size;
+        // +1 because all PosIdx are offset by 1 to begin with, and
+        // another +1 to ensure that all origins can point to EOF, eg
+        // on (invalid) empty inputs.
+        if (2 + offset + size < offset)
+            return Origin{origin, offset, 0};
+        return origins.emplace(offset, Origin{origin, offset, size}).first->second;
+    }
+
+    PosIdx add(const Origin & origin, size_t offset)
+    {
+        if (offset > origin.size)
+            return PosIdx();
+        return PosIdx(1 + origin.offset + offset);
+    }
+
+    Pos operator[](PosIdx p) const;
+
+    Pos::Origin originOf(PosIdx p) const
+    {
+        if (auto o = resolve(p))
+            return o->origin;
+        return std::monostate{};
     }
 };
 

src/libexpr/primops.hh

@@ -51,4 +51,6 @@ void prim_importNative(EvalState & state, const PosIdx pos, Value * * args, Valu
  */
 void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v);
 
+void makePositionThunks(EvalState & state, const PosIdx pos, Value & line, Value & column);
+
 }

src/libexpr/primops/context.cc

@@ -14,8 +14,11 @@ static void prim_unsafeDiscardStringContext(EvalState & state, const PosIdx pos,
 
 static RegisterPrimOp primop_unsafeDiscardStringContext({
     .name = "__unsafeDiscardStringContext",
-    .arity = 1,
-    .fun = prim_unsafeDiscardStringContext
+    .args = {"s"},
+    .doc = R"(
+        Discard the [string context](@docroot@/language/string-context.md) from a value that can be coerced to a string.
+    )",
+    .fun = prim_unsafeDiscardStringContext,
 });
 
 
@@ -75,7 +78,11 @@ static RegisterPrimOp primop_unsafeDiscardOutputDependency({
     .name = "__unsafeDiscardOutputDependency",
     .args = {"s"},
     .doc = R"(
-      Create a copy of the given string where every "derivation deep" string context element is turned into a constant string context element.
+      Create a copy of the given string where every
+      [derivation deep](@docroot@/language/string-context.md#string-context-element-derivation-deep)
+      string context element is turned into a
+      [constant](@docroot@/language/string-context.md#string-context-element-constant)
+      string context element.
 
       This is the opposite of [`builtins.addDrvOutputDependencies`](#builtins-addDrvOutputDependencies).
 
@@ -137,14 +144,18 @@ static RegisterPrimOp primop_addDrvOutputDependencies({
     .name = "__addDrvOutputDependencies",
     .args = {"s"},
     .doc = R"(
-      Create a copy of the given string where a single consant string context element is turned into a "derivation deep" string context element.
+      Create a copy of the given string where a single
+      [constant](@docroot@/language/string-context.md#string-context-element-constant)
+      string context element is turned into a
+      [derivation deep](@docroot@/language/string-context.md#string-context-element-derivation-deep)
+      string context element.
 
       The store path that is the constant string context element should point to a valid derivation, and end in `.drv`.
 
       The original string context element must not be empty or have multiple elements, and it must not have any other type of element other than a constant or derivation deep element.
       The latter is supported so this function is idempotent.
 
-      This is the opposite of [`builtins.unsafeDiscardOutputDependency`](#builtins-addDrvOutputDependencies).
+      This is the opposite of [`builtins.unsafeDiscardOutputDependency`](#builtins-unsafeDiscardOutputDependency).
     )",
     .fun = prim_addDrvOutputDependencies
 });
@@ -207,10 +218,10 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args,
         if (info.second.allOutputs)
             infoAttrs.alloc(sAllOutputs).mkBool(true);
         if (!info.second.outputs.empty()) {
-            auto & outputsVal = infoAttrs.alloc(state.sOutputs);
-            state.mkList(outputsVal, info.second.outputs.size());
+            auto list = state.buildList(info.second.outputs.size());
             for (const auto & [i, output] : enumerate(info.second.outputs))
-                (outputsVal.listElems()[i] = state.allocValue())->mkString(output);
+                (list[i] = state.allocValue())->mkString(output);
+            infoAttrs.alloc(state.sOutputs).mkList(list);
         }
         attrs.alloc(state.store->printStorePath(info.first)).mkAttrs(infoAttrs);
     }
@@ -246,7 +257,7 @@ static RegisterPrimOp primop_getContext({
 
 /* Append the given context to a given string.
 
-   See the commentary above unsafeGetContext for details of the
+   See the commentary above getContext for details of the
    context representation.
 */
 static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * args, Value & v)
@@ -258,7 +269,7 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar
 
     auto sPath = state.symbols.create("path");
     auto sAllOutputs = state.symbols.create("allOutputs");
-    for (auto & i : *args[1]->attrs) {
+    for (auto & i : *args[1]->attrs()) {
         const auto & name = state.symbols[i.name];
         if (!state.store->isStorePath(name))
             state.error<EvalError>(
@@ -269,17 +280,16 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar
         if (!settings.readOnlyMode)
             state.store->ensurePath(namePath);
         state.forceAttrs(*i.value, i.pos, "while evaluating the value of a string context");
-        auto iter = i.value->attrs->find(sPath);
-        if (iter != i.value->attrs->end()) {
-            if (state.forceBool(*iter->value, iter->pos, "while evaluating the `path` attribute of a string context"))
+
+        if (auto attr = i.value->attrs()->get(sPath)) {
+            if (state.forceBool(*attr->value, attr->pos, "while evaluating the `path` attribute of a string context"))
                 context.emplace(NixStringContextElem::Opaque {
                     .path = namePath,
                 });
         }
 
-        iter = i.value->attrs->find(sAllOutputs);
-        if (iter != i.value->attrs->end()) {
-            if (state.forceBool(*iter->value, iter->pos, "while evaluating the `allOutputs` attribute of a string context")) {
+        if (auto attr = i.value->attrs()->get(sAllOutputs)) {
+            if (state.forceBool(*attr->value, attr->pos, "while evaluating the `allOutputs` attribute of a string context")) {
                 if (!isDerivation(name)) {
                     state.error<EvalError>(
                         "tried to add all-outputs context of %s, which is not a derivation, to a string",
@@ -292,17 +302,16 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar
             }
         }
 
-        iter = i.value->attrs->find(state.sOutputs);
-        if (iter != i.value->attrs->end()) {
-            state.forceList(*iter->value, iter->pos, "while evaluating the `outputs` attribute of a string context");
-            if (iter->value->listSize() && !isDerivation(name)) {
+        if (auto attr = i.value->attrs()->get(state.sOutputs)) {
+            state.forceList(*attr->value, attr->pos, "while evaluating the `outputs` attribute of a string context");
+            if (attr->value->listSize() && !isDerivation(name)) {
                 state.error<EvalError>(
                     "tried to add derivation output context of %s, which is not a derivation, to a string",
                     name
                 ).atPos(i.pos).debugThrow();
             }
-            for (auto elem : iter->value->listItems()) {
-                auto outputName = state.forceStringNoCtx(*elem, iter->pos, "while evaluating an output name within a string context");
+            for (auto elem : attr->value->listItems()) {
+                auto outputName = state.forceStringNoCtx(*elem, attr->pos, "while evaluating an output name within a string context");
                 context.emplace(NixStringContextElem::Built {
                     .drvPath = makeConstantStorePathRef(namePath),
                     .output = std::string { outputName },

src/libexpr/primops/derivation.nix

@@ -1,6 +1,31 @@
-/* This is the implementation of the ‘derivation’ builtin function.
-   It's actually a wrapper around the ‘derivationStrict’ primop. */
+# This is the implementation of the ‘derivation’ builtin function.
+# It's actually a wrapper around the ‘derivationStrict’ primop.
+# Note that the following comment will be shown in :doc in the repl, but not in the manual.
 
+/**
+  Create a derivation.
+
+  # Inputs
+
+  The single argument is an attribute set that describes what to build and how to build it.
+  See https://nix.dev/manual/nix/2.23/language/derivations
+
+  # Output
+
+  The result is an attribute set that describes the derivation.
+  Notably it contains the outputs, which in the context of the Nix language are special strings that refer to the output paths, which may not yet exist.
+  The realisation of these outputs only occurs when needed; for example
+
+    * When `nix-build` or a similar command is run, it realises the outputs that were requested on its command line.
+      See https://nix.dev/manual/nix/2.23/command-ref/nix-build
+
+    * When `import`, `readFile`, `readDir` or some other functions are called, they have to realise the outputs they depend on.
+      This is referred to as "import from derivation".
+      See https://nix.dev/manual/nix/2.23/language/import-from-derivation
+
+  Note that `derivation` is very bare-bones, and provides almost no commands during the build.
+  Most likely, you'll want to use functions like `stdenv.mkDerivation` in Nixpkgs to set up a basic environment.
+*/
 drvAttrs @ { outputs ? [ "out" ], ... }:
 
 let

src/libexpr/primops/fetchClosure.cc

@@ -121,7 +121,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg
     std::optional<StorePathOrGap> toPath;
     std::optional<bool> inputAddressedMaybe;
 
-    for (auto & attr : *args[0]->attrs) {
+    for (auto & attr : *args[0]->attrs()) {
         const auto & attrName = state.symbols[attr.name];
         auto attrHint = [&]() -> std::string {
             return "while evaluating the '" + attrName + "' attribute passed to builtins.fetchClosure";

src/libexpr/primops/fetchMercurial.cc

@@ -20,7 +20,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a
 
     if (args[0]->type() == nAttrs) {
 
-        for (auto & attr : *args[0]->attrs) {
+        for (auto & attr : *args[0]->attrs()) {
             std::string_view n(state.symbols[attr.name]);
             if (n == "url")
                 url = state.coerceToString(attr.pos, *attr.value, context,
@@ -53,7 +53,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a
     // whitelist. Ah well.
     state.checkURI(url);
 
-    if (evalSettings.pureEval && !rev)
+    if (state.settings.pureEval && !rev)
         throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision");
 
     fetchers::Attrs attrs;
@@ -62,10 +62,9 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a
     attrs.insert_or_assign("name", std::string(name));
     if (ref) attrs.insert_or_assign("ref", *ref);
     if (rev) attrs.insert_or_assign("rev", rev->gitRev());
-    auto input = fetchers::Input::fromAttrs(std::move(attrs));
+    auto input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs));
 
-    // FIXME: use name
-    auto [storePath, input2] = input.fetch(state.store);
+    auto [storePath, input2] = input.fetchToStore(state.store);
 
     auto attrs2 = state.buildBindings(8);
     state.mkStorePathString(storePath, attrs2.alloc(state.sOutPath));

src/libexpr/primops/fetchTree.cc

@@ -1,4 +1,4 @@
-#include "libfetchers/attrs.hh"
+#include "attrs.hh"
 #include "primops.hh"
 #include "eval-inline.hh"
 #include "eval-settings.hh"
@@ -85,7 +85,7 @@ static void fetchTree(
     Value & v,
     const FetchTreeParams & params = FetchTreeParams{}
 ) {
-    fetchers::Input input;
+    fetchers::Input input { state.fetchSettings };
     NixStringContext context;
     std::optional<std::string> type;
     if (params.isFetchGit) type = "git";
@@ -97,7 +97,7 @@ static void fetchTree(
 
         fetchers::Attrs attrs;
 
-        if (auto aType = args[0]->attrs->get(state.sType)) {
+        if (auto aType = args[0]->attrs()->get(state.sType)) {
             if (type)
                 state.error<EvalError>(
                     "unexpected attribute 'type'"
@@ -110,7 +110,7 @@ static void fetchTree(
 
         attrs.emplace("type", type.value());
 
-        for (auto & attr : *args[0]->attrs) {
+        for (auto & attr : *args[0]->attrs()) {
             if (attr.name == state.sType) continue;
             state.forceValue(*attr.value, attr.pos);
             if (attr.value->type() == nPath || attr.value->type() == nString) {
@@ -121,9 +121,9 @@ static void fetchTree(
                     : s);
             }
             else if (attr.value->type() == nBool)
-                attrs.emplace(state.symbols[attr.name], Explicit<bool>{attr.value->boolean});
+                attrs.emplace(state.symbols[attr.name], Explicit<bool>{attr.value->boolean()});
             else if (attr.value->type() == nInt)
-                attrs.emplace(state.symbols[attr.name], uint64_t(attr.value->integer));
+                attrs.emplace(state.symbols[attr.name], uint64_t(attr.value->integer()));
             else if (state.symbols[attr.name] == "publicKeys") {
                 experimentalFeatureSettings.require(Xp::VerifiedFetches);
                 attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, pos, context).dump());
@@ -137,13 +137,18 @@ static void fetchTree(
             attrs.emplace("exportIgnore", Explicit<bool>{true});
         }
 
+        // fetchTree should fetch git repos with shallow = true by default
+        if (type == "git" && !params.isFetchGit && !attrs.contains("shallow")) {
+            attrs.emplace("shallow", Explicit<bool>{true});
+        }
+
         if (!params.allowNameArgument)
             if (auto nameIter = attrs.find("name"); nameIter != attrs.end())
                 state.error<EvalError>(
                     "attribute 'name' isn’t supported in call to 'fetchTree'"
                 ).atPos(pos).debugThrow();
 
-        input = fetchers::Input::fromAttrs(std::move(attrs));
+        input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs));
     } else {
         auto url = state.coerceToString(pos, *args[0], context,
                 "while evaluating the first argument passed to the fetcher",
@@ -156,20 +161,20 @@ static void fetchTree(
             if (!attrs.contains("exportIgnore") && (!attrs.contains("submodules") || !*fetchers::maybeGetBoolAttr(attrs, "submodules"))) {
                 attrs.emplace("exportIgnore", Explicit<bool>{true});
             }
-            input = fetchers::Input::fromAttrs(std::move(attrs));
+            input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs));
         } else {
             if (!experimentalFeatureSettings.isEnabled(Xp::Flakes))
                 state.error<EvalError>(
                     "passing a string argument to 'fetchTree' requires the 'flakes' experimental feature"
                 ).atPos(pos).debugThrow();
-            input = fetchers::Input::fromURL(url);
+            input = fetchers::Input::fromURL(state.fetchSettings, url);
         }
     }
 
-    if (!evalSettings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes))
+    if (!state.settings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes))
         input = lookupInRegistries(state.store, input).first;
 
-    if (evalSettings.pureEval && !input.isLocked()) {
+    if (state.settings.pureEval && !input.isLocked()) {
         auto fetcher = "fetchTree";
         if (params.isFetchGit)
             fetcher = "fetchGit";
@@ -182,7 +187,7 @@ static void fetchTree(
 
     state.checkURI(input.toURLString());
 
-    auto [storePath, input2] = input.fetch(state.store);
+    auto [storePath, input2] = input.fetchToStore(state.store);
 
     state.allowPath(storePath);
 
@@ -200,8 +205,8 @@ static RegisterPrimOp primop_fetchTree({
     .doc = R"(
       Fetch a file system tree or a plain file using one of the supported backends and return an attribute set with:
 
-      - the resulting fixed-output [store path](@docroot@/glossary.md#gloss-store-path)
-      - the corresponding [NAR](@docroot@/glossary.md#gloss-nar) hash
+      - the resulting fixed-output [store path](@docroot@/store/store-path.md)
+      - the corresponding [NAR](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) hash
       - backend-specific metadata (currently not documented). <!-- TODO: document output attributes -->
 
       *input* must be an attribute set with the following attributes:
@@ -320,6 +325,8 @@ static RegisterPrimOp primop_fetchTree({
 
         - `ref` (String, optional)
 
+          By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
+
           A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name.
 
           Default: `"HEAD"`
@@ -333,8 +340,9 @@ static RegisterPrimOp primop_fetchTree({
         - `shallow` (Bool, optional)
 
           Make a shallow clone when fetching the Git tree.
+          When this is enabled, the options `ref` and `allRefs` have no effect anymore.
 
-          Default: `false`
+          Default: `true`
 
         - `submodules` (Bool, optional)
 
@@ -344,8 +352,11 @@ static RegisterPrimOp primop_fetchTree({
 
         - `allRefs` (Bool, optional)
 
-          If set to `true`, always fetch the entire repository, even if the latest commit is still in the cache.
-          Otherwise, only the latest commit is fetched if it is not already cached.
+          By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
+
+          Whether to fetch all references (eg. branches and tags) of the repository.
+          With this argument being true, it's possible to load a `rev` from *any* `ref`.
+          (Without setting this option, only `rev`s from the specified `ref` are supported).
 
           Default: `false`
 
@@ -372,7 +383,7 @@ static RegisterPrimOp primop_fetchTree({
       - `"mercurial"`
 
      *input* can also be a [URL-like reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references).
-     The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-flakes) to be enabled.
+     The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/development/experimental-features.md#xp-feature-flakes) to be enabled.
 
       > **Example**
       >
@@ -420,16 +431,21 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
 
     state.forceValue(*args[0], pos);
 
-    if (args[0]->type() == nAttrs) {
+    bool isArgAttrs = args[0]->type() == nAttrs;
+    bool nameAttrPassed = false;
 
-        for (auto & attr : *args[0]->attrs) {
+    if (isArgAttrs) {
+
+        for (auto & attr : *args[0]->attrs()) {
             std::string_view n(state.symbols[attr.name]);
             if (n == "url")
                 url = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the url we should fetch");
             else if (n == "sha256")
                 expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the sha256 of the content we should fetch"), HashAlgorithm::SHA256);
-            else if (n == "name")
+            else if (n == "name") {
+                nameAttrPassed = true;
                 name = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the name of the content we should fetch");
+            }
             else
                 state.error<EvalError>("unsupported argument '%s' to '%s'", n, who)
                 .atPos(pos).debugThrow();
@@ -442,14 +458,27 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
         url = state.forceStringNoCtx(*args[0], pos, "while evaluating the url we should fetch");
 
     if (who == "fetchTarball")
-        url = evalSettings.resolvePseudoUrl(*url);
+        url = state.settings.resolvePseudoUrl(*url);
 
     state.checkURI(*url);
 
     if (name == "")
         name = baseNameOf(*url);
 
-    if (evalSettings.pureEval && !expectedHash)
+    try {
+        checkName(name);
+    } catch (BadStorePathName & e) {
+        auto resolution =
+            nameAttrPassed ? HintFmt("Please change the value for the 'name' attribute passed to '%s', so that it can create a valid store path.", who) :
+            isArgAttrs ? HintFmt("Please add a valid 'name' attribute to the argument for '%s', so that it can create a valid store path.", who) :
+            HintFmt("Please pass an attribute set with 'url' and 'name' attributes to '%s',  so that it can create a valid store path.", who);
+
+        state.error<EvalError>(
+            std::string("invalid store path name when fetching URL '%s': %s. %s"), *url, Uncolored(e.message()), Uncolored(resolution.str()))
+        .atPos(pos).debugThrow();
+    }
+
+    if (state.settings.pureEval && !expectedHash)
         state.error<EvalError>("in pure evaluation mode, '%s' requires a 'sha256' argument", who).atPos(pos).debugThrow();
 
     // early exit if pinned and already in the store
@@ -457,7 +486,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
         auto expectedPath = state.store->makeFixedOutputPath(
             name,
             FixedOutputInfo {
-                .method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat,
+                .method = unpack ? FileIngestionMethod::NixArchive : FileIngestionMethod::Flat,
                 .hash = *expectedHash,
                 .references = {}
             });
@@ -473,7 +502,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
     auto storePath =
         unpack
         ? fetchToStore(*state.store, fetchers::downloadTarball(*url).accessor, FetchMode::Copy, name)
-        : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath;
+        : fetchers::downloadFile(state.store, *url, name).storePath;
 
     if (expectedHash) {
         auto hash = unpack
@@ -500,9 +529,19 @@ static void prim_fetchurl(EvalState & state, const PosIdx pos, Value * * args, V
 
 static RegisterPrimOp primop_fetchurl({
     .name = "__fetchurl",
-    .args = {"url"},
+    .args = {"arg"},
     .doc = R"(
       Download the specified URL and return the path of the downloaded file.
+      `arg` can be either a string denoting the URL, or an attribute set with the following attributes:
+
+      - `url`
+
+        The URL of the file to download.
+
+      - `name` (default: the last path component of the URL)
+
+        A name for the file in the store. This can be useful if the URL has any
+        characters that are invalid for the store.
 
       Not available in [restricted evaluation mode](@docroot@/command-ref/conf-file.md#conf-restrict-eval).
     )",
@@ -520,11 +559,11 @@ static RegisterPrimOp primop_fetchTarball({
     .doc = R"(
       Download the specified URL, unpack it and return the path of the
       unpacked tree. The file must be a tape archive (`.tar`) compressed
-      with `gzip`, `bzip2` or `xz`. The top-level path component of the
-      files in the tarball is removed, so it is best if the tarball
-      contains a single directory at top level. The typical use of the
-      function is to obtain external Nix expression dependencies, such as
-      a particular version of Nixpkgs, e.g.
+      with `gzip`, `bzip2` or `xz`. If the tarball consists of a
+      single directory, then the top-level path component of the files
+      in the tarball is removed. The typical use of the function is to
+      obtain external Nix expression dependencies, such as a
+      particular version of Nixpkgs, e.g.
 
       ```nix
       with import (fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz) {};
@@ -599,6 +638,8 @@ static RegisterPrimOp primop_fetchGit({
 
         [Git reference]: https://git-scm.com/book/en/v2/Git-Internals-Git-References
 
+        This option has no effect once `shallow` cloning is enabled.
+
         By default, the `ref` value is prefixed with `refs/heads/`.
         As of 2.3.0, Nix will not prefix `refs/heads/` if `ref` starts with `refs/`.
 
@@ -616,23 +657,25 @@ static RegisterPrimOp primop_fetchGit({
       - `shallow` (default: `false`)
 
         Make a shallow clone when fetching the Git tree.
-
+        When this is enabled, the options `ref` and `allRefs` have no effect anymore.
       - `allRefs`
 
-        Whether to fetch all references of the repository.
-        With this argument being true, it's possible to load a `rev` from *any* `ref`
+        Whether to fetch all references (eg. branches and tags) of the repository.
+        With this argument being true, it's possible to load a `rev` from *any* `ref`.
         (by default only `rev`s from the specified `ref` are supported).
 
+        This option has no effect once `shallow` cloning is enabled.
+
       - `verifyCommit` (default: `true` if `publicKey` or `publicKeys` are provided, otherwise `false`)
 
         Whether to check `rev` for a signature matching `publicKey` or `publicKeys`.
         If `verifyCommit` is enabled, then `fetchGit` cannot use a local repository with uncommitted changes.
-        Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches).
+        Requires the [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches).
 
       - `publicKey`
 
         The public key against which `rev` is verified if `verifyCommit` is enabled.
-        Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches).
+        Requires the [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches).
 
       - `keytype` (default: `"ssh-ed25519"`)
 
@@ -644,19 +687,21 @@ static RegisterPrimOp primop_fetchGit({
         - `"ssh-ed25519"`
         - `"ssh-ed25519-sk"`
         - `"ssh-rsa"`
-        Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches).
+        Requires the [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches).
 
       - `publicKeys`
 
         The public keys against which `rev` is verified if `verifyCommit` is enabled.
         Must be given as a list of attribute sets with the following form:
+
         ```nix
         {
           key = "<public key>";
           type = "<key type>"; # optional, default: "ssh-ed25519"
         }
         ```
-        Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches).
+
+        Requires the [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches).
 
 
       Here are some examples of how to use `fetchGit`.

src/libexpr/primops/fromTOML.cc

@@ -1,10 +1,10 @@
 #include "primops.hh"
 #include "eval-inline.hh"
 
-#include "../../toml11/toml.hpp"
-
 #include <sstream>
 
+#include <toml.hpp>
+
 namespace nix {
 
 static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, Value & val)
@@ -38,10 +38,10 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V
                 {
                     auto array = toml::get<std::vector<toml::value>>(t);
 
-                    size_t size = array.size();
-                    state.mkList(v, size);
-                    for (size_t i = 0; i < size; ++i)
-                        visit(*(v.listElems()[i] = state.allocValue()), array[i]);
+                    auto list = state.buildList(array.size());
+                    for (const auto & [n, v] : enumerate(list))
+                        visit(*(v = state.allocValue()), array[n]);
+                    v.mkList(list);
                 }
                 break;;
             case toml::value_t::boolean:

src/libexpr/primops/meson.build

@@ -0,0 +1,12 @@
+generated_headers += gen_header.process(
+  'derivation.nix',
+  preserve_path_from: meson.project_source_root(),
+)
+
+sources += files(
+  'context.cc',
+  'fetchClosure.cc',
+  'fetchMercurial.cc',
+  'fetchTree.cc',
+  'fromTOML.cc',
+)

src/libexpr/print-ambiguous.cc

@@ -21,10 +21,10 @@ void printAmbiguous(
     }
     switch (v.type()) {
     case nInt:
-        str << v.integer;
+        str << v.integer();
         break;
     case nBool:
-        printLiteralBool(str, v.boolean);
+        printLiteralBool(str, v.boolean());
         break;
     case nString:
         printLiteralString(str, v.string_view());
@@ -36,11 +36,11 @@ void printAmbiguous(
         str << "null";
         break;
     case nAttrs: {
-        if (seen && !v.attrs->empty() && !seen->insert(v.attrs).second)
+        if (seen && !v.attrs()->empty() && !seen->insert(v.attrs()).second)
             str << "«repeated»";
         else {
             str << "{ ";
-            for (auto & i : v.attrs->lexicographicOrder(symbols)) {
+            for (auto & i : v.attrs()->lexicographicOrder(symbols)) {
                 str << symbols[i->name] << " = ";
                 printAmbiguous(*i->value, symbols, str, seen, depth - 1);
                 str << "; ";
@@ -87,14 +87,14 @@ void printAmbiguous(
         }
         break;
     case nExternal:
-        str << *v.external;
+        str << *v.external();
         break;
     case nFloat:
-        str << v.fpoint;
+        str << v.fpoint();
         break;
     default:
         printError("Nix evaluator internal error: printAmbiguous: invalid value type");
-        abort();
+        unreachable();
     }
 }
 

src/libexpr/print-options.hh

@@ -8,6 +8,29 @@
 
 namespace nix {
 
+/**
+ * How errors should be handled when printing values.
+ */
+enum class ErrorPrintBehavior {
+    /**
+     * Print the first line of the error in brackets: `«error: oh no!»`
+     */
+    Print,
+    /**
+     * Throw the error to the code that attempted to print the value, instead
+     * of suppressing it it.
+     */
+    Throw,
+    /**
+     * Only throw the error if encountered at the top level of the expression.
+     *
+     * This will cause expressions like `builtins.throw "uh oh!"` to throw
+     * errors, but will print attribute sets and other nested structures
+     * containing values that error (like `nixpkgs`) normally.
+     */
+    ThrowTopLevel,
+};
+
 /**
  * Options for printing Nix values.
  */
@@ -68,6 +91,11 @@ struct PrintOptions
      */
     size_t prettyIndent = 0;
 
+    /**
+     * How to handle errors encountered while printing values.
+     */
+    ErrorPrintBehavior errors = ErrorPrintBehavior::Print;
+
     /**
      * True if pretty-printing is enabled.
      */
@@ -86,7 +114,7 @@ static PrintOptions errorPrintOptions = PrintOptions {
     .maxDepth = 10,
     .maxAttrs = 10,
     .maxListItems = 10,
-    .maxStringLength = 1024
+    .maxStringLength = 1024,
 };
 
 }

src/libexpr/print.cc

@@ -1,5 +1,6 @@
 #include <limits>
 #include <unordered_set>
+#include <sstream>
 
 #include "print.hh"
 #include "ansicolor.hh"
@@ -162,8 +163,8 @@ private:
     EvalState & state;
     PrintOptions options;
     std::optional<ValuesSeen> seen;
-    size_t attrsPrinted = 0;
-    size_t listItemsPrinted = 0;
+    size_t totalAttrsPrinted = 0;
+    size_t totalListItemsPrinted = 0;
     std::string indent;
 
     void increaseIndent()
@@ -223,7 +224,7 @@ private:
     {
         if (options.ansiColors)
             output << ANSI_CYAN;
-        output << v.integer;
+        output << v.integer();
         if (options.ansiColors)
             output << ANSI_NORMAL;
     }
@@ -232,7 +233,7 @@ private:
     {
         if (options.ansiColors)
             output << ANSI_CYAN;
-        output << v.fpoint;
+        output << v.fpoint();
         if (options.ansiColors)
             output << ANSI_NORMAL;
     }
@@ -241,7 +242,7 @@ private:
     {
         if (options.ansiColors)
             output << ANSI_CYAN;
-        printLiteralBool(output, v.boolean);
+        printLiteralBool(output, v.boolean());
         if (options.ansiColors)
             output << ANSI_NORMAL;
     }
@@ -271,27 +272,36 @@ private:
 
     void printDerivation(Value & v)
     {
-        try {
-            Bindings::iterator i = v.attrs->find(state.sDrvPath);
+        std::optional<StorePath> storePath;
+        if (auto i = v.attrs()->get(state.sDrvPath)) {
             NixStringContext context;
-            std::string storePath;
-            if (i != v.attrs->end())
-                storePath = state.store->printStorePath(state.coerceToStorePath(i->pos, *i->value, context, "while evaluating the drvPath of a derivation"));
-
-            if (options.ansiColors)
-                output << ANSI_GREEN;
-            output << "«derivation";
-            if (!storePath.empty()) {
-                output << " " << storePath;
-            }
-            output << "»";
-            if (options.ansiColors)
-                output << ANSI_NORMAL;
-        } catch (Error & e) {
-            printError_(e);
+            storePath = state.coerceToStorePath(i->pos, *i->value, context, "while evaluating the drvPath of a derivation");
         }
+
+        /* This unfortunately breaks printing nested values because of
+           how the pretty printer is used (when pretting printing and warning
+           to same terminal / std stream). */
+#if 0
+        if (storePath && !storePath->isDerivation())
+            warn(
+                "drvPath attribute '%s' is not a valid store path to a derivation, this value not work properly",
+                state.store->printStorePath(*storePath));
+#endif
+
+        if (options.ansiColors)
+            output << ANSI_GREEN;
+        output << "«derivation";
+        if (storePath) {
+            output << " " << state.store->printStorePath(*storePath);
+        }
+        output << "»";
+        if (options.ansiColors)
+            output << ANSI_NORMAL;
     }
 
+    /**
+     * @note This may force items.
+     */
     bool shouldPrettyPrintAttrs(AttrVec & v)
     {
         if (!options.shouldPrettyPrint() || v.empty()) {
@@ -308,6 +318,9 @@ private:
             return true;
         }
 
+        // It is ok to force the item(s) here, because they will be printed anyway.
+        state.forceValue(*item, item->determinePos(noPos));
+
         // Pretty-print single-item attrsets only if they contain nested
         // structures.
         auto itemType = item->type();
@@ -316,7 +329,7 @@ private:
 
     void printAttrs(Value & v, size_t depth)
     {
-        if (seen && !seen->insert(v.attrs).second) {
+        if (seen && !seen->insert(v.attrs()).second) {
             printRepeated();
             return;
         }
@@ -328,7 +341,7 @@ private:
             output << "{";
 
             AttrVec sorted;
-            for (auto & i : *v.attrs)
+            for (auto & i : *v.attrs())
                 sorted.emplace_back(std::pair(state.symbols[i.name], i.value));
 
             if (options.maxAttrs == std::numeric_limits<size_t>::max())
@@ -338,11 +351,13 @@ private:
 
             auto prettyPrint = shouldPrettyPrintAttrs(sorted);
 
+            size_t currentAttrsPrinted = 0;
+
             for (auto & i : sorted) {
                 printSpace(prettyPrint);
 
-                if (attrsPrinted >= options.maxAttrs) {
-                    printElided(sorted.size() - attrsPrinted, "attribute", "attributes");
+                if (totalAttrsPrinted >= options.maxAttrs) {
+                    printElided(sorted.size() - currentAttrsPrinted, "attribute", "attributes");
                     break;
                 }
 
@@ -350,7 +365,8 @@ private:
                 output << " = ";
                 print(*i.second, depth + 1);
                 output << ";";
-                attrsPrinted++;
+                totalAttrsPrinted++;
+                currentAttrsPrinted++;
             }
 
             decreaseIndent();
@@ -361,6 +377,9 @@ private:
         }
     }
 
+    /**
+     * @note This may force items.
+     */
     bool shouldPrettyPrintList(std::span<Value * const> list)
     {
         if (!options.shouldPrettyPrint() || list.empty()) {
@@ -377,6 +396,9 @@ private:
             return true;
         }
 
+        // It is ok to force the item(s) here, because they will be printed anyway.
+        state.forceValue(*item, item->determinePos(noPos));
+
         // Pretty-print single-item lists only if they contain nested
         // structures.
         auto itemType = item->type();
@@ -395,11 +417,14 @@ private:
             output << "[";
             auto listItems = v.listItems();
             auto prettyPrint = shouldPrettyPrintList(listItems);
+
+            size_t currentListItemsPrinted = 0;
+
             for (auto elem : listItems) {
                 printSpace(prettyPrint);
 
-                if (listItemsPrinted >= options.maxListItems) {
-                    printElided(listItems.size() - listItemsPrinted, "item", "items");
+                if (totalListItemsPrinted >= options.maxListItems) {
+                    printElided(listItems.size() - currentListItemsPrinted, "item", "items");
                     break;
                 }
 
@@ -408,7 +433,8 @@ private:
                 } else {
                     printNullptr();
                 }
-                listItemsPrinted++;
+                totalListItemsPrinted++;
+                currentListItemsPrinted++;
             }
 
             decreaseIndent();
@@ -427,18 +453,18 @@ private:
 
         if (v.isLambda()) {
             output << "lambda";
-            if (v.lambda.fun) {
-                if (v.lambda.fun->name) {
-                    output << " " << state.symbols[v.lambda.fun->name];
+            if (v.payload.lambda.fun) {
+                if (v.payload.lambda.fun->name) {
+                    output << " " << state.symbols[v.payload.lambda.fun->name];
                 }
 
                 std::ostringstream s;
-                s << state.positions[v.lambda.fun->pos];
+                s << state.positions[v.payload.lambda.fun->pos];
                 output << " @ " << filterANSIEscapes(s.str());
             }
         } else if (v.isPrimOp()) {
-            if (v.primOp)
-                output << *v.primOp;
+            if (v.primOp())
+                output << *v.primOp();
             else
                 output << "primop";
         } else if (v.isPrimOpApp()) {
@@ -449,7 +475,7 @@ private:
             else
                 output << "primop";
         } else {
-            abort();
+            unreachable();
         }
 
         output << "»";
@@ -478,13 +504,13 @@ private:
             if (options.ansiColors)
                     output << ANSI_NORMAL;
         } else {
-            abort();
+            unreachable();
         }
     }
 
     void printExternal(Value & v)
     {
-        v.external->print(output);
+        v.external()->print(output);
     }
 
     void printUnknown()
@@ -510,64 +536,68 @@ private:
         output.flush();
         checkInterrupt();
 
-        if (options.force) {
-            try {
+        try {
+            if (options.force) {
                 state.forceValue(v, v.determinePos(noPos));
-            } catch (Error & e) {
-                printError_(e);
-                return;
             }
-        }
 
-        switch (v.type()) {
+            switch (v.type()) {
 
-        case nInt:
-            printInt(v);
-            break;
+            case nInt:
+                printInt(v);
+                break;
 
-        case nFloat:
-            printFloat(v);
-            break;
+            case nFloat:
+                printFloat(v);
+                break;
 
-        case nBool:
-            printBool(v);
-            break;
+            case nBool:
+                printBool(v);
+                break;
 
-        case nString:
-            printString(v);
-            break;
+            case nString:
+                printString(v);
+                break;
 
-        case nPath:
-            printPath(v);
-            break;
+            case nPath:
+                printPath(v);
+                break;
 
-        case nNull:
-            printNull();
-            break;
+            case nNull:
+                printNull();
+                break;
 
-        case nAttrs:
-            printAttrs(v, depth);
-            break;
+            case nAttrs:
+                printAttrs(v, depth);
+                break;
 
-        case nList:
-            printList(v, depth);
-            break;
+            case nList:
+                printList(v, depth);
+                break;
 
-        case nFunction:
-            printFunction(v);
-            break;
+            case nFunction:
+                printFunction(v);
+                break;
 
-        case nThunk:
-            printThunk(v);
-            break;
+            case nThunk:
+                printThunk(v);
+                break;
 
-        case nExternal:
-            printExternal(v);
-            break;
+            case nExternal:
+                printExternal(v);
+                break;
 
-        default:
-            printUnknown();
-            break;
+            default:
+                printUnknown();
+                break;
+            }
+        } catch (Error & e) {
+            if (options.errors == ErrorPrintBehavior::Throw
+                || (options.errors == ErrorPrintBehavior::ThrowTopLevel
+                    && depth == 0)) {
+                throw;
+            }
+            printError_(e);
         }
     }
 
@@ -577,8 +607,8 @@ public:
 
     void print(Value & v)
     {
-        attrsPrinted = 0;
-        listItemsPrinted = 0;
+        totalAttrsPrinted = 0;
+        totalListItemsPrinted = 0;
         indent.clear();
 
         if (options.trackRepeated) {

src/libexpr/search-path.cc

@@ -2,7 +2,7 @@
 
 namespace nix {
 
-std::optional<std::string_view> SearchPath::Prefix::suffixIfPotentialMatch(
+std::optional<std::string_view> LookupPath::Prefix::suffixIfPotentialMatch(
     std::string_view path) const
 {
     auto n = s.size();
@@ -27,11 +27,11 @@ std::optional<std::string_view> SearchPath::Prefix::suffixIfPotentialMatch(
 }
 
 
-SearchPath::Elem SearchPath::Elem::parse(std::string_view rawElem)
+LookupPath::Elem LookupPath::Elem::parse(std::string_view rawElem)
 {
     size_t pos = rawElem.find('=');
 
-    return SearchPath::Elem {
+    return LookupPath::Elem {
         .prefix = Prefix {
             .s = pos == std::string::npos
                 ? std::string { "" }
@@ -44,11 +44,11 @@ SearchPath::Elem SearchPath::Elem::parse(std::string_view rawElem)
 }
 
 
-SearchPath parseSearchPath(const Strings & rawElems)
+LookupPath LookupPath::parse(const Strings & rawElems)
 {
-    SearchPath res;
+    LookupPath res;
     for (auto & rawElem : rawElems)
-        res.elements.emplace_back(SearchPath::Elem::parse(rawElem));
+        res.elements.emplace_back(LookupPath::Elem::parse(rawElem));
     return res;
 }
 

src/libexpr/search-path.hh

@@ -12,10 +12,10 @@ namespace nix {
  * A "search path" is a list of ways look for something, used with
  * `builtins.findFile` and `< >` lookup expressions.
  */
-struct SearchPath
+struct LookupPath
 {
     /**
-     * A single element of a `SearchPath`.
+     * A single element of a `LookupPath`.
      *
      * Each element is tried in succession when looking up a path. The first
      * element to completely match wins.
@@ -23,16 +23,16 @@ struct SearchPath
     struct Elem;
 
     /**
-     * The first part of a `SearchPath::Elem` pair.
+     * The first part of a `LookupPath::Elem` pair.
      *
      * Called a "prefix" because it takes the form of a prefix of a file
      * path (first `n` path components). When looking up a path, to use
-     * a `SearchPath::Elem`, its `Prefix` must match the path.
+     * a `LookupPath::Elem`, its `Prefix` must match the path.
      */
     struct Prefix;
 
     /**
-     * The second part of a `SearchPath::Elem` pair.
+     * The second part of a `LookupPath::Elem` pair.
      *
      * It is either a path or a URL (with certain restrictions / extra
      * structure).
@@ -40,7 +40,7 @@ struct SearchPath
      * If the prefix of the path we are looking up matches, we then
      * check if the rest of the path points to something that exists
      * within the directory denoted by this. If so, the
-     * `SearchPath::Elem` as a whole matches, and that *something* being
+     * `LookupPath::Elem` as a whole matches, and that *something* being
      * pointed to by the rest of the path we are looking up is the
      * result.
      */
@@ -51,24 +51,24 @@ struct SearchPath
      * when looking up. (The actual lookup entry point is in `EvalState`
      * not in this class.)
      */
-    std::list<SearchPath::Elem> elements;
+    std::list<LookupPath::Elem> elements;
 
     /**
-     * Parse a string into a `SearchPath`
+     * Parse a string into a `LookupPath`
      */
-    static SearchPath parse(const Strings & rawElems);
+    static LookupPath parse(const Strings & rawElems);
 };
 
-struct SearchPath::Prefix
+struct LookupPath::Prefix
 {
     /**
      * Underlying string
      *
-     * @todo Should we normalize this when constructing a `SearchPath::Prefix`?
+     * @todo Should we normalize this when constructing a `LookupPath::Prefix`?
      */
     std::string s;
 
-    GENERATE_CMP(SearchPath::Prefix, me->s);
+    GENERATE_CMP(LookupPath::Prefix, me->s);
 
     /**
      * If the path possibly matches this search path element, return the
@@ -79,7 +79,7 @@ struct SearchPath::Prefix
     std::optional<std::string_view> suffixIfPotentialMatch(std::string_view path) const;
 };
 
-struct SearchPath::Path
+struct LookupPath::Path
 {
     /**
      * The location of a search path item, as a path or URL.
@@ -88,21 +88,21 @@ struct SearchPath::Path
      */
     std::string s;
 
-    GENERATE_CMP(SearchPath::Path, me->s);
+    GENERATE_CMP(LookupPath::Path, me->s);
 };
 
-struct SearchPath::Elem
+struct LookupPath::Elem
 {
 
     Prefix prefix;
     Path path;
 
-    GENERATE_CMP(SearchPath::Elem, me->prefix, me->path);
+    GENERATE_CMP(LookupPath::Elem, me->prefix, me->path);
 
     /**
-     * Parse a string into a `SearchPath::Elem`
+     * Parse a string into a `LookupPath::Elem`
      */
-    static SearchPath::Elem parse(std::string_view rawElem);
+    static LookupPath::Elem parse(std::string_view rawElem);
 };
 
 }

src/libexpr/symbol-table.hh

@@ -7,6 +7,7 @@
 
 #include "types.hh"
 #include "chunked-vector.hh"
+#include "error.hh"
 
 namespace nix {
 
@@ -30,9 +31,9 @@ public:
         return *s == s2;
     }
 
-    operator const std::string & () const
+    const char * c_str() const
     {
-        return *s;
+        return s->c_str();
     }
 
     operator const std::string_view () const
@@ -41,6 +42,11 @@ public:
     }
 
     friend std::ostream & operator <<(std::ostream & os, const SymbolStr & symbol);
+
+    bool empty() const
+    {
+        return s->empty();
+    }
 };
 
 /**
@@ -62,9 +68,10 @@ public:
 
     explicit operator bool() const { return id > 0; }
 
-    bool operator<(const Symbol other) const { return id < other.id; }
+    auto operator<=>(const Symbol other) const { return id <=> other.id; }
     bool operator==(const Symbol other) const { return id == other.id; }
-    bool operator!=(const Symbol other) const { return id != other.id; }
+
+    friend class std::hash<Symbol>;
 };
 
 /**
@@ -109,7 +116,7 @@ public:
     SymbolStr operator[](Symbol s) const
     {
         if (s.id == 0 || s.id > store.size())
-            abort();
+            unreachable();
         return SymbolStr(store[s.id - 1]);
     }
 
@@ -128,3 +135,12 @@ public:
 };
 
 }
+
+template<>
+struct std::hash<nix::Symbol>
+{
+    std::size_t operator()(const nix::Symbol & s) const noexcept
+    {
+        return std::hash<decltype(s.id)>{}(s.id);
+    }
+};