Merge remote-tracking branch 'origin/master' into fsync-store-paths

src/build-remote/build-remote.cc

@@ -11,11 +11,13 @@
 
 #include "machines.hh"
 #include "shared.hh"
+#include "plugin.hh"
 #include "pathlocks.hh"
 #include "globals.hh"
 #include "serialise.hh"
 #include "build-result.hh"
 #include "store-api.hh"
+#include "strings.hh"
 #include "derivations.hh"
 #include "local-store.hh"
 #include "legacy.hh"
@@ -37,7 +39,7 @@ static std::string currentLoad;
 
 static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot)
 {
-    return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true);
+    return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri.render()), slot), true);
 }
 
 static bool allSupportedLocally(Store & store, const std::set<std::string>& requiredFeatures) {
@@ -72,6 +74,7 @@ static int main_build_remote(int argc, char * * argv)
             settings.set(name, value);
         }
 
+        auto maxBuildJobs = settings.maxBuildJobs;
         settings.maxBuildJobs.set("1"); // hack to make tests with local?root= work
 
         initPlugins();
@@ -112,10 +115,14 @@ static int main_build_remote(int argc, char * * argv)
             drvPath = store->parseStorePath(readString(source));
             auto requiredFeatures = readStrings<std::set<std::string>>(source);
 
-            auto canBuildLocally = amWilling
+            /* It would be possible to build locally after some builds clear out,
+               so don't show the warning now: */
+            bool couldBuildLocally = maxBuildJobs > 0
                  &&  (  neededSystem == settings.thisSystem
                      || settings.extraPlatforms.get().count(neededSystem) > 0)
                  &&  allSupportedLocally(*store, requiredFeatures);
+            /* It's possible to build this locally right now: */
+            bool canBuildLocally = amWilling && couldBuildLocally;
 
             /* Error ignored here, will be caught later */
             mkdir(currentLoad.c_str(), 0777);
@@ -130,13 +137,10 @@ static int main_build_remote(int argc, char * * argv)
                 Machine * bestMachine = nullptr;
                 uint64_t bestLoad = 0;
                 for (auto & m : machines) {
-                    debug("considering building on remote machine '%s'", m.storeUri);
+                    debug("considering building on remote machine '%s'", m.storeUri.render());
 
-                    if (m.enabled
-                        && (neededSystem == "builtin"
-                            || std::find(m.systemTypes.begin(),
-                                m.systemTypes.end(),
-                                neededSystem) != m.systemTypes.end()) &&
+                    if (m.enabled &&
+                        m.systemSupported(neededSystem) &&
                         m.allSupported(requiredFeatures) &&
                         m.mandatoryMet(requiredFeatures))
                     {
@@ -200,7 +204,7 @@ static int main_build_remote(int argc, char * * argv)
                         else
                             drvstr = "<unknown>";
 
-                        auto error = hintformat(errorText);
+                        auto error = HintFmt::fromFormatString(errorText);
                         error
                             % drvstr
                             % neededSystem
@@ -209,12 +213,12 @@ static int main_build_remote(int argc, char * * argv)
 
                         for (auto & m : machines)
                             error
-                                % concatStringsSep<std::vector<std::string>>(", ", m.systemTypes)
+                                % concatStringsSep<StringSet>(", ", m.systemTypes)
                                 % m.maxJobs
                                 % concatStringsSep<StringSet>(", ", m.supportedFeatures)
                                 % concatStringsSep<StringSet>(", ", m.mandatoryFeatures);
 
-                        printMsg(canBuildLocally ? lvlChatty : lvlWarn, error);
+                        printMsg(couldBuildLocally ? lvlChatty : lvlWarn, error.str());
 
                         std::cerr << "# decline\n";
                     }
@@ -230,17 +234,16 @@ static int main_build_remote(int argc, char * * argv)
                 lock = -1;
 
                 try {
+                    storeUri = bestMachine->storeUri.render();
 
-                    Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri));
+                    Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", storeUri));
 
                     sshStore = bestMachine->openStore();
                     sshStore->connect();
-                    storeUri = bestMachine->storeUri;
-
                 } catch (std::exception & e) {
                     auto msg = chomp(drainFD(5, false));
                     printError("cannot build on '%s': %s%s",
-                        bestMachine->storeUri, e.what(),
+                        storeUri, e.what(),
                         msg.empty() ? "" : ": " + msg);
                     bestMachine->enabled = false;
                     continue;
@@ -253,12 +256,27 @@ static int main_build_remote(int argc, char * * argv)
 connected:
         close(5);
 
+        assert(sshStore);
+
         std::cerr << "# accept\n" << storeUri << "\n";
 
         auto inputs = readStrings<PathSet>(source);
         auto wantedOutputs = readStrings<StringSet>(source);
 
-        AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true);
+        AutoCloseFD uploadLock;
+        {
+            auto setUpdateLock = [&](auto && fileName){
+                uploadLock = openLockFile(currentLoad + "/" + escapeUri(fileName) + ".upload-lock", true);
+            };
+            try {
+                setUpdateLock(storeUri);
+            } catch (SysError & e) {
+                if (e.errNo != ENAMETOOLONG) throw;
+                // Try again hashing the store URL so we have a shorter path
+                auto h = hashString(HashAlgorithm::MD5, storeUri);
+                setUpdateLock(h.to_string(HashFormat::Base64, false));
+            }
+        }
 
         {
             Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri));
@@ -281,33 +299,66 @@ connected:
         uploadLock = -1;
 
         auto drv = store->readDerivation(*drvPath);
+
+        std::optional<BuildResult> optResult;
+
+        // If we don't know whether we are trusted (e.g. `ssh://`
+        // stores), we assume we are. This is necessary for backwards
+        // compat.
+        bool trustedOrLegacy = ({
+            std::optional trusted = sshStore->isTrustedClient();
+            !trusted || *trusted;
+        });
+
+        // See the very large comment in `case WorkerProto::Op::BuildDerivation:` in
+        // `src/libstore/daemon.cc` that explains the trust model here.
+        //
+        // This condition mirrors that: that code enforces the "rules" outlined there;
+        // we do the best we can given those "rules".
+        if (trustedOrLegacy || drv.type().isCA())  {
+            // Hijack the inputs paths of the derivation to include all
+            // the paths that come from the `inputDrvs` set. We don’t do
+            // that for the derivations whose `inputDrvs` is empty
+            // because:
+            //
+            // 1. It’s not needed
+            //
+            // 2. Changing the `inputSrcs` set changes the associated
+            //    output ids, which break CA derivations
+            if (!drv.inputDrvs.map.empty())
+                drv.inputSrcs = store->parseStorePathSet(inputs);
+            optResult = sshStore->buildDerivation(*drvPath, (const BasicDerivation &) drv);
+            auto & result = *optResult;
+            if (!result.success())
+                throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg);
+        } else {
+            copyClosure(*store, *sshStore, StorePathSet {*drvPath}, NoRepair, NoCheckSigs, substitute);
+            auto res = sshStore->buildPathsWithResults({
+                DerivedPath::Built {
+                    .drvPath = makeConstantStorePathRef(*drvPath),
+                    .outputs = OutputsSpec::All {},
+                }
+            });
+            // One path to build should produce exactly one build result
+            assert(res.size() == 1);
+            optResult = std::move(res[0]);
+        }
+
+
         auto outputHashes = staticOutputHashes(*store, drv);
-
-        // Hijack the inputs paths of the derivation to include all the paths
-        // that come from the `inputDrvs` set.
-        // We don’t do that for the derivations whose `inputDrvs` is empty
-        // because
-        // 1. It’s not needed
-        // 2. Changing the `inputSrcs` set changes the associated output ids,
-        //  which break CA derivations
-        if (!drv.inputDrvs.empty())
-            drv.inputSrcs = store->parseStorePathSet(inputs);
-
-        auto result = sshStore->buildDerivation(*drvPath, drv);
-
-        if (!result.success())
-            throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg);
-
         std::set<Realisation> missingRealisations;
         StorePathSet missingPaths;
-        if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) && !drv.type().hasKnownOutputPaths()) {
+        if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && !drv.type().hasKnownOutputPaths()) {
             for (auto & outputName : wantedOutputs) {
                 auto thisOutputHash = outputHashes.at(outputName);
                 auto thisOutputId = DrvOutput{ thisOutputHash, outputName };
                 if (!store->queryRealisation(thisOutputId)) {
                     debug("missing output %s", outputName);
-                    assert(result.builtOutputs.count(thisOutputId));
-                    auto newRealisation = result.builtOutputs.at(thisOutputId);
+                    assert(optResult);
+                    auto & result = *optResult;
+                    auto i = result.builtOutputs.find(outputName);
+                    assert(i != result.builtOutputs.end());
+                    auto & newRealisation = i->second;
                     missingRealisations.insert(newRealisation);
                     missingPaths.insert(newRealisation.outPath);
                 }
@@ -332,7 +383,7 @@ connected:
         for (auto & realisation : missingRealisations) {
             // Should hold, because if the feature isn't enabled the set
             // of missing realisations should be empty
-            settings.requireExperimentalFeature(Xp::CaDerivations);
+            experimentalFeatureSettings.require(Xp::CaDerivations);
             store->registerDrvOutput(realisation);
         }
 

src/external-api-docs/.gitignore

@@ -0,0 +1,3 @@
+/doxygen.cfg
+/html
+/latex

src/external-api-docs/.version

@@ -0,0 +1 @@
+../../.version

src/external-api-docs/README.md

@@ -0,0 +1,121 @@
+# Getting started
+
+> **Warning** These bindings are **experimental**, which means they can change
+> at any time or be removed outright; nevertheless the plan is to provide a
+> stable external C API to the Nix language and the Nix store.
+
+The language library allows evaluating Nix expressions and interacting with Nix
+language values. The Nix store API is still rudimentary, and only allows
+initialising and connecting to a store for the Nix language evaluator to
+interact with.
+
+Currently there are two ways to interface with the Nix language evaluator
+programmatically:
+
+1. Embedding the evaluator
+2. Writing language plug-ins
+
+Embedding means you link the Nix C libraries in your program and use them from
+there. Adding a plug-in means you make a library that gets loaded by the Nix
+language evaluator, specified through a configuration option.
+
+Many of the components and mechanisms involved are not yet documented, therefore
+please refer to the [Nix source code](https://github.com/NixOS/nix/) for
+details. Additions to in-code documentation and the reference manual are highly
+appreciated.
+
+The following examples, for simplicity, don't include error handling. See the
+[Handling errors](@ref errors) section for more information.
+
+# Embedding the Nix Evaluator{#nix_evaluator_example}
+
+In this example we programmatically start the Nix language evaluator with a
+dummy store (that has no store paths and cannot be written to), and evaluate the
+Nix expression `builtins.nixVersion`.
+
+**main.c:**
+
+```C
+#include <nix_api_util.h>
+#include <nix_api_expr.h>
+#include <nix_api_value.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+// NOTE: This example lacks all error handling. Production code must check for
+// errors, as some return values will be undefined.
+
+void my_get_string_cb(const char * start, unsigned int n, void * user_data)
+{
+    *((char **) user_data) = strdup(start);
+}
+
+int main()
+{
+    nix_libexpr_init(NULL);
+
+    Store * store = nix_store_open(NULL, "dummy://", NULL);
+    EvalState * state = nix_state_create(NULL, NULL, store); // empty search path (NIX_PATH)
+    Value * value = nix_alloc_value(NULL, state);
+
+    nix_expr_eval_from_string(NULL, state, "builtins.nixVersion", ".", value);
+    nix_value_force(NULL, state, value);
+
+    char * version;
+    nix_get_string(NULL, value, my_get_string_cb, &version);
+    printf("Nix version: %s\n", version);
+
+    free(version);
+    nix_gc_decref(NULL, value);
+    nix_state_free(state);
+    nix_store_free(store);
+    return 0;
+}
+```
+
+**Usage:**
+
+```ShellSession
+$ gcc main.c $(pkg-config nix-expr-c --libs --cflags) -o main
+$ ./main
+Nix version: 2.17
+```
+
+# Writing a Nix language plug-in
+
+In this example we add a custom primitive operation (_primop_) to `builtins`. It
+will increment the argument if it is an integer and throw an error otherwise.
+
+**plugin.c:**
+
+```C
+#include <nix_api_util.h>
+#include <nix_api_expr.h>
+#include <nix_api_value.h>
+
+void increment(void* user_data, nix_c_context* ctx, EvalState* state, Value** args, Value* v) {
+    nix_value_force(NULL, state, args[0]);
+    if (nix_get_type(NULL, args[0]) == NIX_TYPE_INT) {
+      nix_init_int(NULL, v, nix_get_int(NULL, args[0]) + 1);
+    } else {
+      nix_set_err_msg(ctx, NIX_ERR_UNKNOWN, "First argument should be an integer.");
+    }
+}
+
+void nix_plugin_entry() {
+  const char* args[] = {"n", NULL};
+  PrimOp *p = nix_alloc_primop(NULL, increment, 1, "increment", args, "Example custom built-in function: increments an integer", NULL);
+  nix_register_primop(NULL, p);
+  nix_gc_decref(NULL, p);
+}
+```
+
+**Usage:**
+
+```ShellSession
+$ gcc plugin.c $(pkg-config nix-expr-c --libs --cflags) -shared -o plugin.so
+$ nix --plugin-files ./plugin.so repl
+nix-repl> builtins.increment 1
+2
+```

src/external-api-docs/doxygen.cfg.in

@@ -0,0 +1,58 @@
+# Doxyfile 1.9.5
+
+# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by
+# double-quotes, unless you are using Doxywizard) that should identify the
+# project for which the documentation is generated. This name is used in the
+# title of most generated pages and in a few other places.
+# The default value is: My Project.
+
+PROJECT_NAME           = "Nix"
+
+# The PROJECT_NUMBER tag can be used to enter a project or revision number. This
+# could be handy for archiving the generated documentation or if some version
+# control system is used.
+
+PROJECT_NUMBER         = @PROJECT_NUMBER@
+
+OUTPUT_DIRECTORY       = @OUTPUT_DIRECTORY@
+
+# Using the PROJECT_BRIEF tag one can provide an optional one line description
+# for a project that appears at the top of each page and should give viewer a
+# quick idea about the purpose of the project. Keep the description short.
+
+PROJECT_BRIEF          = "Nix, the purely functional package manager: C API (experimental)"
+
+# If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output.
+# The default value is: YES.
+
+GENERATE_LATEX         = NO
+
+# The INPUT tag is used to specify the files and/or directories that contain
+# documented source files. You may enter file names like myfile.cpp or
+# directories like /usr/src/myproject. Separate the files or directories with
+# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING
+# Note: If this tag is empty the current directory is searched.
+
+# FIXME Make this list more maintainable somehow. We could maybe generate this
+# in the Makefile, but we would need to change how `.in` files are preprocessed
+# so they can expand variables despite configure variables.
+
+INPUT                  = \
+  @src@/src/libutil-c \
+  @src@/src/libexpr-c \
+  @src@/src/libstore-c \
+  @src@/doc/external-api/README.md
+
+FILE_PATTERNS = nix_api_*.h *.md
+
+# The INCLUDE_PATH tag can be used to specify one or more directories that
+# contain include files that are not input files but should be processed by the
+# preprocessor. Note that the INCLUDE_PATH is not recursive, so the setting of
+# RECURSIVE has no effect here.
+# This tag requires that the tag SEARCH_INCLUDES is set to YES.
+
+EXCLUDE_PATTERNS = *_internal.h
+GENERATE_TREEVIEW = YES
+OPTIMIZE_OUTPUT_FOR_C  = YES
+
+USE_MDFILE_AS_MAINPAGE = doc/external-api/README.md

src/external-api-docs/meson.build

@@ -0,0 +1,31 @@
+project('nix-external-api-docs',
+  version : files('.version'),
+  meson_version : '>= 1.1',
+  license : 'LGPL-2.1-or-later',
+)
+
+fs = import('fs')
+
+doxygen_cfg = configure_file(
+  input : 'doxygen.cfg.in',
+  output : 'doxygen.cfg',
+  configuration : {
+    'PROJECT_NUMBER': meson.project_version(),
+    'OUTPUT_DIRECTORY' : meson.current_build_dir(),
+    'src' : fs.parent(fs.parent(meson.project_source_root())),
+  },
+)
+
+doxygen = find_program('doxygen', native : true, required : true)
+
+custom_target(
+  'external-api-docs',
+  command : [ doxygen , doxygen_cfg ],
+  input : [
+    doxygen_cfg,
+  ],
+  output : 'html',
+  install : true,
+  install_dir : get_option('datadir') / 'doc/nix/external-api',
+  build_always_stale : true,
+)

src/external-api-docs/package.nix

@@ -0,0 +1,59 @@
+{ lib
+, mkMesonDerivation
+
+, meson
+, ninja
+, doxygen
+
+# Configuration Options
+
+, version
+}:
+
+let
+  inherit (lib) fileset;
+in
+
+mkMesonDerivation (finalAttrs: {
+  pname = "nix-external-api-docs";
+  inherit version;
+
+  workDir = ./.;
+  fileset =
+    let
+      cpp = fileset.fileFilter (file: file.hasExt "cc" || file.hasExt "h");
+    in
+    fileset.unions [
+      ./.version
+      ../../.version
+      ./meson.build
+      ./doxygen.cfg.in
+      ./README.md
+      # Source is not compiled, but still must be available for Doxygen
+      # to gather comments.
+      (cpp ../libexpr-c)
+      (cpp ../libstore-c)
+      (cpp ../libutil-c)
+    ];
+
+  nativeBuildInputs = [
+    meson
+    ninja
+    doxygen
+  ];
+
+  preConfigure =
+    ''
+      chmod u+w ./.version
+      echo ${finalAttrs.version} > ./.version
+    '';
+
+  postInstall = ''
+    mkdir -p ''${!outputDoc}/nix-support
+    echo "doc external-api-docs $out/share/doc/nix/external-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products
+  '';
+
+  meta = {
+    platforms = lib.platforms.all;
+  };
+})

src/internal-api-docs/.gitignore

@@ -0,0 +1,3 @@
+/doxygen.cfg
+/html
+/latex

src/internal-api-docs/.version

@@ -0,0 +1 @@
+../../.version

src/internal-api-docs/doxygen.cfg.in

@@ -0,0 +1,99 @@
+# Doxyfile 1.9.5
+
+# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by
+# double-quotes, unless you are using Doxywizard) that should identify the
+# project for which the documentation is generated. This name is used in the
+# title of most generated pages and in a few other places.
+# The default value is: My Project.
+
+PROJECT_NAME           = "Nix"
+
+# The PROJECT_NUMBER tag can be used to enter a project or revision number. This
+# could be handy for archiving the generated documentation or if some version
+# control system is used.
+
+PROJECT_NUMBER         = @PROJECT_NUMBER@
+
+OUTPUT_DIRECTORY       = @OUTPUT_DIRECTORY@
+
+# Using the PROJECT_BRIEF tag one can provide an optional one line description
+# for a project that appears at the top of each page and should give viewer a
+# quick idea about the purpose of the project. Keep the description short.
+
+PROJECT_BRIEF          = "Nix, the purely functional package manager; unstable internal interfaces"
+
+# If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output.
+# The default value is: YES.
+
+GENERATE_LATEX         = NO
+
+# The INPUT tag is used to specify the files and/or directories that contain
+# documented source files. You may enter file names like myfile.cpp or
+# directories like /usr/src/myproject. Separate the files or directories with
+# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING
+# Note: If this tag is empty the current directory is searched.
+
+# FIXME Make this list more maintainable somehow. We could maybe generate this
+# in the Makefile, but we would need to change how `.in` files are preprocessed
+# so they can expand variables despite configure variables.
+
+INPUT                  = \
+  @src@/libcmd \
+  @src@/libexpr \
+  @src@/libexpr/flake \
+  @src@/nix-expr-tests \
+  @src@/nix-expr-tests/value \
+  @src@/nix-expr-test-support/test \
+  @src@/nix-expr-test-support/test/value \
+  @src@/libexpr/value \
+  @src@/libfetchers \
+  @src@/libmain \
+  @src@/libstore \
+  @src@/libstore/build \
+  @src@/libstore/builtins \
+  @src@/nix-store-tests \
+  @src@/nix-store-test-support/test \
+  @src@/libutil \
+  @src@/nix-util-tests \
+  @src@/nix-util-test-support/test \
+  @src@/nix \
+  @src@/nix-env \
+  @src@/nix-store
+
+# If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names
+# in the source code. If set to NO, only conditional compilation will be
+# performed. Macro expansion can be done in a controlled way by setting
+# EXPAND_ONLY_PREDEF to YES.
+# The default value is: NO.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+MACRO_EXPANSION        = YES
+
+# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then
+# the macro expansion is limited to the macros specified with the PREDEFINED and
+# EXPAND_AS_DEFINED tags.
+# The default value is: NO.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+EXPAND_ONLY_PREDEF     = YES
+
+# The INCLUDE_PATH tag can be used to specify one or more directories that
+# contain include files that are not input files but should be processed by the
+# preprocessor. Note that the INCLUDE_PATH is not recursive, so the setting of
+# RECURSIVE has no effect here.
+# This tag requires that the tag SEARCH_INCLUDES is set to YES.
+
+INCLUDE_PATH           =
+
+# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this
+# tag can be used to specify a list of macro names that should be expanded. The
+# macro definition that is found in the sources will be used. Use the PREDEFINED
+# tag if you want to use a different macro definition that overrules the
+# definition found in the source code.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+EXPAND_AS_DEFINED      = \
+  DECLARE_COMMON_SERIALISER \
+  DECLARE_WORKER_SERIALISER \
+  DECLARE_SERVE_SERIALISER \
+  LENGTH_PREFIXED_PROTO_HELPER

src/internal-api-docs/meson.build

@@ -0,0 +1,31 @@
+project('nix-internal-api-docs',
+  version : files('.version'),
+  meson_version : '>= 1.1',
+  license : 'LGPL-2.1-or-later',
+)
+
+fs = import('fs')
+
+doxygen_cfg = configure_file(
+  input : 'doxygen.cfg.in',
+  output : 'doxygen.cfg',
+  configuration : {
+    'PROJECT_NUMBER': meson.project_version(),
+    'OUTPUT_DIRECTORY' : meson.current_build_dir(),
+    'src' : fs.parent(fs.parent(meson.project_source_root())) / 'src',
+  },
+)
+
+doxygen = find_program('doxygen', native : true, required : true)
+
+custom_target(
+  'internal-api-docs',
+  command : [ doxygen , doxygen_cfg ],
+  input : [
+    doxygen_cfg,
+  ],
+  output : 'html',
+  install : true,
+  install_dir : get_option('datadir') / 'doc/nix/internal-api',
+  build_always_stale : true,
+)

src/internal-api-docs/package.nix

@@ -0,0 +1,54 @@
+{ lib
+, mkMesonDerivation
+
+, meson
+, ninja
+, doxygen
+
+# Configuration Options
+
+, version
+}:
+
+let
+  inherit (lib) fileset;
+in
+
+mkMesonDerivation (finalAttrs: {
+  pname = "nix-internal-api-docs";
+  inherit version;
+
+  workDir = ./.;
+  fileset = let
+    cpp = fileset.fileFilter (file: file.hasExt "cc" || file.hasExt "hh");
+  in fileset.unions [
+    ./.version
+    ../../.version
+    ./meson.build
+    ./doxygen.cfg.in
+    # Source is not compiled, but still must be available for Doxygen
+    # to gather comments.
+    (cpp ../.)
+  ];
+
+  nativeBuildInputs = [
+    meson
+    ninja
+    doxygen
+  ];
+
+  preConfigure =
+    ''
+      chmod u+w ./.version
+      echo ${finalAttrs.version} > ./.version
+    '';
+
+  postInstall = ''
+    mkdir -p ''${!outputDoc}/nix-support
+    echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products
+  '';
+
+  meta = {
+    platforms = lib.platforms.all;
+  };
+})

src/libcmd/.version

@@ -0,0 +1 @@
+../../.version

src/libcmd/build-utils-meson

@@ -0,0 +1 @@
+../../build-utils-meson

src/libcmd/built-path.cc

@@ -0,0 +1,144 @@
+#include "built-path.hh"
+#include "derivations.hh"
+#include "store-api.hh"
+#include "comparator.hh"
+
+#include <nlohmann/json.hpp>
+
+#include <optional>
+
+namespace nix {
+
+// Custom implementation to avoid `ref` ptr equality
+GENERATE_CMP_EXT(
+    ,
+    std::strong_ordering,
+    SingleBuiltPathBuilt,
+    *me->drvPath,
+    me->output);
+
+// Custom implementation to avoid `ref` ptr equality
+
+// TODO no `GENERATE_CMP_EXT` because no `std::set::operator<=>` on
+// Darwin, per header.
+GENERATE_EQUAL(
+    ,
+    BuiltPathBuilt ::,
+    BuiltPathBuilt,
+    *me->drvPath,
+    me->outputs);
+
+StorePath SingleBuiltPath::outPath() const
+{
+    return std::visit(
+        overloaded{
+            [](const SingleBuiltPath::Opaque & p) { return p.path; },
+            [](const SingleBuiltPath::Built & b) { return b.output.second; },
+        }, raw()
+    );
+}
+
+StorePathSet BuiltPath::outPaths() const
+{
+    return std::visit(
+        overloaded{
+            [](const BuiltPath::Opaque & p) { return StorePathSet{p.path}; },
+            [](const BuiltPath::Built & b) {
+                StorePathSet res;
+                for (auto & [_, path] : b.outputs)
+                    res.insert(path);
+                return res;
+            },
+        }, raw()
+    );
+}
+
+SingleDerivedPath::Built SingleBuiltPath::Built::discardOutputPath() const
+{
+    return SingleDerivedPath::Built {
+        .drvPath = make_ref<SingleDerivedPath>(drvPath->discardOutputPath()),
+        .output = output.first,
+    };
+}
+
+SingleDerivedPath SingleBuiltPath::discardOutputPath() const
+{
+    return std::visit(
+        overloaded{
+            [](const SingleBuiltPath::Opaque & p) -> SingleDerivedPath {
+                return p;
+            },
+            [](const SingleBuiltPath::Built & b) -> SingleDerivedPath {
+                return b.discardOutputPath();
+            },
+        }, raw()
+    );
+}
+
+nlohmann::json BuiltPath::Built::toJSON(const StoreDirConfig & store) const
+{
+    nlohmann::json res;
+    res["drvPath"] = drvPath->toJSON(store);
+    for (const auto & [outputName, outputPath] : outputs) {
+        res["outputs"][outputName] = store.printStorePath(outputPath);
+    }
+    return res;
+}
+
+nlohmann::json SingleBuiltPath::Built::toJSON(const StoreDirConfig & store) const
+{
+    nlohmann::json res;
+    res["drvPath"] = drvPath->toJSON(store);
+    auto & [outputName, outputPath] = output;
+    res["output"] = outputName;
+    res["outputPath"] = store.printStorePath(outputPath);
+    return res;
+}
+
+nlohmann::json SingleBuiltPath::toJSON(const StoreDirConfig & store) const
+{
+    return std::visit([&](const auto & buildable) {
+        return buildable.toJSON(store);
+    }, raw());
+}
+
+nlohmann::json BuiltPath::toJSON(const StoreDirConfig & store) const
+{
+    return std::visit([&](const auto & buildable) {
+        return buildable.toJSON(store);
+    }, raw());
+}
+
+RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const
+{
+    RealisedPath::Set res;
+    std::visit(
+        overloaded{
+            [&](const BuiltPath::Opaque & p) { res.insert(p.path); },
+            [&](const BuiltPath::Built & p) {
+                auto drvHashes =
+                    staticOutputHashes(store, store.readDerivation(p.drvPath->outPath()));
+                for (auto& [outputName, outputPath] : p.outputs) {
+                    if (experimentalFeatureSettings.isEnabled(
+                                Xp::CaDerivations)) {
+                        auto drvOutput = get(drvHashes, outputName);
+                        if (!drvOutput)
+                            throw Error(
+                                "the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)",
+                                store.printStorePath(p.drvPath->outPath()), outputName);
+                        auto thisRealisation = store.queryRealisation(
+                            DrvOutput{*drvOutput, outputName});
+                        assert(thisRealisation);  // We’ve built it, so we must
+                                                  // have the realisation
+                        res.insert(*thisRealisation);
+                    } else {
+                        res.insert(outputPath);
+                    }
+                }
+            },
+        },
+        raw());
+    return res;
+}
+
+}

src/libcmd/built-path.hh

@@ -0,0 +1,107 @@
+#pragma once
+///@file
+
+#include "derived-path.hh"
+#include "realisation.hh"
+
+namespace nix {
+
+struct SingleBuiltPath;
+
+struct SingleBuiltPathBuilt {
+    ref<SingleBuiltPath> drvPath;
+    std::pair<std::string, StorePath> output;
+
+    SingleDerivedPathBuilt discardOutputPath() const;
+
+    std::string to_string(const StoreDirConfig & store) const;
+    static SingleBuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view);
+    nlohmann::json toJSON(const StoreDirConfig & store) const;
+
+    bool operator ==(const SingleBuiltPathBuilt &) const noexcept;
+    std::strong_ordering operator <=>(const SingleBuiltPathBuilt &) const noexcept;
+};
+
+using _SingleBuiltPathRaw = std::variant<
+    DerivedPathOpaque,
+    SingleBuiltPathBuilt
+>;
+
+struct SingleBuiltPath : _SingleBuiltPathRaw {
+    using Raw = _SingleBuiltPathRaw;
+    using Raw::Raw;
+
+    using Opaque = DerivedPathOpaque;
+    using Built = SingleBuiltPathBuilt;
+
+    bool operator == (const SingleBuiltPath &) const = default;
+    auto operator <=> (const SingleBuiltPath &) const = default;
+
+    inline const Raw & raw() const {
+        return static_cast<const Raw &>(*this);
+    }
+
+    StorePath outPath() const;
+
+    SingleDerivedPath discardOutputPath() const;
+
+    static SingleBuiltPath parse(const StoreDirConfig & store, std::string_view);
+    nlohmann::json toJSON(const StoreDirConfig & store) const;
+};
+
+static inline ref<SingleBuiltPath> staticDrv(StorePath drvPath)
+{
+    return make_ref<SingleBuiltPath>(SingleBuiltPath::Opaque { drvPath });
+}
+
+/**
+ * A built derived path with hints in the form of optional concrete output paths.
+ *
+ * See 'BuiltPath' for more an explanation.
+ */
+struct BuiltPathBuilt {
+    ref<SingleBuiltPath> drvPath;
+    std::map<std::string, StorePath> outputs;
+
+    bool operator == (const BuiltPathBuilt &) const noexcept;
+    // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet.
+    //std::strong_ordering operator <=> (const BuiltPathBuilt &) const noexcept;
+
+    std::string to_string(const StoreDirConfig & store) const;
+    static BuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view);
+    nlohmann::json toJSON(const StoreDirConfig & store) const;
+};
+
+using _BuiltPathRaw = std::variant<
+    DerivedPath::Opaque,
+    BuiltPathBuilt
+>;
+
+/**
+ * A built path. Similar to a DerivedPath, but enriched with the corresponding
+ * output path(s).
+ */
+struct BuiltPath : _BuiltPathRaw {
+    using Raw = _BuiltPathRaw;
+    using Raw::Raw;
+
+    using Opaque = DerivedPathOpaque;
+    using Built = BuiltPathBuilt;
+
+    bool operator == (const BuiltPath &) const = default;
+    // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet.
+    //auto operator <=> (const BuiltPath &) const = default;
+
+    inline const Raw & raw() const {
+        return static_cast<const Raw &>(*this);
+    }
+
+    StorePathSet outPaths() const;
+    RealisedPath::Set toRealisedPaths(Store & store) const;
+
+    nlohmann::json toJSON(const StoreDirConfig & store) const;
+};
+
+typedef std::vector<BuiltPath> BuiltPaths;
+
+}

src/libcmd/command-installable-value.cc

@@ -0,0 +1,11 @@
+#include "command-installable-value.hh"
+
+namespace nix {
+
+void InstallableValueCommand::run(ref<Store> store, ref<Installable> installable)
+{
+    auto installableValue = InstallableValue::require(installable);
+    run(store, installableValue);
+}
+
+}

src/libcmd/command-installable-value.hh

@@ -0,0 +1,23 @@
+#pragma once
+///@file
+
+#include "installable-value.hh"
+#include "command.hh"
+
+namespace nix {
+
+/**
+ * An InstallableCommand where the single positional argument must be an
+ * InstallableValue in particular.
+ */
+struct InstallableValueCommand : InstallableCommand
+{
+    /**
+     * Entry point to this command
+     */
+    virtual void run(ref<Store> store, ref<InstallableValue> installable) = 0;
+
+    void run(ref<Store> store, ref<Installable> installable) override;
+};
+
+}

src/libcmd/command.cc

@@ -1,11 +1,14 @@
+#include <nlohmann/json.hpp>
+
 #include "command.hh"
+#include "markdown.hh"
 #include "store-api.hh"
 #include "local-fs-store.hh"
 #include "derivations.hh"
 #include "nixexpr.hh"
 #include "profiles.hh"
-
-#include <nlohmann/json.hpp>
+#include "repl.hh"
+#include "strings.hh"
 
 extern char * * environ __attribute__((weak));
 
@@ -33,6 +36,19 @@ nlohmann::json NixMultiCommand::toJSON()
     return MultiCommand::toJSON();
 }
 
+void NixMultiCommand::run()
+{
+    if (!command) {
+        std::set<std::string> subCommandTextLines;
+        for (auto & [name, _] : commands)
+            subCommandTextLines.insert(fmt("- `%s`", name));
+        std::string markdownError = fmt("`nix %s` requires a sub-command. Available sub-commands:\n\n%s\n",
+                commandName, concatStringsSep("\n", subCommandTextLines));
+        throw UsageError(renderMarkdownToTerminal(markdownError));
+    }
+    command->second->run();
+}
+
 StoreCommand::StoreCommand()
 {
 }
@@ -97,7 +113,7 @@ EvalCommand::EvalCommand()
 EvalCommand::~EvalCommand()
 {
     if (evalState)
-        evalState->printStats();
+        evalState->maybePrintStats();
 }
 
 ref<Store> EvalCommand::getEvalStore()
@@ -112,21 +128,33 @@ ref<EvalState> EvalCommand::getEvalState()
     if (!evalState) {
         evalState =
             #if HAVE_BOEHMGC
-            std::allocate_shared<EvalState>(traceable_allocator<EvalState>(),
-                searchPath, getEvalStore(), getStore())
+            std::allocate_shared<EvalState>(
+                traceable_allocator<EvalState>(),
             #else
             std::make_shared<EvalState>(
-                searchPath, getEvalStore(), getStore())
             #endif
+                lookupPath, getEvalStore(), fetchSettings, evalSettings, getStore())
             ;
 
+        evalState->repair = repair;
+
         if (startReplOnEvalErrors) {
-            evalState->debugRepl = &runRepl;
+            evalState->debugRepl = &AbstractNixRepl::runSimple;
         };
     }
     return ref<EvalState>(evalState);
 }
 
+MixOperateOnOptions::MixOperateOnOptions()
+{
+    addFlag({
+        .longName = "derivation",
+        .description = "Operate on the [store derivation](@docroot@/glossary.md#gloss-store-derivation) rather than its outputs.",
+        .category = installablesCategory,
+        .handler = {&operateOn, OperateOn::Derivation},
+    });
+}
+
 BuiltPathsCommand::BuiltPathsCommand(bool recursive)
     : recursive(recursive)
 {
@@ -154,7 +182,7 @@ BuiltPathsCommand::BuiltPathsCommand(bool recursive)
     });
 }
 
-void BuiltPathsCommand::run(ref<Store> store)
+void BuiltPathsCommand::run(ref<Store> store, Installables && installables)
 {
     BuiltPaths paths;
     if (all) {
@@ -162,7 +190,7 @@ void BuiltPathsCommand::run(ref<Store> store)
             throw UsageError("'--all' does not expect arguments");
         // XXX: Only uses opaque paths, ignores all the realisations
         for (auto & p : store->queryAllValidPaths())
-            paths.push_back(BuiltPath::Opaque{p});
+            paths.emplace_back(BuiltPath::Opaque{p});
     } else {
         paths = Installable::toBuiltPaths(getEvalStore(), store, realiseMode, operateOn, installables);
         if (recursive) {
@@ -175,7 +203,7 @@ void BuiltPathsCommand::run(ref<Store> store)
             }
             store->computeFSClosure(pathsRoots, pathsClosure);
             for (auto & path : pathsClosure)
-                paths.push_back(BuiltPath::Opaque{path});
+                paths.emplace_back(BuiltPath::Opaque{path});
         }
     }
 
@@ -200,7 +228,7 @@ void StorePathsCommand::run(ref<Store> store, BuiltPaths && paths)
     run(store, std::move(sorted));
 }
 
-void StorePathCommand::run(ref<Store> store, std::vector<StorePath> && storePaths)
+void StorePathCommand::run(ref<Store> store, StorePaths && storePaths)
 {
     if (storePaths.size() != 1)
         throw UsageError("this command requires exactly one store path");
@@ -208,20 +236,6 @@ void StorePathCommand::run(ref<Store> store, std::vector<StorePath> && storePath
     run(store, *storePaths.begin());
 }
 
-Strings editorFor(const Path & file, uint32_t line)
-{
-    auto editor = getEnv("EDITOR").value_or("cat");
-    auto args = tokenizeString<Strings>(editor);
-    if (line > 0 && (
-        editor.find("emacs") != std::string::npos ||
-        editor.find("nano") != std::string::npos ||
-        editor.find("vim") != std::string::npos ||
-        editor.find("kak") != std::string::npos))
-        args.push_back(fmt("+%d", line));
-    args.push_back(file);
-    return args;
-}
-
 MixProfile::MixProfile()
 {
     addFlag({
@@ -240,16 +254,14 @@ void MixProfile::updateProfile(const StorePath & storePath)
     if (!store) throw Error("'--profile' is not supported for this Nix store");
     auto profile2 = absPath(*profile);
     switchLink(profile2,
-        createGeneration(
-            ref<LocalFSStore>(store),
-            profile2, storePath));
+        createGeneration(*store, profile2, storePath));
 }
 
 void MixProfile::updateProfile(const BuiltPaths & buildables)
 {
     if (!profile) return;
 
-    std::vector<StorePath> result;
+    StorePaths result;
 
     for (auto & buildable : buildables) {
         std::visit(overloaded {

src/libcmd/command.hh

@@ -1,6 +1,7 @@
 #pragma once
+///@file
 
-#include "installables.hh"
+#include "installable-value.hh"
 #include "args.hh"
 #include "common-eval-args.hh"
 #include "path.hh"
@@ -18,32 +19,47 @@ class EvalState;
 struct Pos;
 class Store;
 
+static constexpr Command::Category catHelp = -1;
 static constexpr Command::Category catSecondary = 100;
 static constexpr Command::Category catUtility = 101;
 static constexpr Command::Category catNixInstallation = 102;
 
-static constexpr auto installablesCategory = "Options that change the interpretation of installables";
+static constexpr auto installablesCategory = "Options that change the interpretation of [installables](@docroot@/command-ref/new-cli/nix.md#installables)";
 
-struct NixMultiCommand : virtual MultiCommand, virtual Command
+struct NixMultiCommand : MultiCommand, virtual Command
 {
     nlohmann::json toJSON() override;
+
+    using MultiCommand::MultiCommand;
+
+    virtual void run() override;
 };
 
-/* A command that requires a Nix store. */
+// For the overloaded run methods
+#pragma GCC diagnostic ignored "-Woverloaded-virtual"
+
+/**
+ * A command that requires a \ref Store "Nix store".
+ */
 struct StoreCommand : virtual Command
 {
     StoreCommand();
     void run() override;
     ref<Store> getStore();
     virtual ref<Store> createStore();
+    /**
+     * Main entry point, with a `Store` provided
+     */
     virtual void run(ref<Store>) = 0;
 
 private:
     std::shared_ptr<Store> _store;
 };
 
-/* A command that copies something between `--from` and `--to`
-   stores. */
+/**
+ * A command that copies something between `--from` and `--to` \ref
+ * Store stores.
+ */
 struct CopyCommand : virtual StoreCommand
 {
     std::string srcUri, dstUri;
@@ -55,6 +71,9 @@ struct CopyCommand : virtual StoreCommand
     ref<Store> getDstStore();
 };
 
+/**
+ * A command that needs to evaluate Nix language expressions.
+ */
 struct EvalCommand : virtual StoreCommand, MixEvalArgs
 {
     bool startReplOnEvalErrors = false;
@@ -74,87 +93,137 @@ private:
     std::shared_ptr<EvalState> evalState;
 };
 
+/**
+ * A mixin class for commands that process flakes, adding a few standard
+ * flake-related options/flags.
+ */
 struct MixFlakeOptions : virtual Args, EvalCommand
 {
     flake::LockFlags lockFlags;
 
-    std::optional<std::string> needsFlakeInputCompletion = {};
-
     MixFlakeOptions();
 
-    virtual std::vector<std::string> getFlakesForCompletion()
+    /**
+     * The completion for some of these flags depends on the flake(s) in
+     * question.
+     *
+     * This method should be implemented to gather all flakerefs the
+     * command is operating with (presumably specified via some other
+     * arguments) so that the completions for these flags can use them.
+     */
+    virtual std::vector<FlakeRef> getFlakeRefsForCompletion()
     { return {}; }
-
-    void completeFlakeInput(std::string_view prefix);
-
-    void completionHook() override;
 };
 
 struct SourceExprCommand : virtual Args, MixFlakeOptions
 {
     std::optional<Path> file;
     std::optional<std::string> expr;
-    bool readOnlyMode = false;
 
-    // FIXME: move this; not all commands (e.g. 'nix run') use it.
-    OperateOn operateOn = OperateOn::Output;
+    SourceExprCommand();
 
-    SourceExprCommand(bool supportReadOnlyMode = false);
-
-    std::vector<std::shared_ptr<Installable>> parseInstallables(
+    Installables parseInstallables(
         ref<Store> store, std::vector<std::string> ss);
 
-    std::shared_ptr<Installable> parseInstallable(
+    ref<Installable> parseInstallable(
         ref<Store> store, const std::string & installable);
 
     virtual Strings getDefaultFlakeAttrPaths();
 
     virtual Strings getDefaultFlakeAttrPathPrefixes();
 
-    void completeInstallable(std::string_view prefix);
+    /**
+     * Complete an installable from the given prefix.
+     */
+    void completeInstallable(AddCompletions & completions, std::string_view prefix);
+
+    /**
+     * Convenience wrapper around the underlying function to make setting the
+     * callback easier.
+     */
+    CompleterClosure getCompleteInstallable();
 };
 
-/* A command that operates on a list of "installables", which can be
-   store paths, attribute paths, Nix expressions, etc. */
-struct InstallablesCommand : virtual Args, SourceExprCommand
+/**
+ * A mixin class for commands that need a read-only flag.
+ *
+ * What exactly is "read-only" is unspecified, but it will usually be
+ * the \ref Store "Nix store".
+ */
+struct MixReadOnlyOption : virtual Args
 {
-    std::vector<std::shared_ptr<Installable>> installables;
-
-    InstallablesCommand();
-
-    void prepare() override;
-    Installables load();
-
-    virtual bool useDefaultInstallables() { return true; }
-
-    std::vector<std::string> getFlakesForCompletion() override;
-
-protected:
-
-    std::vector<std::string> _installables;
+    MixReadOnlyOption();
 };
 
-/* A command that operates on exactly one "installable" */
+/**
+ * Like InstallablesCommand but the installables are not loaded.
+ *
+ * This is needed by `CmdRepl` which wants to load (and reload) the
+ * installables itself.
+ */
+struct RawInstallablesCommand : virtual Args, SourceExprCommand
+{
+    RawInstallablesCommand();
+
+    virtual void run(ref<Store> store, std::vector<std::string> && rawInstallables) = 0;
+
+    void run(ref<Store> store) override;
+
+    // FIXME make const after `CmdRepl`'s override is fixed up
+    virtual void applyDefaultInstallables(std::vector<std::string> & rawInstallables);
+
+    bool readFromStdIn = false;
+
+    std::vector<FlakeRef> getFlakeRefsForCompletion() override;
+
+private:
+
+    std::vector<std::string> rawInstallables;
+};
+
+/**
+ * A command that operates on a list of "installables", which can be
+ * store paths, attribute paths, Nix expressions, etc.
+ */
+struct InstallablesCommand : RawInstallablesCommand
+{
+    virtual void run(ref<Store> store, Installables && installables) = 0;
+
+    void run(ref<Store> store, std::vector<std::string> && rawInstallables) override;
+};
+
+/**
+ * A command that operates on exactly one "installable".
+ */
 struct InstallableCommand : virtual Args, SourceExprCommand
 {
-    std::shared_ptr<Installable> installable;
+    InstallableCommand();
 
-    InstallableCommand(bool supportReadOnlyMode = false);
+    virtual void run(ref<Store> store, ref<Installable> installable) = 0;
 
-    void prepare() override;
+    void run(ref<Store> store) override;
 
-    std::vector<std::string> getFlakesForCompletion() override
-    {
-        return {_installable};
-    }
+    std::vector<FlakeRef> getFlakeRefsForCompletion() override;
 
 private:
 
     std::string _installable{"."};
 };
 
-/* A command that operates on zero or more store paths. */
-struct BuiltPathsCommand : public InstallablesCommand
+struct MixOperateOnOptions : virtual Args
+{
+    OperateOn operateOn = OperateOn::Output;
+
+    MixOperateOnOptions();
+};
+
+/**
+ * A command that operates on zero or more extant store paths.
+ *
+ * If the argument the user passes is a some sort of recipe for a path
+ * not yet built, it must be built first.
+ */
+struct BuiltPathsCommand : InstallablesCommand, virtual MixOperateOnOptions
 {
 private:
 
@@ -169,37 +238,35 @@ public:
 
     BuiltPathsCommand(bool recursive = false);
 
-    using StoreCommand::run;
-
     virtual void run(ref<Store> store, BuiltPaths && paths) = 0;
 
-    void run(ref<Store> store) override;
+    void run(ref<Store> store, Installables && installables) override;
 
-    bool useDefaultInstallables() override { return !all; }
+    void applyDefaultInstallables(std::vector<std::string> & rawInstallables) override;
 };
 
 struct StorePathsCommand : public BuiltPathsCommand
 {
     StorePathsCommand(bool recursive = false);
 
-    using BuiltPathsCommand::run;
-
-    virtual void run(ref<Store> store, std::vector<StorePath> && storePaths) = 0;
+    virtual void run(ref<Store> store, StorePaths && storePaths) = 0;
 
     void run(ref<Store> store, BuiltPaths && paths) override;
 };
 
-/* A command that operates on exactly one store path. */
+/**
+ * A command that operates on exactly one store path.
+ */
 struct StorePathCommand : public StorePathsCommand
 {
-    using StorePathsCommand::run;
-
     virtual void run(ref<Store> store, const StorePath & storePath) = 0;
 
-    void run(ref<Store> store, std::vector<StorePath> && storePaths) override;
+    void run(ref<Store> store, StorePaths && storePaths) override;
 };
 
-/* A helper class for registering commands globally. */
+/**
+ * A helper class for registering \ref Command commands globally.
+ */
 struct RegisterCommand
 {
     typedef std::map<std::vector<std::string>, std::function<ref<Command>()>> Commands;
@@ -227,10 +294,6 @@ static RegisterCommand registerCommand2(std::vector<std::string> && name)
     return RegisterCommand(std::move(name), [](){ return make_ref<T>(); });
 }
 
-/* Helper function to generate args that invoke $EDITOR on
-   filename:lineno. */
-Strings editorFor(const Path & file, uint32_t line);
-
 struct MixProfile : virtual StoreCommand
 {
     std::optional<Path> profile;
@@ -259,13 +322,24 @@ struct MixEnvironment : virtual Args {
 
     MixEnvironment();
 
-    /* Modify global environ based on ignoreEnvironment, keep, and unset. It's expected that exec will be called before this class goes out of scope, otherwise environ will become invalid. */
+    /***
+     * Modify global environ based on `ignoreEnvironment`, `keep`, and
+     * `unset`. It's expected that exec will be called before this class
+     * goes out of scope, otherwise `environ` will become invalid.
+     */
     void setEnviron();
 };
 
-void completeFlakeRef(ref<Store> store, std::string_view prefix);
+void completeFlakeInputPath(
+    AddCompletions & completions,
+    ref<EvalState> evalState,
+    const std::vector<FlakeRef> & flakeRefs,
+    std::string_view prefix);
+
+void completeFlakeRef(AddCompletions & completions, ref<Store> store, std::string_view prefix);
 
 void completeFlakeRefWithFragment(
+    AddCompletions & completions,
     ref<EvalState> evalState,
     flake::LockFlags lockFlags,
     Strings attrPathPrefixes,
@@ -280,8 +354,4 @@ void printClosureDiff(
     const StorePath & afterPath,
     std::string_view indent);
 
-
-void runRepl(
-    ref<EvalState> evalState,
-    const ValMap & extraEnv);
 }

src/libcmd/common-eval-args.cc

@@ -1,16 +1,57 @@
+#include "fetch-settings.hh"
+#include "eval-settings.hh"
 #include "common-eval-args.hh"
 #include "shared.hh"
+#include "config-global.hh"
 #include "filetransfer.hh"
-#include "util.hh"
 #include "eval.hh"
 #include "fetchers.hh"
 #include "registry.hh"
 #include "flake/flakeref.hh"
+#include "flake/settings.hh"
 #include "store-api.hh"
 #include "command.hh"
+#include "tarball.hh"
+#include "fetch-to-store.hh"
+#include "compatibility-settings.hh"
+#include "eval-settings.hh"
 
 namespace nix {
 
+fetchers::Settings fetchSettings;
+
+static GlobalConfig::Register rFetchSettings(&fetchSettings);
+
+EvalSettings evalSettings {
+    settings.readOnlyMode,
+    {
+        {
+            "flake",
+            [](ref<Store> store, std::string_view rest) {
+                experimentalFeatureSettings.require(Xp::Flakes);
+                // FIXME `parseFlakeRef` should take a `std::string_view`.
+                auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false);
+                debug("fetching flake search path element '%s''", rest);
+                auto storePath = flakeRef.resolve(store).fetchTree(store).first;
+                return store->toRealPath(storePath);
+            },
+        },
+    },
+};
+
+static GlobalConfig::Register rEvalSettings(&evalSettings);
+
+
+flake::Settings flakeSettings;
+
+static GlobalConfig::Register rFlakeSettings(&flakeSettings);
+
+
+CompatibilitySettings compatibilitySettings {};
+
+static GlobalConfig::Register rCompatibilitySettings(&compatibilitySettings);
+
+
 MixEvalArgs::MixEvalArgs()
 {
     addFlag({
@@ -18,7 +59,7 @@ MixEvalArgs::MixEvalArgs()
         .description = "Pass the value *expr* as the argument *name* to Nix functions.",
         .category = category,
         .labels = {"name", "expr"},
-        .handler = {[&](std::string name, std::string expr) { autoArgs[name] = 'E' + expr; }}
+        .handler = {[&](std::string name, std::string expr) { autoArgs.insert_or_assign(name, AutoArg{AutoArgExpr{expr}}); }}
     });
 
     addFlag({
@@ -26,86 +67,41 @@ MixEvalArgs::MixEvalArgs()
         .description = "Pass the string *string* as the argument *name* to Nix functions.",
         .category = category,
         .labels = {"name", "string"},
-        .handler = {[&](std::string name, std::string s) { autoArgs[name] = 'S' + s; }},
+        .handler = {[&](std::string name, std::string s) { autoArgs.insert_or_assign(name, AutoArg{AutoArgString{s}}); }},
+    });
+
+    addFlag({
+        .longName = "arg-from-file",
+        .description = "Pass the contents of file *path* as the argument *name* to Nix functions.",
+        .category = category,
+        .labels = {"name", "path"},
+        .handler = {[&](std::string name, std::string path) { autoArgs.insert_or_assign(name, AutoArg{AutoArgFile{path}}); }},
+        .completer = completePath
+    });
+
+    addFlag({
+        .longName = "arg-from-stdin",
+        .description = "Pass the contents of stdin as the argument *name* to Nix functions.",
+        .category = category,
+        .labels = {"name"},
+        .handler = {[&](std::string name) { autoArgs.insert_or_assign(name, AutoArg{AutoArgStdin{}}); }},
     });
 
     addFlag({
         .longName = "include",
         .shortName = 'I',
         .description = R"(
-  Add *path* to the Nix search path. The Nix search path is
-  initialized from the colon-separated [`NIX_PATH`](./env-common.md#env-NIX_PATH) environment
-  variable, and is used to look up the location of Nix expressions using [paths](../language/values.md#type-path) enclosed in angle
-  brackets (i.e., `<nixpkgs>`).
+  Add *path* to search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md)
 
-  For instance, passing
+  This option may be given multiple times.
 
-  ```
-  -I /home/eelco/Dev
-  -I /etc/nixos
-  ```
-
-  will cause Nix to look for paths relative to `/home/eelco/Dev` and
-  `/etc/nixos`, in that order. This is equivalent to setting the
-  `NIX_PATH` environment variable to
-
-  ```
-  /home/eelco/Dev:/etc/nixos
-  ```
-
-  It is also possible to match paths against a prefix. For example,
-  passing
-
-  ```
-  -I nixpkgs=/home/eelco/Dev/nixpkgs-branch
-  -I /etc/nixos
-  ```
-
-  will cause Nix to search for `<nixpkgs/path>` in
-  `/home/eelco/Dev/nixpkgs-branch/path` and `/etc/nixos/nixpkgs/path`.
-
-  If a path in the Nix search path starts with `http://` or `https://`,
-  it is interpreted as the URL of a tarball that will be downloaded and
-  unpacked to a temporary location. The tarball must consist of a single
-  top-level directory. For example, passing
-
-  ```
-  -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/master.tar.gz
-  ```
-
-  tells Nix to download and use the current contents of the `master`
-  branch in the `nixpkgs` repository.
-
-  The URLs of the tarballs from the official `nixos.org` channels
-  (see [the manual page for `nix-channel`](../nix-channel.md)) can be
-  abbreviated as `channel:<channel-name>`.  For instance, the
-  following two flags are equivalent:
-
-  ```
-  -I nixpkgs=channel:nixos-21.05
-  -I nixpkgs=https://nixos.org/channels/nixos-21.05/nixexprs.tar.xz
-  ```
-
-  You can also fetch source trees using [flake URLs](./nix3-flake.md#url-like-syntax) and add them to the
-  search path. For instance,
-
-  ```
-  -I nixpkgs=flake:nixpkgs
-  ```
-
-  specifies that the prefix `nixpkgs` shall refer to the source tree
-  downloaded from the `nixpkgs` entry in the flake registry. Similarly,
-
-  ```
-  -I nixpkgs=flake:github:NixOS/nixpkgs/nixos-22.05
-  ```
-
-  makes `<nixpkgs>` refer to a particular branch of the
-  `NixOS/nixpkgs` repository on GitHub.
+  Paths added through `-I` take precedence over the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path) and the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH).
   )",
         .category = category,
         .labels = {"path"},
-        .handler = {[&](std::string s) { searchPath.push_back(s); }}
+        .handler = {[&](std::string s) {
+            lookupPath.elements.emplace_back(LookupPath::Elem::parse(s));
+        }}
     });
 
     addFlag({
@@ -123,20 +119,24 @@ MixEvalArgs::MixEvalArgs()
         .category = category,
         .labels = {"original-ref", "resolved-ref"},
         .handler = {[&](std::string _from, std::string _to) {
-            auto from = parseFlakeRef(_from, absPath("."));
-            auto to = parseFlakeRef(_to, absPath("."));
+            auto from = parseFlakeRef(fetchSettings, _from, absPath("."));
+            auto to = parseFlakeRef(fetchSettings, _to, absPath("."));
             fetchers::Attrs extraAttrs;
             if (to.subdir != "") extraAttrs["dir"] = to.subdir;
             fetchers::overrideRegistry(from.input, to.input, extraAttrs);
         }},
-        .completer = {[&](size_t, std::string_view prefix) {
-            completeFlakeRef(openStore(), prefix);
+        .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
+            completeFlakeRef(completions, openStore(), prefix);
         }}
     });
 
     addFlag({
         .longName = "eval-store",
-        .description = "The Nix store to use for evaluations.",
+        .description =
+          R"(
+            The [URL of the Nix store](@docroot@/store/types/index.md#store-url-format)
+            to use for evaluation, i.e. to store derivations (`.drv` files) and inputs referenced by them.
+          )",
         .category = category,
         .labels = {"store-url"},
         .handler = {&evalStoreUrl},
@@ -146,30 +146,43 @@ MixEvalArgs::MixEvalArgs()
 Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
 {
     auto res = state.buildBindings(autoArgs.size());
-    for (auto & i : autoArgs) {
+    for (auto & [name, arg] : autoArgs) {
         auto v = state.allocValue();
-        if (i.second[0] == 'E')
-            state.mkThunk_(*v, state.parseExprFromString(i.second.substr(1), absPath(".")));
-        else
-            v->mkString(((std::string_view) i.second).substr(1));
-        res.insert(state.symbols.create(i.first), v);
+        std::visit(overloaded {
+            [&](const AutoArgExpr & arg) {
+                state.mkThunk_(*v, state.parseExprFromString(arg.expr, compatibilitySettings.nixShellShebangArgumentsRelativeToScript ? state.rootPath(absPath(getCommandBaseDir())) : state.rootPath(".")));
+            },
+            [&](const AutoArgString & arg) {
+                v->mkString(arg.s);
+            },
+            [&](const AutoArgFile & arg) {
+                v->mkString(readFile(arg.path.string()));
+            },
+            [&](const AutoArgStdin & arg) {
+                v->mkString(readFile(STDIN_FILENO));
+            }
+        }, arg);
+        res.insert(state.symbols.create(name), v);
     }
     return res.finish();
 }
 
-Path lookupFileArg(EvalState & state, std::string_view s)
+SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * baseDir)
 {
     if (EvalSettings::isPseudoUrl(s)) {
-        auto storePath = fetchers::downloadTarball(
-            state.store, EvalSettings::resolvePseudoUrl(s), "source", false).first.storePath;
-        return state.store->toRealPath(storePath);
+        auto accessor = fetchers::downloadTarball(
+            state.store,
+            state.fetchSettings,
+            EvalSettings::resolvePseudoUrl(s));
+        auto storePath = fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy);
+        return state.rootPath(CanonPath(state.store->toRealPath(storePath)));
     }
 
     else if (hasPrefix(s, "flake:")) {
-        settings.requireExperimentalFeature(Xp::Flakes);
-        auto flakeRef = parseFlakeRef(std::string(s.substr(6)), {}, true, false);
-        auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first.storePath;
-        return state.store->toRealPath(storePath);
+        experimentalFeatureSettings.require(Xp::Flakes);
+        auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false);
+        auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first;
+        return state.rootPath(CanonPath(state.store->toRealPath(storePath)));
     }
 
     else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {
@@ -178,7 +191,7 @@ Path lookupFileArg(EvalState & state, std::string_view s)
     }
 
     else
-        return absPath(std::string(s));
+        return state.rootPath(baseDir ? absPath(s, *baseDir) : absPath(s));
 }
 
 }

src/libcmd/common-eval-args.hh

@@ -1,14 +1,48 @@
 #pragma once
+///@file
 
 #include "args.hh"
+#include "canon-path.hh"
+#include "common-args.hh"
+#include "search-path.hh"
+
+#include <filesystem>
 
 namespace nix {
 
 class Store;
-class EvalState;
-class Bindings;
 
-struct MixEvalArgs : virtual Args
+namespace fetchers { struct Settings; }
+
+class EvalState;
+struct EvalSettings;
+struct CompatibilitySettings;
+class Bindings;
+struct SourcePath;
+
+namespace flake { struct Settings; }
+
+/**
+ * @todo Get rid of global setttings variables
+ */
+extern fetchers::Settings fetchSettings;
+
+/**
+ * @todo Get rid of global setttings variables
+ */
+extern EvalSettings evalSettings;
+
+/**
+ * @todo Get rid of global setttings variables
+ */
+extern flake::Settings flakeSettings;
+
+/**
+ * Settings that control behaviors that have changed since Nix 2.3.
+ */
+extern CompatibilitySettings compatibilitySettings;
+
+struct MixEvalArgs : virtual Args, virtual MixRepair
 {
     static constexpr auto category = "Common evaluation options";
 
@@ -16,14 +50,24 @@ struct MixEvalArgs : virtual Args
 
     Bindings * getAutoArgs(EvalState & state);
 
-    Strings searchPath;
+    LookupPath lookupPath;
 
     std::optional<std::string> evalStoreUrl;
 
 private:
-    std::map<std::string, std::string> autoArgs;
+    struct AutoArgExpr { std::string expr; };
+    struct AutoArgString { std::string s; };
+    struct AutoArgFile { std::filesystem::path path; };
+    struct AutoArgStdin { };
+
+    using AutoArg = std::variant<AutoArgExpr, AutoArgString, AutoArgFile, AutoArgStdin>;
+
+    std::map<std::string, AutoArg> autoArgs;
 };
 
-Path lookupFileArg(EvalState & state, std::string_view s);
+/**
+ * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory)
+ */
+SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * baseDir = nullptr);
 
 }

src/libcmd/compatibility-settings.hh

@@ -0,0 +1,36 @@
+#pragma once
+#include "config.hh"
+
+namespace nix {
+struct CompatibilitySettings : public Config
+{
+
+    CompatibilitySettings() = default;
+
+    // Added in Nix 2.24, July 2024.
+    Setting<bool> nixShellAlwaysLooksForShellNix{this, true, "nix-shell-always-looks-for-shell-nix", R"(
+        Before Nix 2.24, [`nix-shell`](@docroot@/command-ref/nix-shell.md) would only look at `shell.nix` if it was in the working directory - when no file was specified.
+
+        Since Nix 2.24, `nix-shell` always looks for a `shell.nix`, whether that's in the working directory, or in a directory that was passed as an argument.
+
+        You may set this to `false` to temporarily revert to the behavior of Nix 2.23 and older.
+
+        Using this setting is not recommended.
+        It will be deprecated and removed.
+    )"};
+
+    // Added in Nix 2.24, July 2024.
+    Setting<bool> nixShellShebangArgumentsRelativeToScript{
+        this, true, "nix-shell-shebang-arguments-relative-to-script", R"(
+        Before Nix 2.24, relative file path expressions in arguments in a `nix-shell` shebang were resolved relative to the working directory.
+
+        Since Nix 2.24, `nix-shell` resolves these paths in a manner that is relative to the [base directory](@docroot@/glossary.md#gloss-base-directory), defined as the script's directory.
+
+        You may set this to `false` to temporarily revert to the behavior of Nix 2.23 and older.
+
+        Using this setting is not recommended.
+        It will be deprecated and removed.
+    )"};
+};
+
+};

src/libcmd/editor-for.cc

@@ -0,0 +1,24 @@
+#include "editor-for.hh"
+#include "environment-variables.hh"
+#include "source-path.hh"
+
+namespace nix {
+
+Strings editorFor(const SourcePath & file, uint32_t line)
+{
+    auto path = file.getPhysicalPath();
+    if (!path)
+        throw Error("cannot open '%s' in an editor because it has no physical path", file);
+    auto editor = getEnv("EDITOR").value_or("cat");
+    auto args = tokenizeString<Strings>(editor);
+    if (line > 0 && (
+        editor.find("emacs") != std::string::npos ||
+        editor.find("nano") != std::string::npos ||
+        editor.find("vim") != std::string::npos ||
+        editor.find("kak") != std::string::npos))
+        args.push_back(fmt("+%d", line));
+    args.push_back(path->string());
+    return args;
+}
+
+}

src/libcmd/editor-for.hh

@@ -0,0 +1,15 @@
+#pragma once
+///@file
+
+#include "types.hh"
+#include "source-path.hh"
+
+namespace nix {
+
+/**
+ * Helper function to generate args that invoke $EDITOR on
+ * filename:lineno.
+ */
+Strings editorFor(const SourcePath & file, uint32_t line);
+
+}

src/libcmd/installable-attr-path.cc

@@ -0,0 +1,124 @@
+#include "globals.hh"
+#include "installable-attr-path.hh"
+#include "outputs-spec.hh"
+#include "util.hh"
+#include "command.hh"
+#include "attr-path.hh"
+#include "common-eval-args.hh"
+#include "derivations.hh"
+#include "eval-inline.hh"
+#include "eval.hh"
+#include "get-drvs.hh"
+#include "store-api.hh"
+#include "shared.hh"
+#include "flake/flake.hh"
+#include "eval-cache.hh"
+#include "url.hh"
+#include "registry.hh"
+#include "build-result.hh"
+
+#include <regex>
+#include <queue>
+
+#include <nlohmann/json.hpp>
+
+namespace nix {
+
+InstallableAttrPath::InstallableAttrPath(
+    ref<EvalState> state,
+    SourceExprCommand & cmd,
+    Value * v,
+    const std::string & attrPath,
+    ExtendedOutputsSpec extendedOutputsSpec)
+    : InstallableValue(state)
+    , cmd(cmd)
+    , v(allocRootValue(v))
+    , attrPath(attrPath)
+    , extendedOutputsSpec(std::move(extendedOutputsSpec))
+{ }
+
+std::pair<Value *, PosIdx> InstallableAttrPath::toValue(EvalState & state)
+{
+    auto [vRes, pos] = findAlongAttrPath(state, attrPath, *cmd.getAutoArgs(state), **v);
+    state.forceValue(*vRes, pos);
+    return {vRes, pos};
+}
+
+DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths()
+{
+    auto [v, pos] = toValue(*state);
+
+    if (std::optional derivedPathWithInfo = trySinglePathToDerivedPaths(
+        *v,
+        pos,
+        fmt("while evaluating the attribute '%s'", attrPath)))
+    {
+        return { *derivedPathWithInfo };
+    }
+
+    Bindings & autoArgs = *cmd.getAutoArgs(*state);
+
+    PackageInfos packageInfos;
+    getDerivations(*state, *v, "", autoArgs, packageInfos, false);
+
+    // Backward compatibility hack: group results by drvPath. This
+    // helps keep .all output together.
+    std::map<StorePath, OutputsSpec> byDrvPath;
+
+    for (auto & packageInfo : packageInfos) {
+        auto drvPath = packageInfo.queryDrvPath();
+        if (!drvPath)
+            throw Error("'%s' is not a derivation", what());
+
+        auto newOutputs = std::visit(overloaded {
+            [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec {
+                std::set<std::string> outputsToInstall;
+                for (auto & output : packageInfo.queryOutputs(false, true))
+                    outputsToInstall.insert(output.first);
+                if (outputsToInstall.empty())
+                    outputsToInstall.insert("out");
+                return OutputsSpec::Names { std::move(outputsToInstall) };
+            },
+            [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec {
+                return e;
+            },
+        }, extendedOutputsSpec.raw);
+
+        auto [iter, didInsert] = byDrvPath.emplace(*drvPath, newOutputs);
+
+        if (!didInsert)
+            iter->second = iter->second.union_(newOutputs);
+    }
+
+    DerivedPathsWithInfo res;
+    for (auto & [drvPath, outputs] : byDrvPath)
+        res.push_back({
+            .path = DerivedPath::Built {
+                .drvPath = makeConstantStorePathRef(drvPath),
+                .outputs = outputs,
+            },
+            .info = make_ref<ExtraPathInfoValue>(ExtraPathInfoValue::Value {
+                .extendedOutputsSpec = outputs,
+                /* FIXME: reconsider backwards compatibility above
+                   so we can fill in this info. */
+            }),
+        });
+
+    return res;
+}
+
+InstallableAttrPath InstallableAttrPath::parse(
+    ref<EvalState> state,
+    SourceExprCommand & cmd,
+    Value * v,
+    std::string_view prefix,
+    ExtendedOutputsSpec extendedOutputsSpec)
+{
+    return {
+        state, cmd, v,
+        prefix == "." ? "" : std::string { prefix },
+        std::move(extendedOutputsSpec),
+    };
+}
+
+}

src/libcmd/installable-attr-path.hh

@@ -0,0 +1,58 @@
+#pragma once
+///@file
+
+#include "globals.hh"
+#include "installable-value.hh"
+#include "outputs-spec.hh"
+#include "command.hh"
+#include "attr-path.hh"
+#include "common-eval-args.hh"
+#include "derivations.hh"
+#include "eval-inline.hh"
+#include "eval.hh"
+#include "get-drvs.hh"
+#include "store-api.hh"
+#include "shared.hh"
+#include "eval-cache.hh"
+#include "url.hh"
+#include "registry.hh"
+#include "build-result.hh"
+
+#include <regex>
+#include <queue>
+
+#include <nlohmann/json.hpp>
+
+namespace nix {
+
+class InstallableAttrPath : public InstallableValue
+{
+    SourceExprCommand & cmd;
+    RootValue v;
+    std::string attrPath;
+    ExtendedOutputsSpec extendedOutputsSpec;
+
+    InstallableAttrPath(
+        ref<EvalState> state,
+        SourceExprCommand & cmd,
+        Value * v,
+        const std::string & attrPath,
+        ExtendedOutputsSpec extendedOutputsSpec);
+
+    std::string what() const override { return attrPath; };
+
+    std::pair<Value *, PosIdx> toValue(EvalState & state) override;
+
+    DerivedPathsWithInfo toDerivedPaths() override;
+
+public:
+
+    static InstallableAttrPath parse(
+        ref<EvalState> state,
+        SourceExprCommand & cmd,
+        Value * v,
+        std::string_view prefix,
+        ExtendedOutputsSpec extendedOutputsSpec);
+};
+
+}

src/libcmd/installable-derived-path.cc

@@ -0,0 +1,65 @@
+#include "installable-derived-path.hh"
+#include "derivations.hh"
+
+namespace nix {
+
+std::string InstallableDerivedPath::what() const
+{
+    return derivedPath.to_string(*store);
+}
+
+DerivedPathsWithInfo InstallableDerivedPath::toDerivedPaths()
+{
+    return {{
+        .path = derivedPath,
+        .info = make_ref<ExtraPathInfo>(),
+    }};
+}
+
+std::optional<StorePath> InstallableDerivedPath::getStorePath()
+{
+    return derivedPath.getBaseStorePath();
+}
+
+InstallableDerivedPath InstallableDerivedPath::parse(
+    ref<Store> store,
+    std::string_view prefix,
+    ExtendedOutputsSpec extendedOutputsSpec)
+{
+    auto derivedPath = std::visit(overloaded {
+        // If the user did not use ^, we treat the output more
+        // liberally: we accept a symlink chain or an actual
+        // store path.
+        [&](const ExtendedOutputsSpec::Default &) -> DerivedPath {
+            auto storePath = store->followLinksToStorePath(prefix);
+            // Remove this prior to stabilizing the new CLI.
+            if (storePath.isDerivation()) {
+                auto oldDerivedPath = DerivedPath::Built {
+                    .drvPath = makeConstantStorePathRef(storePath),
+                    .outputs = OutputsSpec::All { },
+                };
+                warn(
+                    "The interpretation of store paths arguments ending in `.drv` recently changed. If this command is now failing try again with '%s'",
+                    oldDerivedPath.to_string(*store));
+            };
+            return DerivedPath::Opaque {
+                .path = std::move(storePath),
+            };
+        },
+        // If the user did use ^, we just do exactly what is written.
+        [&](const ExtendedOutputsSpec::Explicit & outputSpec) -> DerivedPath {
+            auto drv = make_ref<SingleDerivedPath>(SingleDerivedPath::parse(*store, prefix));
+            drvRequireExperiment(*drv);
+            return DerivedPath::Built {
+                .drvPath = std::move(drv),
+                .outputs = outputSpec,
+            };
+        },
+    }, extendedOutputsSpec.raw);
+    return InstallableDerivedPath {
+        store,
+        std::move(derivedPath),
+    };
+}
+
+}

src/libcmd/installable-derived-path.hh

@@ -0,0 +1,29 @@
+#pragma once
+///@file
+
+#include "installables.hh"
+
+namespace nix {
+
+struct InstallableDerivedPath : Installable
+{
+    ref<Store> store;
+    DerivedPath derivedPath;
+
+    InstallableDerivedPath(ref<Store> store, DerivedPath && derivedPath)
+        : store(store), derivedPath(std::move(derivedPath))
+    { }
+
+    std::string what() const override;
+
+    DerivedPathsWithInfo toDerivedPaths() override;
+
+    std::optional<StorePath> getStorePath() override;
+
+    static InstallableDerivedPath parse(
+        ref<Store> store,
+        std::string_view prefix,
+        ExtendedOutputsSpec extendedOutputsSpec);
+};
+
+}

src/libcmd/installable-flake.cc

@@ -0,0 +1,219 @@
+#include "globals.hh"
+#include "installable-flake.hh"
+#include "installable-derived-path.hh"
+#include "outputs-spec.hh"
+#include "util.hh"
+#include "command.hh"
+#include "attr-path.hh"
+#include "common-eval-args.hh"
+#include "derivations.hh"
+#include "eval-inline.hh"
+#include "eval.hh"
+#include "get-drvs.hh"
+#include "store-api.hh"
+#include "shared.hh"
+#include "flake/flake.hh"
+#include "eval-cache.hh"
+#include "url.hh"
+#include "registry.hh"
+#include "build-result.hh"
+
+#include <regex>
+#include <queue>
+
+#include <nlohmann/json.hpp>
+
+namespace nix {
+
+std::vector<std::string> InstallableFlake::getActualAttrPaths()
+{
+    std::vector<std::string> res;
+    if (attrPaths.size() == 1 && attrPaths.front().starts_with(".")){
+        attrPaths.front().erase(0,1);
+        res.push_back(attrPaths.front());
+        return res;
+    }
+
+    for (auto & prefix : prefixes)
+        res.push_back(prefix + *attrPaths.begin());
+
+    for (auto & s : attrPaths)
+        res.push_back(s);
+
+    return res;
+}
+
+static std::string showAttrPaths(const std::vector<std::string> & paths)
+{
+    std::string s;
+    for (const auto & [n, i] : enumerate(paths)) {
+        if (n > 0) s += n + 1 == paths.size() ? " or " : ", ";
+        s += '\''; s += i; s += '\'';
+    }
+    return s;
+}
+
+InstallableFlake::InstallableFlake(
+    SourceExprCommand * cmd,
+    ref<EvalState> state,
+    FlakeRef && flakeRef,
+    std::string_view fragment,
+    ExtendedOutputsSpec extendedOutputsSpec,
+    Strings attrPaths,
+    Strings prefixes,
+    const flake::LockFlags & lockFlags)
+    : InstallableValue(state),
+      flakeRef(flakeRef),
+      attrPaths(fragment == "" ? attrPaths : Strings{(std::string) fragment}),
+      prefixes(fragment == "" ? Strings{} : prefixes),
+      extendedOutputsSpec(std::move(extendedOutputsSpec)),
+      lockFlags(lockFlags)
+{
+    if (cmd && cmd->getAutoArgs(*state)->size())
+        throw UsageError("'--arg' and '--argstr' are incompatible with flakes");
+}
+
+DerivedPathsWithInfo InstallableFlake::toDerivedPaths()
+{
+    Activity act(*logger, lvlTalkative, actUnknown, fmt("evaluating derivation '%s'", what()));
+
+    auto attr = getCursor(*state);
+
+    auto attrPath = attr->getAttrPathStr();
+
+    if (!attr->isDerivation()) {
+
+        // FIXME: use eval cache?
+        auto v = attr->forceValue();
+
+        if (std::optional derivedPathWithInfo = trySinglePathToDerivedPaths(
+            v,
+            noPos,
+            fmt("while evaluating the flake output attribute '%s'", attrPath)))
+        {
+            return { *derivedPathWithInfo };
+        } else {
+            throw Error(
+                "expected flake output attribute '%s' to be a derivation or path but found %s: %s",
+                attrPath,
+                showType(v),
+                ValuePrinter(*this->state, v, errorPrintOptions)
+            );
+        }
+    }
+
+    auto drvPath = attr->forceDerivation();
+
+    std::optional<NixInt::Inner> priority;
+
+    if (attr->maybeGetAttr(state->sOutputSpecified)) {
+    } else if (auto aMeta = attr->maybeGetAttr(state->sMeta)) {
+        if (auto aPriority = aMeta->maybeGetAttr("priority"))
+            priority = aPriority->getInt().value;
+    }
+
+    return {{
+        .path = DerivedPath::Built {
+            .drvPath = makeConstantStorePathRef(std::move(drvPath)),
+            .outputs = std::visit(overloaded {
+                [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec {
+                    std::set<std::string> outputsToInstall;
+                    if (auto aOutputSpecified = attr->maybeGetAttr(state->sOutputSpecified)) {
+                        if (aOutputSpecified->getBool()) {
+                            if (auto aOutputName = attr->maybeGetAttr("outputName"))
+                                outputsToInstall = { aOutputName->getString() };
+                        }
+                    } else if (auto aMeta = attr->maybeGetAttr(state->sMeta)) {
+                        if (auto aOutputsToInstall = aMeta->maybeGetAttr("outputsToInstall"))
+                            for (auto & s : aOutputsToInstall->getListOfStrings())
+                                outputsToInstall.insert(s);
+                    }
+
+                    if (outputsToInstall.empty())
+                        outputsToInstall.insert("out");
+
+                    return OutputsSpec::Names { std::move(outputsToInstall) };
+                },
+                [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec {
+                    return e;
+                },
+            }, extendedOutputsSpec.raw),
+        },
+        .info = make_ref<ExtraPathInfoFlake>(
+            ExtraPathInfoValue::Value {
+                .priority = priority,
+                .attrPath = attrPath,
+                .extendedOutputsSpec = extendedOutputsSpec,
+            },
+            ExtraPathInfoFlake::Flake {
+                .originalRef = flakeRef,
+                .lockedRef = getLockedFlake()->flake.lockedRef,
+            }),
+    }};
+}
+
+std::pair<Value *, PosIdx> InstallableFlake::toValue(EvalState & state)
+{
+    return {&getCursor(state)->forceValue(), noPos};
+}
+
+std::vector<ref<eval_cache::AttrCursor>>
+InstallableFlake::getCursors(EvalState & state)
+{
+    auto evalCache = openEvalCache(state, getLockedFlake());
+
+    auto root = evalCache->getRoot();
+
+    std::vector<ref<eval_cache::AttrCursor>> res;
+
+    Suggestions suggestions;
+    auto attrPaths = getActualAttrPaths();
+
+    for (auto & attrPath : attrPaths) {
+        debug("trying flake output attribute '%s'", attrPath);
+
+        auto attr = root->findAlongAttrPath(parseAttrPath(state, attrPath));
+        if (attr) {
+            res.push_back(ref(*attr));
+        } else {
+            suggestions += attr.getSuggestions();
+        }
+    }
+
+    if (res.size() == 0)
+        throw Error(
+            suggestions,
+            "flake '%s' does not provide attribute %s",
+            flakeRef,
+            showAttrPaths(attrPaths));
+
+    return res;
+}
+
+std::shared_ptr<flake::LockedFlake> InstallableFlake::getLockedFlake() const
+{
+    if (!_lockedFlake) {
+        flake::LockFlags lockFlagsApplyConfig = lockFlags;
+        // FIXME why this side effect?
+        lockFlagsApplyConfig.applyNixConfig = true;
+        _lockedFlake = std::make_shared<flake::LockedFlake>(lockFlake(
+            flakeSettings, *state, flakeRef, lockFlagsApplyConfig));
+    }
+    return _lockedFlake;
+}
+
+FlakeRef InstallableFlake::nixpkgsFlakeRef() const
+{
+    auto lockedFlake = getLockedFlake();
+
+    if (auto nixpkgsInput = lockedFlake->lockFile.findInput({"nixpkgs"})) {
+        if (auto lockedNode = std::dynamic_pointer_cast<const flake::LockedNode>(nixpkgsInput)) {
+            debug("using nixpkgs flake '%s'", lockedNode->lockedRef);
+            return std::move(lockedNode->lockedRef);
+        }
+    }
+
+    return defaultNixpkgsFlakeRef();
+}
+
+}

src/libcmd/installable-flake.hh

@@ -0,0 +1,89 @@
+#pragma once
+///@file
+
+#include "common-eval-args.hh"
+#include "installable-value.hh"
+
+namespace nix {
+
+/**
+ * Extra info about a \ref DerivedPath "derived path" that ultimately
+ * come from a Flake.
+ *
+ * Invariant: every ExtraPathInfo gotten from an InstallableFlake should
+ * be possible to downcast to an ExtraPathInfoFlake.
+ */
+struct ExtraPathInfoFlake : ExtraPathInfoValue
+{
+    /**
+     * Extra struct to get around C++ designated initializer limitations
+     */
+    struct Flake {
+        FlakeRef originalRef;
+        FlakeRef lockedRef;
+    };
+
+    Flake flake;
+
+    ExtraPathInfoFlake(Value && v, Flake && f)
+        : ExtraPathInfoValue(std::move(v)), flake(f)
+    { }
+};
+
+struct InstallableFlake : InstallableValue
+{
+    FlakeRef flakeRef;
+    Strings attrPaths;
+    Strings prefixes;
+    ExtendedOutputsSpec extendedOutputsSpec;
+    const flake::LockFlags & lockFlags;
+    mutable std::shared_ptr<flake::LockedFlake> _lockedFlake;
+
+    InstallableFlake(
+        SourceExprCommand * cmd,
+        ref<EvalState> state,
+        FlakeRef && flakeRef,
+        std::string_view fragment,
+        ExtendedOutputsSpec extendedOutputsSpec,
+        Strings attrPaths,
+        Strings prefixes,
+        const flake::LockFlags & lockFlags);
+
+    std::string what() const override { return flakeRef.to_string() + "#" + *attrPaths.begin(); }
+
+    std::vector<std::string> getActualAttrPaths();
+
+    DerivedPathsWithInfo toDerivedPaths() override;
+
+    std::pair<Value *, PosIdx> toValue(EvalState & state) override;
+
+    /**
+     * Get a cursor to every attrpath in getActualAttrPaths() that
+     * exists. However if none exists, throw an exception.
+     */
+    std::vector<ref<eval_cache::AttrCursor>>
+    getCursors(EvalState & state) override;
+
+    std::shared_ptr<flake::LockedFlake> getLockedFlake() const;
+
+    FlakeRef nixpkgsFlakeRef() const;
+};
+
+/**
+ * Default flake ref for referring to Nixpkgs. For flakes that don't
+ * have their own Nixpkgs input, or other installables.
+ *
+ * It is a layer violation for Nix to know about Nixpkgs; currently just
+ * `nix develop` does. Be wary of using this /
+ * `InstallableFlake::nixpkgsFlakeRef` more places.
+ */
+static inline FlakeRef defaultNixpkgsFlakeRef()
+{
+    return FlakeRef::fromAttrs(fetchSettings, {{"type","indirect"}, {"id", "nixpkgs"}});
+}
+
+ref<eval_cache::EvalCache> openEvalCache(
+    EvalState & state,
+    std::shared_ptr<flake::LockedFlake> lockedFlake);
+
+}

src/libcmd/installable-value.cc

@@ -0,0 +1,68 @@
+#include "installable-value.hh"
+#include "eval-cache.hh"
+#include "fetch-to-store.hh"
+
+namespace nix {
+
+std::vector<ref<eval_cache::AttrCursor>>
+InstallableValue::getCursors(EvalState & state)
+{
+    auto evalCache =
+        std::make_shared<nix::eval_cache::EvalCache>(std::nullopt, state,
+            [&]() { return toValue(state).first; });
+    return {evalCache->getRoot()};
+}
+
+ref<eval_cache::AttrCursor>
+InstallableValue::getCursor(EvalState & state)
+{
+    /* Although getCursors should return at least one element, in case it doesn't,
+       bound check to avoid an undefined behavior for vector[0] */
+    return getCursors(state).at(0);
+}
+
+static UsageError nonValueInstallable(Installable & installable)
+{
+    return UsageError("installable '%s' does not correspond to a Nix language value", installable.what());
+}
+
+InstallableValue & InstallableValue::require(Installable & installable)
+{
+    auto * castedInstallable = dynamic_cast<InstallableValue *>(&installable);
+    if (!castedInstallable)
+        throw nonValueInstallable(installable);
+    return *castedInstallable;
+}
+
+ref<InstallableValue> InstallableValue::require(ref<Installable> installable)
+{
+    auto castedInstallable = installable.dynamic_pointer_cast<InstallableValue>();
+    if (!castedInstallable)
+        throw nonValueInstallable(*installable);
+    return ref { castedInstallable };
+}
+
+std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx)
+{
+    if (v.type() == nPath) {
+        auto storePath = fetchToStore(*state->store, v.path(), FetchMode::Copy);
+        return {{
+            .path = DerivedPath::Opaque {
+                .path = std::move(storePath),
+            },
+            .info = make_ref<ExtraPathInfo>(),
+        }};
+    }
+
+    else if (v.type() == nString) {
+        return {{
+            .path = DerivedPath::fromSingle(
+                state->coerceToSingleDerivedPath(pos, v, errorCtx)),
+            .info = make_ref<ExtraPathInfo>(),
+        }};
+    }
+
+    else return std::nullopt;
+}
+
+}

src/libcmd/installable-value.hh

@@ -0,0 +1,121 @@
+#pragma once
+///@file
+
+#include "installables.hh"
+#include "flake/flake.hh"
+
+namespace nix {
+
+struct PackageInfo;
+struct SourceExprCommand;
+
+namespace eval_cache { class EvalCache; class AttrCursor; }
+
+struct App
+{
+    std::vector<DerivedPath> context;
+    Path program;
+    // FIXME: add args, sandbox settings, metadata, ...
+};
+
+struct UnresolvedApp
+{
+    App unresolved;
+    App resolve(ref<Store> evalStore, ref<Store> store);
+};
+
+/**
+ * Extra info about a \ref DerivedPath "derived path" that ultimately
+ * come from a Nix language value.
+ *
+ * Invariant: every ExtraPathInfo gotten from an InstallableValue should
+ * be possible to downcast to an ExtraPathInfoValue.
+ */
+struct ExtraPathInfoValue : ExtraPathInfo
+{
+    /**
+     * Extra struct to get around C++ designated initializer limitations
+     */
+    struct Value {
+        /**
+         * An optional priority for use with "build envs". See Package
+         */
+        std::optional<NixInt::Inner> priority;
+
+        /**
+         * The attribute path associated with this value. The idea is
+         * that an installable referring to a value typically refers to
+         * a larger value, from which we project a smaller value out
+         * with this.
+         */
+        std::string attrPath;
+
+        /**
+         * \todo merge with DerivedPath's 'outputs' field?
+         */
+        ExtendedOutputsSpec extendedOutputsSpec;
+    };
+
+    Value value;
+
+    ExtraPathInfoValue(Value && v)
+        : value(v)
+    { }
+
+    virtual ~ExtraPathInfoValue() = default;
+};
+
+/**
+ * An Installable which corresponds a Nix language value, in addition to
+ * a collection of \ref DerivedPath "derived paths".
+ */
+struct InstallableValue : Installable
+{
+    ref<EvalState> state;
+
+    InstallableValue(ref<EvalState> state) : state(state) {}
+
+    virtual ~InstallableValue() { }
+
+    virtual std::pair<Value *, PosIdx> toValue(EvalState & state) = 0;
+
+    /**
+     * Get a cursor to each value this Installable could refer to.
+     * However if none exists, throw exception instead of returning
+     * empty vector.
+     */
+    virtual std::vector<ref<eval_cache::AttrCursor>>
+    getCursors(EvalState & state);
+
+    /**
+     * Get the first and most preferred cursor this Installable could
+     * refer to, or throw an exception if none exists.
+     */
+    virtual ref<eval_cache::AttrCursor>
+    getCursor(EvalState & state);
+
+    UnresolvedApp toApp(EvalState & state);
+
+    static InstallableValue & require(Installable & installable);
+    static ref<InstallableValue> require(ref<Installable> installable);
+
+protected:
+
+    /**
+     * Handles either a plain path, or a string with a single string
+     * context elem in the right format. The latter case is handled by
+     * `EvalState::coerceToDerivedPath()`; see it for details.
+     *
+     * @param v Value that is hopefully a string or path per the above.
+     *
+     * @param pos Position of value to aid with diagnostics.
+     *
+     * @param errorCtx Arbitrary message for use in potential error message when something is wrong with `v`.
+     *
+     * @result A derived path (with empty info, for now) if the value
+     * matched the above criteria.
+     */
+    std::optional<DerivedPathWithInfo> trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx);
+};
+
+}

src/libcmd/installables.hh

@@ -1,134 +1,194 @@
 #pragma once
+///@file
 
-#include "util.hh"
 #include "path.hh"
-#include "path-with-outputs.hh"
+#include "outputs-spec.hh"
 #include "derived-path.hh"
-#include "eval.hh"
+#include "built-path.hh"
 #include "store-api.hh"
-#include "flake/flake.hh"
 #include "build-result.hh"
 
 #include <optional>
 
 namespace nix {
 
-struct DrvInfo;
-struct SourceExprCommand;
-
-namespace eval_cache { class EvalCache; class AttrCursor; }
-
-struct App
-{
-    std::vector<StorePathWithOutputs> context;
-    Path program;
-    // FIXME: add args, sandbox settings, metadata, ...
-};
-
-struct UnresolvedApp
-{
-    App unresolved;
-    App resolve(ref<Store> evalStore, ref<Store> store);
-};
+struct PackageInfo;
 
 enum class Realise {
-    /* Build the derivation. Postcondition: the
-       derivation outputs exist. */
+    /**
+     * Build the derivation.
+     *
+     * Postcondition: the derivation outputs exist.
+     */
     Outputs,
-    /* Don't build the derivation. Postcondition: the store derivation
-       exists. */
+    /**
+     * Don't build the derivation.
+     *
+     * Postcondition: the store derivation exists.
+     */
     Derivation,
-    /* Evaluate in dry-run mode. Postcondition: nothing. */
-    // FIXME: currently unused, but could be revived if we can
-    // evaluate derivations in-memory.
+    /**
+     * Evaluate in dry-run mode.
+     *
+     * Postcondition: nothing.
+     *
+     * \todo currently unused, but could be revived if we can evaluate
+     * derivations in-memory.
+     */
     Nothing
 };
 
-/* How to handle derivations in commands that operate on store paths. */
+/**
+ * How to handle derivations in commands that operate on store paths.
+ */
 enum class OperateOn {
-    /* Operate on the output path. */
+    /**
+     * Operate on the output path.
+     */
     Output,
-    /* Operate on the .drv path. */
+    /**
+     * Operate on the .drv path.
+     */
     Derivation
 };
 
+/**
+ * Extra info about a DerivedPath
+ *
+ * Yes, this is empty, but that is intended. It will be sub-classed by
+ * the subclasses of Installable to allow those to provide more info.
+ * Certain commands will make use of this info.
+ */
+struct ExtraPathInfo
+{
+    virtual ~ExtraPathInfo() = default;
+};
+
+/**
+ * A DerivedPath with \ref ExtraPathInfo "any additional info" that
+ * commands might need from the derivation.
+ */
+struct DerivedPathWithInfo
+{
+    DerivedPath path;
+    ref<ExtraPathInfo> info;
+};
+
+/**
+ * Like DerivedPathWithInfo but extending BuiltPath with \ref
+ * ExtraPathInfo "extra info" and also possibly the \ref BuildResult
+ * "result of building".
+ */
 struct BuiltPathWithResult
 {
     BuiltPath path;
+    ref<ExtraPathInfo> info;
     std::optional<BuildResult> result;
 };
 
+/**
+ * Shorthand, for less typing and helping us keep the choice of
+ * collection in sync.
+ */
+typedef std::vector<DerivedPathWithInfo> DerivedPathsWithInfo;
+
+struct Installable;
+
+/**
+ * Shorthand, for less typing and helping us keep the choice of
+ * collection in sync.
+ */
+typedef std::vector<ref<Installable>> Installables;
+
+/**
+ * Installables are the main positional arguments for the Nix
+ * Command-line.
+ *
+ * This base class is very flexible, and just assumes and the
+ * Installable refers to a collection of \ref DerivedPath "derived paths" with
+ * \ref ExtraPathInfo "extra info".
+ */
 struct Installable
 {
     virtual ~Installable() { }
 
+    /**
+     * What Installable is this?
+     *
+     * Prints back valid CLI syntax that would result in this same
+     * installable. It doesn't need to be exactly what the user wrote,
+     * just something that means the same thing.
+     */
     virtual std::string what() const = 0;
 
-    virtual DerivedPaths toDerivedPaths() = 0;
+    /**
+     * Get the collection of \ref DerivedPathWithInfo "derived paths
+     * with info" that this \ref Installable instalallable denotes.
+     *
+     * This is the main method of this class
+     */
+    virtual DerivedPathsWithInfo toDerivedPaths() = 0;
 
-    virtual StorePathSet toDrvPaths(ref<Store> store)
-    {
-        throw Error("'%s' cannot be converted to a derivation path", what());
-    }
+    /**
+     * A convenience wrapper of the above for when we expect an
+     * installable to produce a single \ref DerivedPath "derived path"
+     * only.
+     *
+     * If no or multiple \ref DerivedPath "derived paths" are produced,
+     * and error is raised.
+     */
+    DerivedPathWithInfo toDerivedPath();
 
-    DerivedPath toDerivedPath();
-
-    UnresolvedApp toApp(EvalState & state);
-
-    virtual std::pair<Value *, PosIdx> toValue(EvalState & state)
-    {
-        throw Error("argument '%s' cannot be evaluated", what());
-    }
-
-    /* Return a value only if this installable is a store path or a
-       symlink to it. */
+    /**
+     * Return a value only if this installable is a store path or a
+     * symlink to it.
+     *
+     * \todo should we move this to InstallableDerivedPath? It is only
+     * supposed to work there anyways. Can always downcast.
+     */
     virtual std::optional<StorePath> getStorePath()
     {
         return {};
     }
 
-    virtual std::vector<ref<eval_cache::AttrCursor>>
-    getCursors(EvalState & state);
-
-    virtual ref<eval_cache::AttrCursor>
-    getCursor(EvalState & state);
-
-    virtual FlakeRef nixpkgsFlakeRef() const
-    {
-        return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}});
-    }
-
     static std::vector<BuiltPathWithResult> build(
         ref<Store> evalStore,
         ref<Store> store,
         Realise mode,
-        const std::vector<std::shared_ptr<Installable>> & installables,
+        const Installables & installables,
         BuildMode bMode = bmNormal);
 
-    static std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> build2(
+    static std::vector<std::pair<ref<Installable>, BuiltPathWithResult>> build2(
         ref<Store> evalStore,
         ref<Store> store,
         Realise mode,
-        const std::vector<std::shared_ptr<Installable>> & installables,
+        const Installables & installables,
         BuildMode bMode = bmNormal);
 
-    static std::set<StorePath> toStorePaths(
+    static std::set<StorePath> toStorePathSet(
         ref<Store> evalStore,
         ref<Store> store,
         Realise mode,
         OperateOn operateOn,
-        const std::vector<std::shared_ptr<Installable>> & installables);
+        const Installables & installables);
+
+    static std::vector<StorePath> toStorePaths(
+        ref<Store> evalStore,
+        ref<Store> store,
+        Realise mode,
+        OperateOn operateOn,
+        const Installables & installables);
 
     static StorePath toStorePath(
         ref<Store> evalStore,
         ref<Store> store,
         Realise mode,
         OperateOn operateOn,
-        std::shared_ptr<Installable> installable);
+        ref<Installable> installable);
 
     static std::set<StorePath> toDerivations(
         ref<Store> store,
-        const std::vector<std::shared_ptr<Installable>> & installables,
+        const Installables & installables,
         bool useDeriver = false);
 
     static BuiltPaths toBuiltPaths(
@@ -136,78 +196,7 @@ struct Installable
         ref<Store> store,
         Realise mode,
         OperateOn operateOn,
-        const std::vector<std::shared_ptr<Installable>> & installables);
+        const Installables & installables);
 };
 
-typedef std::vector<std::shared_ptr<Installable>> Installables;
-
-struct InstallableValue : Installable
-{
-    ref<EvalState> state;
-
-    InstallableValue(ref<EvalState> state) : state(state) {}
-
-    struct DerivationInfo
-    {
-        StorePath drvPath;
-        std::set<std::string> outputsToInstall;
-        std::optional<NixInt> priority;
-    };
-
-    virtual std::vector<DerivationInfo> toDerivations() = 0;
-
-    DerivedPaths toDerivedPaths() override;
-
-    StorePathSet toDrvPaths(ref<Store> store) override;
-};
-
-struct InstallableFlake : InstallableValue
-{
-    FlakeRef flakeRef;
-    Strings attrPaths;
-    Strings prefixes;
-    OutputsSpec outputsSpec;
-    const flake::LockFlags & lockFlags;
-    mutable std::shared_ptr<flake::LockedFlake> _lockedFlake;
-
-    InstallableFlake(
-        SourceExprCommand * cmd,
-        ref<EvalState> state,
-        FlakeRef && flakeRef,
-        std::string_view fragment,
-        OutputsSpec outputsSpec,
-        Strings attrPaths,
-        Strings prefixes,
-        const flake::LockFlags & lockFlags);
-
-    std::string what() const override { return flakeRef.to_string() + "#" + *attrPaths.begin(); }
-
-    std::vector<std::string> getActualAttrPaths();
-
-    Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake);
-
-    std::tuple<std::string, FlakeRef, DerivationInfo> toDerivation();
-
-    std::vector<DerivationInfo> toDerivations() override;
-
-    std::pair<Value *, PosIdx> toValue(EvalState & state) override;
-
-    /* Get a cursor to every attrpath in getActualAttrPaths() that
-       exists. */
-    std::vector<ref<eval_cache::AttrCursor>>
-    getCursors(EvalState & state) override;
-
-    /* Get a cursor to the first attrpath in getActualAttrPaths() that
-       exists, or throw an exception with suggestions if none exists. */
-    ref<eval_cache::AttrCursor> getCursor(EvalState & state) override;
-
-    std::shared_ptr<flake::LockedFlake> getLockedFlake() const;
-
-    FlakeRef nixpkgsFlakeRef() const override;
-};
-
-ref<eval_cache::EvalCache> openEvalCache(
-    EvalState & state,
-    std::shared_ptr<flake::LockedFlake> lockedFlake);
-
 }

src/libcmd/legacy.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include <functional>
 #include <map>

src/libcmd/local.mk

@@ -6,10 +6,10 @@ libcmd_DIR := $(d)
 
 libcmd_SOURCES := $(wildcard $(d)/*.cc)
 
-libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers -I src/nix
+libcmd_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) $(INCLUDE_libflake) $(INCLUDE_libmain)
 
-libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) -pthread
+libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) $(THREAD_LDFLAGS)
 
-libcmd_LIBS = libstore libutil libexpr libmain libfetchers
+libcmd_LIBS = libutil libstore libfetchers libflake libexpr libmain
 
-$(eval $(call install-file-in, $(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644))
+$(eval $(call install-file-in, $(buildprefix)$(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644))

src/libcmd/markdown.cc

@@ -1,17 +1,23 @@
 #include "markdown.hh"
-#include "util.hh"
+#include "environment-variables.hh"
+#include "error.hh"
 #include "finally.hh"
+#include "terminal.hh"
 
-#include <sys/queue.h>
-#include <lowdown.h>
+#if HAVE_LOWDOWN
+#  include <sys/queue.h>
+#  include <lowdown.h>
+#endif
 
 namespace nix {
 
-std::string renderMarkdownToTerminal(std::string_view markdown)
+#if HAVE_LOWDOWN
+static std::string doRenderMarkdownToTerminal(std::string_view markdown)
 {
     int windowWidth = getWindowSize().second;
 
-    struct lowdown_opts opts {
+    struct lowdown_opts opts
+    {
         .type = LOWDOWN_TERM,
         .maxdepth = 20,
         .cols = (size_t) std::max(windowWidth - 5, 60),
@@ -46,7 +52,22 @@ std::string renderMarkdownToTerminal(std::string_view markdown)
     if (!rndr_res)
         throw Error("allocation error while rendering Markdown");
 
-    return filterANSIEscapes(std::string(buf->data, buf->size), !shouldANSI());
+    return filterANSIEscapes(std::string(buf->data, buf->size), !isTTY());
 }
 
+std::string renderMarkdownToTerminal(std::string_view markdown)
+{
+    if (auto e = getEnv("_NIX_TEST_RAW_MARKDOWN"); e && *e == "1")
+        return std::string(markdown);
+    else
+        return doRenderMarkdownToTerminal(markdown);
 }
+
+#else
+std::string renderMarkdownToTerminal(std::string_view markdown)
+{
+    return std::string(markdown);
+}
+#endif
+
+} // namespace nix

src/libcmd/markdown.hh

@@ -1,7 +1,17 @@
-#include "types.hh"
+#pragma once
+///@file
+
+#include <string_view>
 
 namespace nix {
 
+/**
+ * Render the given Markdown text to the terminal.
+ *
+ * If Nix is compiled without Markdown support, this function will return the input text as-is.
+ *
+ * The renderer takes into account the terminal width, and wraps text accordingly.
+ */
 std::string renderMarkdownToTerminal(std::string_view markdown);
 
 }

src/libcmd/meson.build

@@ -0,0 +1,130 @@
+project('nix-cmd', 'cpp',
+  version : files('.version'),
+  default_options : [
+    'cpp_std=c++2a',
+    # TODO(Qyriad): increase the warning level
+    'warning_level=1',
+    'debug=true',
+    'optimization=2',
+    'errorlogs=true', # Please print logs for tests that fail
+  ],
+  meson_version : '>= 1.1',
+  license : 'LGPL-2.1-or-later',
+)
+
+cxx = meson.get_compiler('cpp')
+
+subdir('build-utils-meson/deps-lists')
+
+configdata = configuration_data()
+
+deps_private_maybe_subproject = [
+]
+deps_public_maybe_subproject = [
+  dependency('nix-util'),
+  dependency('nix-store'),
+  dependency('nix-fetchers'),
+  dependency('nix-expr'),
+  dependency('nix-flake'),
+  dependency('nix-main'),
+]
+subdir('build-utils-meson/subprojects')
+
+subdir('build-utils-meson/threads')
+
+nlohmann_json = dependency('nlohmann_json', version : '>= 3.9')
+deps_public += nlohmann_json
+
+lowdown = dependency('lowdown', version : '>= 0.9.0', required : get_option('markdown'))
+deps_private += lowdown
+configdata.set('HAVE_LOWDOWN', lowdown.found().to_int())
+
+readline_flavor = get_option('readline-flavor')
+if readline_flavor == 'editline'
+  editline = dependency('libeditline', 'editline', version : '>=1.14')
+  deps_private += editline
+elif readline_flavor == 'readline'
+  readline = dependency('readline')
+  deps_private += readline
+  configdata.set(
+    'USE_READLINE',
+    1,
+    description: 'Use readline instead of editline',
+  )
+else
+  error('illegal editline flavor', readline_flavor)
+endif
+
+config_h = configure_file(
+  configuration : configdata,
+  output : 'config-cmd.hh',
+)
+
+add_project_arguments(
+  # TODO(Qyriad): Yes this is how the autoconf+Make system did it.
+  # It would be nice for our headers to be idempotent instead.
+  '-include', 'config-util.hh',
+  '-include', 'config-store.hh',
+  # '-include', 'config-fetchers.h',
+  '-include', 'config-expr.hh',
+  '-include', 'config-main.hh',
+  '-include', 'config-cmd.hh',
+  language : 'cpp',
+)
+
+subdir('build-utils-meson/diagnostics')
+
+sources = files(
+  'built-path.cc',
+  'command-installable-value.cc',
+  'command.cc',
+  'common-eval-args.cc',
+  'editor-for.cc',
+  'installable-attr-path.cc',
+  'installable-derived-path.cc',
+  'installable-flake.cc',
+  'installable-value.cc',
+  'installables.cc',
+  'legacy.cc',
+  'markdown.cc',
+  'misc-store-flags.cc',
+  'network-proxy.cc',
+  'repl-interacter.cc',
+  'repl.cc',
+)
+
+include_dirs = [include_directories('.')]
+
+headers = [config_h] + files(
+  'built-path.hh',
+  'command-installable-value.hh',
+  'command.hh',
+  'common-eval-args.hh',
+  'compatibility-settings.hh',
+  'editor-for.hh',
+  'installable-attr-path.hh',
+  'installable-derived-path.hh',
+  'installable-flake.hh',
+  'installable-value.hh',
+  'installables.hh',
+  'legacy.hh',
+  'markdown.hh',
+  'misc-store-flags.hh',
+  'network-proxy.hh',
+  'repl-interacter.hh',
+  'repl.hh',
+)
+
+this_library = library(
+  'nixcmd',
+  sources,
+  dependencies : deps_public + deps_private + deps_other,
+  prelink : true, # For C++ static initializers
+  install : true,
+)
+
+install_headers(headers, subdir : 'nix', preserve_path : true)
+
+libraries_private = []
+
+subdir('build-utils-meson/export')

src/libcmd/meson.options

@@ -0,0 +1,15 @@
+# vim: filetype=meson
+
+option(
+  'markdown',
+  type: 'feature',
+  description: 'Enable Markdown rendering in the Nix binary (requires lowdown)',
+)
+
+option(
+  'readline-flavor',
+  type : 'combo',
+  choices : ['editline', 'readline'],
+  value : 'editline',
+  description : 'Which library to use for nice line editing with the Nix language REPL',
+)

src/libcmd/misc-store-flags.cc

@@ -0,0 +1,135 @@
+#include "misc-store-flags.hh"
+
+namespace nix::flag
+{
+
+static void hashFormatCompleter(AddCompletions & completions, size_t index, std::string_view prefix)
+{
+    for (auto & format : hashFormats) {
+        if (hasPrefix(format, prefix)) {
+            completions.add(format);
+        }
+    }
+}
+
+Args::Flag hashFormatWithDefault(std::string && longName, HashFormat * hf)
+{
+    assert(*hf == nix::HashFormat::SRI);
+    return Args::Flag {
+            .longName = std::move(longName),
+            .description = "Hash format (`base16`, `nix32`, `base64`, `sri`). Default: `sri`.",
+            .labels = {"hash-format"},
+            .handler = {[hf](std::string s) {
+                *hf = parseHashFormat(s);
+            }},
+            .completer = hashFormatCompleter,
+    };
+}
+
+Args::Flag hashFormatOpt(std::string && longName, std::optional<HashFormat> * ohf)
+{
+    return Args::Flag {
+            .longName = std::move(longName),
+            .description = "Hash format (`base16`, `nix32`, `base64`, `sri`).",
+            .labels = {"hash-format"},
+            .handler = {[ohf](std::string s) {
+                *ohf = std::optional<HashFormat>{parseHashFormat(s)};
+            }},
+            .completer = hashFormatCompleter,
+    };
+}
+
+static void hashAlgoCompleter(AddCompletions & completions, size_t index, std::string_view prefix)
+{
+    for (auto & algo : hashAlgorithms)
+        if (hasPrefix(algo, prefix))
+            completions.add(algo);
+}
+
+Args::Flag hashAlgo(std::string && longName, HashAlgorithm * ha)
+{
+    return Args::Flag {
+            .longName = std::move(longName),
+            .description = "Hash algorithm (`md5`, `sha1`, `sha256`, or `sha512`).",
+            .labels = {"hash-algo"},
+            .handler = {[ha](std::string s) {
+                *ha = parseHashAlgo(s);
+            }},
+            .completer = hashAlgoCompleter,
+    };
+}
+
+Args::Flag hashAlgoOpt(std::string && longName, std::optional<HashAlgorithm> * oha)
+{
+    return Args::Flag {
+            .longName = std::move(longName),
+            .description = "Hash algorithm (`md5`, `sha1`, `sha256`, or `sha512`). Can be omitted for SRI hashes.",
+            .labels = {"hash-algo"},
+            .handler = {[oha](std::string s) {
+                *oha = std::optional<HashAlgorithm>{parseHashAlgo(s)};
+            }},
+            .completer = hashAlgoCompleter,
+    };
+}
+
+Args::Flag fileIngestionMethod(FileIngestionMethod * method)
+{
+    return Args::Flag {
+        .longName  = "mode",
+        // FIXME indentation carefully made for context, this is messed up.
+        .description = R"(
+    How to compute the hash of the input.
+    One of:
+
+    - `nar` (the default):
+      Serialises the input as a
+      [Nix Archive](@docroot@/store/file-system-object/content-address.md#serial-nix-archive)
+      and passes that to the hash function.
+
+    - `flat`:
+      Assumes that the input is a single file and
+      [directly passes](@docroot@/store/file-system-object/content-address.md#serial-flat)
+      it to the hash function.
+        )",
+        .labels = {"file-ingestion-method"},
+        .handler = {[method](std::string s) {
+            *method = parseFileIngestionMethod(s);
+        }},
+    };
+}
+
+Args::Flag contentAddressMethod(ContentAddressMethod * method)
+{
+    return Args::Flag {
+        .longName  = "mode",
+        // FIXME indentation carefully made for context, this is messed up.
+        .description = R"(
+    How to compute the content-address of the store object.
+    One of:
+
+    - [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive)
+      (the default):
+      Serialises the input as a
+      [Nix Archive](@docroot@/store/file-system-object/content-address.md#serial-nix-archive)
+      and passes that to the hash function.
+
+    - [`flat`](@docroot@/store/store-object/content-address.md#method-flat):
+      Assumes that the input is a single file and
+      [directly passes](@docroot@/store/file-system-object/content-address.md#serial-flat)
+      it to the hash function.
+
+    - [`text`](@docroot@/store/store-object/content-address.md#method-text):
+      Like `flat`, but used for
+      [derivations](@docroot@/glossary.md#store-derivation) serialized in store object and
+      [`builtins.toFile`](@docroot@/language/builtins.html#builtins-toFile).
+      For advanced use-cases only;
+      for regular usage prefer `nar` and `flat`.
+        )",
+        .labels = {"content-address-method"},
+        .handler = {[method](std::string s) {
+            *method = ContentAddressMethod::parse(s);
+        }},
+    };
+}
+
+}

src/libcmd/misc-store-flags.hh

@@ -0,0 +1,21 @@
+#include "args.hh"
+#include "content-address.hh"
+
+namespace nix::flag {
+
+Args::Flag hashAlgo(std::string && longName, HashAlgorithm * ha);
+static inline Args::Flag hashAlgo(HashAlgorithm * ha)
+{
+    return hashAlgo("hash-algo", ha);
+}
+Args::Flag hashAlgoOpt(std::string && longName, std::optional<HashAlgorithm> * oha);
+Args::Flag hashFormatWithDefault(std::string && longName, HashFormat * hf);
+Args::Flag hashFormatOpt(std::string && longName, std::optional<HashFormat> * ohf);
+static inline Args::Flag hashAlgoOpt(std::optional<HashAlgorithm> * oha)
+{
+    return hashAlgoOpt("hash-algo", oha);
+}
+Args::Flag fileIngestionMethod(FileIngestionMethod * method);
+Args::Flag contentAddressMethod(ContentAddressMethod * method);
+
+}

src/libcmd/network-proxy.cc

@@ -0,0 +1,50 @@
+#include "network-proxy.hh"
+
+#include <algorithm>
+
+#include "environment-variables.hh"
+
+namespace nix {
+
+static const StringSet lowercaseVariables{"http_proxy", "https_proxy", "ftp_proxy", "all_proxy", "no_proxy"};
+
+static StringSet getAllVariables()
+{
+    StringSet variables = lowercaseVariables;
+    for (const auto & variable : lowercaseVariables) {
+        std::string upperVariable;
+        std::transform(
+            variable.begin(), variable.end(), upperVariable.begin(), [](unsigned char c) { return std::toupper(c); });
+        variables.insert(std::move(upperVariable));
+    }
+    return variables;
+}
+
+const StringSet networkProxyVariables = getAllVariables();
+
+static StringSet getExcludingNoProxyVariables()
+{
+    static const StringSet excludeVariables{"no_proxy", "NO_PROXY"};
+    StringSet variables;
+    std::set_difference(
+        networkProxyVariables.begin(),
+        networkProxyVariables.end(),
+        excludeVariables.begin(),
+        excludeVariables.end(),
+        std::inserter(variables, variables.begin()));
+    return variables;
+}
+
+static const StringSet excludingNoProxyVariables = getExcludingNoProxyVariables();
+
+bool haveNetworkProxyConnection()
+{
+    for (const auto & variable : excludingNoProxyVariables) {
+        if (getEnv(variable).has_value()) {
+            return true;
+        }
+    }
+    return false;
+}
+
+}

src/libcmd/network-proxy.hh

@@ -0,0 +1,22 @@
+#pragma once
+///@file
+
+#include "types.hh"
+
+namespace nix {
+
+/**
+ * Environment variables relating to network proxying. These are used by
+ * a few misc commands.
+ *
+ * See the Environment section of https://curl.se/docs/manpage.html for details.
+ */
+extern const StringSet networkProxyVariables;
+
+/**
+ * Heuristically check if there is a proxy connection by checking for defined
+ * proxy variables.
+ */
+bool haveNetworkProxyConnection();
+
+}

src/libcmd/nix-cmd.pc.in

@@ -6,4 +6,4 @@ Name: Nix
 Description: Nix Package Manager
 Version: @PACKAGE_VERSION@
 Libs: -L${libdir} -lnixcmd
-Cflags: -I${includedir}/nix -std=c++17
+Cflags: -I${includedir}/nix -std=c++2a

src/libcmd/package.nix

@@ -0,0 +1,104 @@
+{ lib
+, stdenv
+, mkMesonDerivation
+, releaseTools
+
+, meson
+, ninja
+, pkg-config
+
+, nix-util
+, nix-store
+, nix-fetchers
+, nix-expr
+, nix-flake
+, nix-main
+, editline
+, readline
+, lowdown
+, nlohmann_json
+
+# Configuration Options
+
+, version
+
+# Whether to enable Markdown rendering in the Nix binary.
+, enableMarkdown ? !stdenv.hostPlatform.isWindows
+
+# Which interactive line editor library to use for Nix's repl.
+#
+# Currently supported choices are:
+#
+# - editline (default)
+# - readline
+, readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline"
+}:
+
+let
+  inherit (lib) fileset;
+in
+
+mkMesonDerivation (finalAttrs: {
+  pname = "nix-cmd";
+  inherit version;
+
+  workDir = ./.;
+  fileset = fileset.unions [
+    ../../build-utils-meson
+    ./build-utils-meson
+    ../../.version
+    ./.version
+    ./meson.build
+    ./meson.options
+    (fileset.fileFilter (file: file.hasExt "cc") ./.)
+    (fileset.fileFilter (file: file.hasExt "hh") ./.)
+  ];
+
+  outputs = [ "out" "dev" ];
+
+  nativeBuildInputs = [
+    meson
+    ninja
+    pkg-config
+  ];
+
+  buildInputs = [
+    ({ inherit editline readline; }.${readlineFlavor})
+  ] ++ lib.optional enableMarkdown lowdown;
+
+  propagatedBuildInputs = [
+    nix-util
+    nix-store
+    nix-fetchers
+    nix-expr
+    nix-flake
+    nix-main
+    nlohmann_json
+  ];
+
+  preConfigure =
+    # "Inline" .version so it's not a symlink, and includes the suffix.
+    # Do the meson utils, without modification.
+    ''
+      chmod u+w ./.version
+      echo ${version} > ../../.version
+    '';
+
+  mesonFlags = [
+    (lib.mesonEnable "markdown" enableMarkdown)
+    (lib.mesonOption "readline-flavor" readlineFlavor)
+  ];
+
+  env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) {
+    LDFLAGS = "-fuse-ld=gold";
+  };
+
+  separateDebugInfo = !stdenv.hostPlatform.isStatic;
+
+  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
+
+  meta = {
+    platforms = lib.platforms.unix ++ lib.platforms.windows;
+  };
+
+})

src/libcmd/repl-interacter.cc

@@ -0,0 +1,206 @@
+#include <cstdio>
+
+#ifdef USE_READLINE
+#include <readline/history.h>
+#include <readline/readline.h>
+#else
+// editline < 1.15.2 don't wrap their API for C++ usage
+// (added in https://github.com/troglobit/editline/commit/91398ceb3427b730995357e9d120539fb9bb7461).
+// This results in linker errors due to to name-mangling of editline C symbols.
+// For compatibility with these versions, we wrap the API here
+// (wrapping multiple times on newer versions is no problem).
+extern "C" {
+#include <editline.h>
+}
+#endif
+
+#include "signals.hh"
+#include "finally.hh"
+#include "repl-interacter.hh"
+#include "file-system.hh"
+#include "repl.hh"
+#include "environment-variables.hh"
+
+namespace nix {
+
+namespace {
+// Used to communicate to NixRepl::getLine whether a signal occurred in ::readline.
+volatile sig_atomic_t g_signal_received = 0;
+
+void sigintHandler(int signo)
+{
+    g_signal_received = signo;
+}
+};
+
+static detail::ReplCompleterMixin * curRepl; // ugly
+
+#ifndef USE_READLINE
+static char * completionCallback(char * s, int * match)
+{
+    auto possible = curRepl->completePrefix(s);
+    if (possible.size() == 1) {
+        *match = 1;
+        auto * res = strdup(possible.begin()->c_str() + strlen(s));
+        if (!res)
+            throw Error("allocation failure");
+        return res;
+    } else if (possible.size() > 1) {
+        auto checkAllHaveSameAt = [&](size_t pos) {
+            auto & first = *possible.begin();
+            for (auto & p : possible) {
+                if (p.size() <= pos || p[pos] != first[pos])
+                    return false;
+            }
+            return true;
+        };
+        size_t start = strlen(s);
+        size_t len = 0;
+        while (checkAllHaveSameAt(start + len))
+            ++len;
+        if (len > 0) {
+            *match = 1;
+            auto * res = strdup(std::string(*possible.begin(), start, len).c_str());
+            if (!res)
+                throw Error("allocation failure");
+            return res;
+        }
+    }
+
+    *match = 0;
+    return nullptr;
+}
+
+static int listPossibleCallback(char * s, char *** avp)
+{
+    auto possible = curRepl->completePrefix(s);
+
+    if (possible.size() > (std::numeric_limits<int>::max() / sizeof(char *)))
+        throw Error("too many completions");
+
+    int ac = 0;
+    char ** vp = nullptr;
+
+    auto check = [&](auto * p) {
+        if (!p) {
+            if (vp) {
+                while (--ac >= 0)
+                    free(vp[ac]);
+                free(vp);
+            }
+            throw Error("allocation failure");
+        }
+        return p;
+    };
+
+    vp = check((char **) malloc(possible.size() * sizeof(char *)));
+
+    for (auto & p : possible)
+        vp[ac++] = check(strdup(p.c_str()));
+
+    *avp = vp;
+
+    return ac;
+}
+#endif
+
+ReadlineLikeInteracter::Guard ReadlineLikeInteracter::init(detail::ReplCompleterMixin * repl)
+{
+    // Allow nix-repl specific settings in .inputrc
+    rl_readline_name = "nix-repl";
+    try {
+        createDirs(dirOf(historyFile));
+    } catch (SystemError & e) {
+        logWarning(e.info());
+    }
+#ifndef USE_READLINE
+    el_hist_size = 1000;
+#endif
+    read_history(historyFile.c_str());
+    auto oldRepl = curRepl;
+    curRepl = repl;
+    Guard restoreRepl([oldRepl] { curRepl = oldRepl; });
+#ifndef USE_READLINE
+    rl_set_complete_func(completionCallback);
+    rl_set_list_possib_func(listPossibleCallback);
+#endif
+    return restoreRepl;
+}
+
+static constexpr const char * promptForType(ReplPromptType promptType)
+{
+    switch (promptType) {
+    case ReplPromptType::ReplPrompt:
+        return "nix-repl> ";
+    case ReplPromptType::ContinuationPrompt:
+        return "          ";
+    }
+    assert(false);
+}
+
+bool ReadlineLikeInteracter::getLine(std::string & input, ReplPromptType promptType)
+{
+#ifndef _WIN32 // TODO use more signals.hh for this
+    struct sigaction act, old;
+    sigset_t savedSignalMask, set;
+
+    auto setupSignals = [&]() {
+        act.sa_handler = sigintHandler;
+        sigfillset(&act.sa_mask);
+        act.sa_flags = 0;
+        if (sigaction(SIGINT, &act, &old))
+            throw SysError("installing handler for SIGINT");
+
+        sigemptyset(&set);
+        sigaddset(&set, SIGINT);
+        if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask))
+            throw SysError("unblocking SIGINT");
+    };
+    auto restoreSignals = [&]() {
+        if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr))
+            throw SysError("restoring signals");
+
+        if (sigaction(SIGINT, &old, 0))
+            throw SysError("restoring handler for SIGINT");
+    };
+
+    setupSignals();
+#endif
+    char * s = readline(promptForType(promptType));
+    Finally doFree([&]() { free(s); });
+#ifndef _WIN32 // TODO use more signals.hh for this
+    restoreSignals();
+#endif
+
+    if (g_signal_received) {
+        g_signal_received = 0;
+        input.clear();
+        return true;
+    }
+
+    // editline doesn't echo the input to the output when non-interactive, unlike readline
+    // this results in a different behavior when running tests. The echoing is
+    // quite useful for reading the test output, so we add it here.
+    if (auto e = getEnv("_NIX_TEST_REPL_ECHO"); s && e && *e == "1")
+    {
+#ifndef USE_READLINE
+        // This is probably not right for multi-line input, but we don't use that
+        // in the characterisation tests, so it's fine.
+        std::cout << promptForType(promptType) << s << std::endl;
+#endif
+    }
+
+    if (!s)
+        return false;
+    input += s;
+    input += '\n';
+
+    return true;
+}
+
+ReadlineLikeInteracter::~ReadlineLikeInteracter()
+{
+    write_history(historyFile.c_str());
+}
+
+};

src/libcmd/repl-interacter.hh

@@ -0,0 +1,48 @@
+#pragma once
+/// @file
+
+#include "finally.hh"
+#include "types.hh"
+#include <functional>
+#include <string>
+
+namespace nix {
+
+namespace detail {
+/** Provides the completion hooks for the repl, without exposing its complete
+ * internals. */
+struct ReplCompleterMixin {
+    virtual StringSet completePrefix(const std::string & prefix) = 0;
+};
+};
+
+enum class ReplPromptType {
+    ReplPrompt,
+    ContinuationPrompt,
+};
+
+class ReplInteracter
+{
+public:
+    using Guard = Finally<std::function<void()>>;
+
+    virtual Guard init(detail::ReplCompleterMixin * repl) = 0;
+    /** Returns a boolean of whether the interacter got EOF */
+    virtual bool getLine(std::string & input, ReplPromptType promptType) = 0;
+    virtual ~ReplInteracter(){};
+};
+
+class ReadlineLikeInteracter : public virtual ReplInteracter
+{
+    std::string historyFile;
+public:
+    ReadlineLikeInteracter(std::string historyFile)
+        : historyFile(historyFile)
+    {
+    }
+    virtual Guard init(detail::ReplCompleterMixin * repl) override;
+    virtual bool getLine(std::string & input, ReplPromptType promptType) override;
+    virtual ~ReadlineLikeInteracter() override;
+};
+
+};

src/libcmd/repl.hh

@@ -0,0 +1,45 @@
+#pragma once
+///@file
+
+#include "eval.hh"
+
+namespace nix {
+
+struct AbstractNixRepl
+{
+    ref<EvalState> state;
+    Bindings * autoArgs;
+
+    AbstractNixRepl(ref<EvalState> state)
+        : state(state)
+    { }
+
+    virtual ~AbstractNixRepl()
+    { }
+
+    typedef std::vector<std::pair<Value*,std::string>> AnnotatedValues;
+
+    using RunNix = void(Path program, const Strings & args, const std::optional<std::string> & input);
+
+    /**
+     * @param runNix Function to run the nix CLI to support various
+     * `:<something>` commands. Optional; if not provided,
+     * everything else will still work fine, but those commands won't.
+     */
+    static std::unique_ptr<AbstractNixRepl> create(
+        const LookupPath & lookupPath,
+        nix::ref<Store> store,
+        ref<EvalState> state,
+        std::function<AnnotatedValues()> getValues,
+        RunNix * runNix = nullptr);
+
+    static ReplExitStatus runSimple(
+        ref<EvalState> evalState,
+        const ValMap & extraEnv);
+
+    virtual void initEnv() = 0;
+
+    virtual ReplExitStatus mainLoop() = 0;
+};
+
+}

src/libexpr-c/.version

@@ -0,0 +1 @@
+../../.version

src/libexpr-c/build-utils-meson

@@ -0,0 +1 @@
+../../build-utils-meson

src/libexpr-c/local.mk

@@ -0,0 +1,25 @@
+libraries += libexprc
+
+libexprc_NAME = libnixexprc
+
+libexprc_DIR := $(d)
+
+libexprc_SOURCES := \
+  $(wildcard $(d)/*.cc) \
+
+# Not just for this library itself, but also for downstream libraries using this library
+
+INCLUDE_libexprc := -I $(d)
+libexprc_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libutilc) \
+                     $(INCLUDE_libfetchers) \
+                     $(INCLUDE_libstore) $(INCLUDE_libstorec) \
+                     $(INCLUDE_libexpr) $(INCLUDE_libexprc)
+
+libexprc_LIBS = libutil libutilc libstore libstorec libfetchers libexpr
+
+libexprc_LDFLAGS += $(THREAD_LDFLAGS)
+
+$(eval $(call install-file-in, $(d)/nix-expr-c.pc, $(libdir)/pkgconfig, 0644))
+
+libexprc_FORCE_INSTALL := 1
+

src/libexpr-c/meson.build

@@ -0,0 +1,93 @@
+project('nix-expr-c', 'cpp',
+  version : files('.version'),
+  default_options : [
+    'cpp_std=c++2a',
+    # TODO(Qyriad): increase the warning level
+    'warning_level=1',
+    'debug=true',
+    'optimization=2',
+    'errorlogs=true', # Please print logs for tests that fail
+  ],
+  meson_version : '>= 1.1',
+  license : 'LGPL-2.1-or-later',
+)
+
+cxx = meson.get_compiler('cpp')
+
+subdir('build-utils-meson/deps-lists')
+
+configdata = configuration_data()
+
+deps_private_maybe_subproject = [
+  dependency('nix-util'),
+  dependency('nix-store'),
+  dependency('nix-expr'),
+]
+deps_public_maybe_subproject = [
+  dependency('nix-util-c'),
+  dependency('nix-store-c'),
+]
+subdir('build-utils-meson/subprojects')
+
+subdir('build-utils-meson/threads')
+
+# TODO rename, because it will conflict with downstream projects
+configdata.set_quoted('PACKAGE_VERSION', meson.project_version())
+
+config_h = configure_file(
+  configuration : configdata,
+  output : 'config-expr.h',
+)
+
+add_project_arguments(
+  # TODO(Qyriad): Yes this is how the autoconf+Make system did it.
+  # It would be nice for our headers to be idempotent instead.
+
+  # From C++ libraries, only for internals
+  '-include', 'config-util.hh',
+  '-include', 'config-store.hh',
+  '-include', 'config-expr.hh',
+
+  # From C libraries, for our public, installed headers too
+  '-include', 'config-util.h',
+  '-include', 'config-store.h',
+  '-include', 'config-expr.h',
+  language : 'cpp',
+)
+
+subdir('build-utils-meson/diagnostics')
+
+sources = files(
+  'nix_api_expr.cc',
+  'nix_api_external.cc',
+  'nix_api_value.cc',
+)
+
+include_dirs = [include_directories('.')]
+
+headers = [config_h] + files(
+  'nix_api_expr.h',
+  'nix_api_external.h',
+  'nix_api_value.h',
+)
+
+# TODO move this header to libexpr, maybe don't use it in tests?
+headers += files('nix_api_expr_internal.h')
+
+subdir('build-utils-meson/export-all-symbols')
+
+this_library = library(
+  'nixexprc',
+  sources,
+  dependencies : deps_public + deps_private + deps_other,
+  include_directories : include_dirs,
+  link_args: linker_export_flags,
+  prelink : true, # For C++ static initializers
+  install : true,
+)
+
+install_headers(headers, subdir : 'nix', preserve_path : true)
+
+libraries_private = []
+
+subdir('build-utils-meson/export')

src/libexpr-c/nix-expr-c.pc.in

@@ -0,0 +1,10 @@
+prefix=@prefix@
+libdir=@libdir@
+includedir=@includedir@
+
+Name: Nix
+Description: Nix Language Evaluator - C API
+Version: @PACKAGE_VERSION@
+Requires: nix-store-c
+Libs: -L${libdir} -lnixexprc
+Cflags: -I${includedir}/nix

src/libexpr-c/nix_api_expr.cc

@@ -0,0 +1,213 @@
+#include <cstring>
+#include <stdexcept>
+#include <string>
+
+#include "eval.hh"
+#include "eval-gc.hh"
+#include "globals.hh"
+#include "eval-settings.hh"
+
+#include "nix_api_expr.h"
+#include "nix_api_expr_internal.h"
+#include "nix_api_store.h"
+#include "nix_api_store_internal.h"
+#include "nix_api_util.h"
+#include "nix_api_util_internal.h"
+
+#if HAVE_BOEHMGC
+#  include <mutex>
+#  define GC_INCLUDE_NEW 1
+#  include "gc_cpp.h"
+#endif
+
+nix_err nix_libexpr_init(nix_c_context * context)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    {
+        auto ret = nix_libutil_init(context);
+        if (ret != NIX_OK)
+            return ret;
+    }
+    {
+        auto ret = nix_libstore_init(context);
+        if (ret != NIX_OK)
+            return ret;
+    }
+    try {
+        nix::initGC();
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_expr_eval_from_string(
+    nix_c_context * context, EvalState * state, const char * expr, const char * path, nix_value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        nix::Expr * parsedExpr = state->state.parseExprFromString(expr, state->state.rootPath(nix::CanonPath(path)));
+        state->state.eval(parsedExpr, value->value);
+        state->state.forceValue(value->value, nix::noPos);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, nix_value * arg, nix_value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        state->state.callFunction(fn->value, arg->value, value->value, nix::noPos);
+        state->state.forceValue(value->value, nix::noPos);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_value_call_multi(nix_c_context * context, EvalState * state, nix_value * fn, size_t nargs, nix_value ** args, nix_value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        state->state.callFunction(fn->value, nargs, (nix::Value * *)args, value->value, nix::noPos);
+        state->state.forceValue(value->value, nix::noPos);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_value_force(nix_c_context * context, EvalState * state, nix_value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        state->state.forceValue(value->value, nix::noPos);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        state->state.forceValueDeep(value->value);
+    }
+    NIXC_CATCH_ERRS
+}
+
+EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c, Store * store)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        nix::Strings lookupPath;
+        if (lookupPath_c != nullptr)
+            for (size_t i = 0; lookupPath_c[i] != nullptr; i++)
+                lookupPath.push_back(lookupPath_c[i]);
+
+        void * p = ::operator new(
+            sizeof(EvalState),
+            static_cast<std::align_val_t>(alignof(EvalState)));
+        auto * p2 = static_cast<EvalState *>(p);
+        new (p) EvalState {
+            .fetchSettings = nix::fetchers::Settings{},
+            .settings = nix::EvalSettings{
+                nix::settings.readOnlyMode,
+            },
+            .state = nix::EvalState(
+                nix::LookupPath::parse(lookupPath),
+                store->ptr,
+                p2->fetchSettings,
+                p2->settings),
+        };
+        loadConfFile(p2->settings);
+        return p2;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+void nix_state_free(EvalState * state)
+{
+    delete state;
+}
+
+#if HAVE_BOEHMGC
+std::unordered_map<
+    const void *,
+    unsigned int,
+    std::hash<const void *>,
+    std::equal_to<const void *>,
+    traceable_allocator<std::pair<const void * const, unsigned int>>>
+    nix_refcounts;
+
+std::mutex nix_refcount_lock;
+
+nix_err nix_gc_incref(nix_c_context * context, const void * p)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        std::scoped_lock lock(nix_refcount_lock);
+        auto f = nix_refcounts.find(p);
+        if (f != nix_refcounts.end()) {
+            f->second++;
+        } else {
+            nix_refcounts[p] = 1;
+        }
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_gc_decref(nix_c_context * context, const void * p)
+{
+
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        std::scoped_lock lock(nix_refcount_lock);
+        auto f = nix_refcounts.find(p);
+        if (f != nix_refcounts.end()) {
+            if (--f->second == 0)
+                nix_refcounts.erase(f);
+        } else
+            throw std::runtime_error("nix_gc_decref: object was not referenced");
+    }
+    NIXC_CATCH_ERRS
+}
+
+void nix_gc_now()
+{
+    GC_gcollect();
+}
+
+#else
+nix_err nix_gc_incref(nix_c_context * context, const void *)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    return NIX_OK;
+}
+nix_err nix_gc_decref(nix_c_context * context, const void *)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    return NIX_OK;
+}
+void nix_gc_now() {}
+#endif
+
+nix_err nix_value_incref(nix_c_context * context, nix_value *x)
+{
+    return nix_gc_incref(context, (const void *) x);
+}
+nix_err nix_value_decref(nix_c_context * context, nix_value *x)
+{
+    return nix_gc_decref(context, (const void *) x);
+}
+
+void nix_gc_register_finalizer(void * obj, void * cd, void (*finalizer)(void * obj, void * cd))
+{
+#if HAVE_BOEHMGC
+    GC_REGISTER_FINALIZER(obj, finalizer, cd, 0, 0);
+#endif
+}

src/libexpr-c/nix_api_expr.h

@@ -0,0 +1,257 @@
+#ifndef NIX_API_EXPR_H
+#define NIX_API_EXPR_H
+/** @defgroup libexpr libexpr
+ * @brief Bindings to the Nix language evaluator
+ *
+ * See *[Embedding the Nix Evaluator](@ref nix_evaluator_example)* for an example.
+ * @{
+ */
+/** @file
+ * @brief Main entry for the libexpr C bindings
+ */
+
+#include "nix_api_store.h"
+#include "nix_api_util.h"
+#include <stddef.h>
+
+#ifndef __has_c_attribute
+#  define __has_c_attribute(x) 0
+#endif
+
+#if __has_c_attribute(deprecated)
+#  define NIX_DEPRECATED(msg) [[deprecated(msg)]]
+#else
+#  define NIX_DEPRECATED(msg)
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+// cffi start
+
+// Type definitions
+/**
+ * @brief Represents a state of the Nix language evaluator.
+ *
+ * Multiple states can be created for multi-threaded
+ * operation.
+ * @struct EvalState
+ * @see nix_state_create
+ */
+typedef struct EvalState EvalState; // nix::EvalState
+
+/** @brief A Nix language value, or thunk that may evaluate to a value.
+ *
+ * Values are the primary objects manipulated in the Nix language.
+ * They are considered to be immutable from a user's perspective, but the process of evaluating a value changes its
+ * ValueType if it was a thunk. After a value has been evaluated, its ValueType does not change.
+ *
+ * Evaluation in this context refers to the process of evaluating a single value object, also called "forcing" the
+ * value; see `nix_value_force`.
+ *
+ * The evaluator manages its own memory, but your use of the C API must follow the reference counting rules.
+ *
+ * @see value_manip
+ * @see nix_value_incref, nix_value_decref
+ */
+typedef struct nix_value nix_value;
+NIX_DEPRECATED("use nix_value instead") typedef nix_value Value;
+
+// Function prototypes
+/**
+ * @brief Initialize the Nix language evaluator.
+ *
+ * This function must be called at least once,
+ * at some point before constructing a EvalState for the first time.
+ * This function can be called multiple times, and is idempotent.
+ *
+ * @param[out] context Optional, stores error information
+ * @return NIX_OK if the initialization was successful, an error code otherwise.
+ */
+nix_err nix_libexpr_init(nix_c_context * context);
+
+/**
+ * @brief Parses and evaluates a Nix expression from a string.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] state The state of the evaluation.
+ * @param[in] expr The Nix expression to parse.
+ * @param[in] path The file path to associate with the expression.
+ * This is required for expressions that contain relative paths (such as `./.`) that are resolved relative to the given
+ * directory.
+ * @param[out] value The result of the evaluation. You must allocate this
+ * yourself.
+ * @return NIX_OK if the evaluation was successful, an error code otherwise.
+ */
+nix_err nix_expr_eval_from_string(
+    nix_c_context * context, EvalState * state, const char * expr, const char * path, nix_value * value);
+
+/**
+ * @brief Calls a Nix function with an argument.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] state The state of the evaluation.
+ * @param[in] fn The Nix function to call.
+ * @param[in] arg The argument to pass to the function.
+ * @param[out] value The result of the function call.
+ * @return NIX_OK if the function call was successful, an error code otherwise.
+ * @see nix_init_apply() for a similar function that does not performs the call immediately, but stores it as a thunk.
+ *      Note the different argument order.
+ */
+nix_err nix_value_call(nix_c_context * context, EvalState * state, nix_value * fn, nix_value * arg, nix_value * value);
+
+/**
+ * @brief Calls a Nix function with multiple arguments.
+ *
+ * Technically these are functions that return functions. It is common for Nix
+ * functions to be curried, so this function is useful for calling them.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] state The state of the evaluation.
+ * @param[in] fn The Nix function to call.
+ * @param[in] nargs The number of arguments.
+ * @param[in] args The arguments to pass to the function.
+ * @param[out] value The result of the function call.
+ *
+ * @see nix_value_call     For the single argument primitive.
+ * @see NIX_VALUE_CALL           For a macro that wraps this function for convenience.
+ */
+nix_err nix_value_call_multi(
+    nix_c_context * context, EvalState * state, nix_value * fn, size_t nargs, nix_value ** args, nix_value * value);
+
+/**
+ * @brief Calls a Nix function with multiple arguments.
+ *
+ * Technically these are functions that return functions. It is common for Nix
+ * functions to be curried, so this function is useful for calling them.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] state The state of the evaluation.
+ * @param[out] value The result of the function call.
+ * @param[in] fn The Nix function to call.
+ * @param[in] args The arguments to pass to the function.
+ *
+ * @see nix_value_call_multi
+ */
+#define NIX_VALUE_CALL(context, state, value, fn, ...)                      \
+    do {                                                                    \
+        nix_value * args_array[] = {__VA_ARGS__};                           \
+        size_t nargs = sizeof(args_array) / sizeof(args_array[0]);          \
+        nix_value_call_multi(context, state, fn, nargs, args_array, value); \
+    } while (0)
+
+/**
+ * @brief Forces the evaluation of a Nix value.
+ *
+ * The Nix interpreter is lazy, and not-yet-evaluated values can be
+ * of type NIX_TYPE_THUNK instead of their actual value.
+ *
+ * This function mutates such a `nix_value`, so that, if successful, it has its final type.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] state The state of the evaluation.
+ * @param[in,out] value The Nix value to force.
+ * @post value is not of type NIX_TYPE_THUNK
+ * @return NIX_OK if the force operation was successful, an error code
+ * otherwise.
+ */
+nix_err nix_value_force(nix_c_context * context, EvalState * state, nix_value * value);
+
+/**
+ * @brief Forces the deep evaluation of a Nix value.
+ *
+ * Recursively calls nix_value_force
+ *
+ * @see nix_value_force
+ * @warning Calling this function on a recursive data structure will cause a
+ * stack overflow.
+ * @param[out] context Optional, stores error information
+ * @param[in] state The state of the evaluation.
+ * @param[in,out] value The Nix value to force.
+ * @return NIX_OK if the deep force operation was successful, an error code
+ * otherwise.
+ */
+nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_value * value);
+
+/**
+ * @brief Create a new Nix language evaluator state.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] lookupPath Null-terminated array of strings corresponding to entries in NIX_PATH.
+ * @param[in] store The Nix store to use.
+ * @return A new Nix state or NULL on failure.
+ */
+EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath, Store * store);
+
+/**
+ * @brief Frees a Nix state.
+ *
+ * Does not fail.
+ *
+ * @param[in] state The state to free.
+ */
+void nix_state_free(EvalState * state);
+
+/** @addtogroup GC
+ * @brief Reference counting and garbage collector operations
+ *
+ * The Nix language evaluator uses a garbage collector. To ease C interop, we implement
+ * a reference counting scheme, where objects will be deallocated
+ * when there are no references from the Nix side, and the reference count kept
+ * by the C API reaches `0`.
+ *
+ * Functions returning a garbage-collected object will automatically increase
+ * the refcount for you. You should make sure to call `nix_gc_decref` when
+ * you're done with a value returned by the evaluator.
+ * @{
+ */
+
+// TODO: Deprecate nix_gc_incref in favor of the type-specific reference counting functions?
+//       e.g. nix_value_incref.
+//       It gives implementors more flexibility, and adds safety, so that generated
+//       bindings can be used without fighting the host type system (where applicable).
+/**
+ * @brief Increment the garbage collector reference counter for the given object.
+ *
+ * The Nix language evaluator C API keeps track of alive objects by reference counting.
+ * When you're done with a refcounted pointer, call nix_gc_decref().
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] object The object to keep alive
+ */
+nix_err nix_gc_incref(nix_c_context * context, const void * object);
+/**
+ * @brief Decrement the garbage collector reference counter for the given object
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] object The object to stop referencing
+ */
+nix_err nix_gc_decref(nix_c_context * context, const void * object);
+
+/**
+ * @brief Trigger the garbage collector manually
+ *
+ * You should not need to do this, but it can be useful for debugging.
+ */
+void nix_gc_now();
+
+/**
+ * @brief Register a callback that gets called when the object is garbage
+ * collected.
+ * @note Objects can only have a single finalizer. This function overwrites existing values
+ * silently.
+ * @param[in] obj the object to watch
+ * @param[in] cd the data to pass to the finalizer
+ * @param[in] finalizer the callback function, called with obj and cd
+ */
+void nix_gc_register_finalizer(void * obj, void * cd, void (*finalizer)(void * obj, void * cd));
+
+/** @} */
+// cffi end
+#ifdef __cplusplus
+}
+#endif
+
+/** @} */
+
+#endif // NIX_API_EXPR_H

src/libexpr-c/nix_api_expr_internal.h

@@ -0,0 +1,53 @@
+#ifndef NIX_API_EXPR_INTERNAL_H
+#define NIX_API_EXPR_INTERNAL_H
+
+#include "fetch-settings.hh"
+#include "eval.hh"
+#include "eval-settings.hh"
+#include "attr-set.hh"
+#include "nix_api_value.h"
+
+struct EvalState
+{
+    nix::fetchers::Settings fetchSettings;
+    nix::EvalSettings settings;
+    nix::EvalState state;
+};
+
+struct BindingsBuilder
+{
+    nix::BindingsBuilder builder;
+};
+
+struct ListBuilder
+{
+    nix::ListBuilder builder;
+};
+
+struct nix_value
+{
+    nix::Value value;
+};
+
+struct nix_string_return
+{
+    std::string str;
+};
+
+struct nix_printer
+{
+    std::ostream & s;
+};
+
+struct nix_string_context
+{
+    nix::NixStringContext & ctx;
+};
+
+struct nix_realised_string
+{
+    std::string str;
+    std::vector<StorePath> storePaths;
+};
+
+#endif // NIX_API_EXPR_INTERNAL_H

src/libexpr-c/nix_api_external.cc

@@ -0,0 +1,198 @@
+#include "attr-set.hh"
+#include "config.hh"
+#include "eval.hh"
+#include "globals.hh"
+#include "value.hh"
+
+#include "nix_api_expr.h"
+#include "nix_api_expr_internal.h"
+#include "nix_api_external.h"
+#include "nix_api_util.h"
+#include "nix_api_util_internal.h"
+#include "nix_api_value.h"
+#include "value/context.hh"
+
+#include <nlohmann/json.hpp>
+
+#if HAVE_BOEHMGC
+# include "gc/gc.h"
+# define GC_INCLUDE_NEW 1
+# include "gc_cpp.h"
+#endif
+
+void nix_set_string_return(nix_string_return * str, const char * c)
+{
+    str->str = c;
+}
+
+nix_err nix_external_print(nix_c_context * context, nix_printer * printer, const char * c)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        printer->s << c;
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_external_add_string_context(nix_c_context * context, nix_string_context * ctx, const char * c)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto r = nix::NixStringContextElem::parse(c);
+        ctx->ctx.insert(r);
+    }
+    NIXC_CATCH_ERRS
+}
+
+class NixCExternalValue : public nix::ExternalValueBase
+{
+    NixCExternalValueDesc & desc;
+    void * v;
+
+public:
+    NixCExternalValue(NixCExternalValueDesc & desc, void * v)
+        : desc(desc)
+        , v(v){};
+    void * get_ptr()
+    {
+        return v;
+    }
+    /**
+     * Print out the value
+     */
+    virtual std::ostream & print(std::ostream & str) const override
+    {
+        nix_printer p{str};
+        desc.print(v, &p);
+        return str;
+    }
+
+    /**
+     * Return a simple string describing the type
+     */
+    virtual std::string showType() const override
+    {
+        nix_string_return res;
+        desc.showType(v, &res);
+        return std::move(res.str);
+    }
+
+    /**
+     * Return a string to be used in builtins.typeOf
+     */
+    virtual std::string typeOf() const override
+    {
+        nix_string_return res;
+        desc.typeOf(v, &res);
+        return std::move(res.str);
+    }
+
+    /**
+     * Coerce the value to a string.
+     */
+    virtual std::string coerceToString(
+        nix::EvalState & state,
+        const nix::PosIdx & pos,
+        nix::NixStringContext & context,
+        bool copyMore,
+        bool copyToStore) const override
+    {
+        if (!desc.coerceToString) {
+            return nix::ExternalValueBase::coerceToString(state, pos, context, copyMore, copyToStore);
+        }
+        nix_string_context ctx{context};
+        nix_string_return res{""};
+        // todo: pos, errors
+        desc.coerceToString(v, &ctx, copyMore, copyToStore, &res);
+        if (res.str.empty()) {
+            return nix::ExternalValueBase::coerceToString(state, pos, context, copyMore, copyToStore);
+        }
+        return std::move(res.str);
+    }
+
+    /**
+     * Compare to another value of the same type.
+     */
+    virtual bool operator==(const ExternalValueBase & b) const noexcept override
+    {
+        if (!desc.equal) {
+            return false;
+        }
+        auto r = dynamic_cast<const NixCExternalValue *>(&b);
+        if (!r)
+            return false;
+        return desc.equal(v, r->v);
+    }
+
+    /**
+     * Print the value as JSON.
+     */
+    virtual nlohmann::json printValueAsJSON(
+        nix::EvalState & state, bool strict, nix::NixStringContext & context, bool copyToStore = true) const override
+    {
+        if (!desc.printValueAsJSON) {
+            return nix::ExternalValueBase::printValueAsJSON(state, strict, context, copyToStore);
+        }
+        nix_string_context ctx{context};
+        nix_string_return res{""};
+        desc.printValueAsJSON(v, (EvalState *) &state, strict, &ctx, copyToStore, &res);
+        if (res.str.empty()) {
+            return nix::ExternalValueBase::printValueAsJSON(state, strict, context, copyToStore);
+        }
+        return nlohmann::json::parse(res.str);
+    }
+
+    /**
+     * Print the value as XML.
+     */
+    virtual void printValueAsXML(
+        nix::EvalState & state,
+        bool strict,
+        bool location,
+        nix::XMLWriter & doc,
+        nix::NixStringContext & context,
+        nix::PathSet & drvsSeen,
+        const nix::PosIdx pos) const override
+    {
+        if (!desc.printValueAsXML) {
+            return nix::ExternalValueBase::printValueAsXML(state, strict, location, doc, context, drvsSeen, pos);
+        }
+        nix_string_context ctx{context};
+        desc.printValueAsXML(
+            v, (EvalState *) &state, strict, location, &doc, &ctx, &drvsSeen,
+            *reinterpret_cast<const uint32_t *>(&pos));
+    }
+
+    virtual ~NixCExternalValue() override{};
+};
+
+ExternalValue * nix_create_external_value(nix_c_context * context, NixCExternalValueDesc * desc, void * v)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto ret = new
+#if HAVE_BOEHMGC
+            (GC)
+#endif
+                NixCExternalValue(*desc, v);
+        nix_gc_incref(nullptr, ret);
+        return (ExternalValue *) ret;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+void * nix_get_external_value_content(nix_c_context * context, ExternalValue * b)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto r = dynamic_cast<NixCExternalValue *>((nix::ExternalValueBase *) b);
+        if (r)
+            return r->get_ptr();
+        return nullptr;
+    }
+    NIXC_CATCH_ERRS_NULL
+}

src/libexpr-c/nix_api_external.h

@@ -0,0 +1,196 @@
+#ifndef NIX_API_EXTERNAL_H
+#define NIX_API_EXTERNAL_H
+/** @ingroup libexpr
+ * @addtogroup Externals
+ * @brief Deal with external values
+ * @{
+ */
+/** @file
+ * @brief libexpr C bindings dealing with external values
+ */
+
+#include "nix_api_expr.h"
+#include "nix_api_util.h"
+#include "nix_api_value.h"
+#include "stdbool.h"
+#include "stddef.h"
+#include "stdint.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+// cffi start
+
+/**
+ * @brief Represents a string owned by the Nix language evaluator.
+ * @see nix_set_owned_string
+ */
+typedef struct nix_string_return nix_string_return;
+/**
+ * @brief Wraps a stream that can output multiple string pieces.
+ */
+typedef struct nix_printer nix_printer;
+/**
+ * @brief A list of string context items
+ */
+typedef struct nix_string_context nix_string_context;
+
+/**
+ * @brief Sets the contents of a nix_string_return
+ *
+ * Copies the passed string.
+ * @param[out] str the nix_string_return to write to
+ * @param[in]  c   The string to copy
+ */
+void nix_set_string_return(nix_string_return * str, const char * c);
+
+/**
+ * Print to the nix_printer
+ *
+ * @param[out] context Optional, stores error information
+ * @param[out] printer The nix_printer to print to
+ * @param[in] str The string to print
+ * @returns NIX_OK if everything worked
+ */
+nix_err nix_external_print(nix_c_context * context, nix_printer * printer, const char * str);
+
+/**
+ * Add string context to the nix_string_context object
+ * @param[out] context Optional, stores error information
+ * @param[out] string_context The nix_string_context to add to
+ * @param[in] c The context string to add
+ * @returns NIX_OK if everything worked
+ */
+nix_err nix_external_add_string_context(nix_c_context * context, nix_string_context * string_context, const char * c);
+
+/**
+ * @brief Definition for a class of external values
+ *
+ * Create and implement one of these, then pass it to nix_create_external_value
+ * Make sure to keep it alive while the external value lives.
+ *
+ * Optional functions can be set to NULL
+ *
+ * @see nix_create_external_value
+ */
+typedef struct NixCExternalValueDesc
+{
+    /**
+     * @brief Called when printing the external value
+     *
+     * @param[in] self the void* passed to nix_create_external_value
+     * @param[out] printer The printer to print to, pass to nix_external_print
+     */
+    void (*print)(void * self, nix_printer * printer);
+    /**
+     * @brief Called on :t
+     * @param[in] self the void* passed to nix_create_external_value
+     * @param[out] res the return value
+     */
+    void (*showType)(void * self, nix_string_return * res);
+    /**
+     * @brief Called on `builtins.typeOf`
+     * @param self the void* passed to nix_create_external_value
+     * @param[out] res the return value
+     */
+    void (*typeOf)(void * self, nix_string_return * res);
+    /**
+     * @brief Called on "${str}" and builtins.toString.
+     *
+     * The latter with coerceMore=true
+     * Optional, the default is to throw an error.
+     * @param[in] self the void* passed to nix_create_external_value
+     * @param[out] c writable string context for the resulting string
+     * @param[in] coerceMore boolean, try to coerce to strings in more cases
+     * instead of throwing an error
+     * @param[in] copyToStore boolean, whether to copy referenced paths to store
+     * or keep them as-is
+     * @param[out] res the return value. Not touching this, or setting it to the
+     * empty string, will make the conversion throw an error.
+     */
+    void (*coerceToString)(
+        void * self, nix_string_context * c, int coerceMore, int copyToStore, nix_string_return * res);
+    /**
+     * @brief Try to compare two external values
+     *
+     * Optional, the default is always false.
+     * If the other object was not a Nix C external value, this comparison will
+     * also return false
+     * @param[in] self the void* passed to nix_create_external_value
+     * @param[in] other the void* passed to the other object's
+     * nix_create_external_value
+     * @returns true if the objects are deemed to be equal
+     */
+    int (*equal)(void * self, void * other);
+    /**
+     * @brief Convert the external value to json
+     *
+     * Optional, the default is to throw an error
+     * @param[in] self the void* passed to nix_create_external_value
+     * @param[in] state The evaluator state
+     * @param[in] strict boolean Whether to force the value before printing
+     * @param[out] c writable string context for the resulting string
+     * @param[in] copyToStore whether to copy referenced paths to store or keep
+     * them as-is
+     * @param[out] res the return value. Gets parsed as JSON. Not touching this,
+     * or setting it to the empty string, will make the conversion throw an error.
+     */
+    void (*printValueAsJSON)(
+        void * self, EvalState * state, bool strict, nix_string_context * c, bool copyToStore, nix_string_return * res);
+    /**
+     * @brief Convert the external value to XML
+     *
+     * Optional, the default is to throw an error
+     * @todo The mechanisms for this call are incomplete. There are no C
+     *       bindings to work with XML, pathsets and positions.
+     * @param[in] self the void* passed to nix_create_external_value
+     * @param[in] state The evaluator state
+     * @param[in] strict boolean Whether to force the value before printing
+     * @param[in] location boolean Whether to include position information in the
+     * xml
+     * @param[out] doc XML document to output to
+     * @param[out] c writable string context for the resulting string
+     * @param[in,out] drvsSeen a path set to avoid duplicating derivations
+     * @param[in] pos The position of the call.
+     */
+    void (*printValueAsXML)(
+        void * self,
+        EvalState * state,
+        int strict,
+        int location,
+        void * doc,
+        nix_string_context * c,
+        void * drvsSeen,
+        int pos);
+} NixCExternalValueDesc;
+
+/**
+ * @brief Create an external value, that can be given to nix_init_external
+ *
+ * Owned by the GC. Use nix_gc_decref when you're done with the pointer.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] desc a NixCExternalValueDesc, you should keep this alive as long
+ * as the ExternalValue lives
+ * @param[in] v the value to store
+ * @returns external value, owned by the garbage collector
+ * @see nix_init_external
+ */
+ExternalValue * nix_create_external_value(nix_c_context * context, NixCExternalValueDesc * desc, void * v);
+
+/**
+ * @brief Extract the pointer from a nix c external value.
+ * @param[out] context Optional, stores error information
+ * @param[in] b The external value
+ * @returns The pointer, or null if the external value was not from nix c.
+ * @see nix_get_external
+ */
+void * nix_get_external_value_content(nix_c_context * context, ExternalValue * b);
+
+// cffi end
+#ifdef __cplusplus
+}
+#endif
+/** @} */
+
+#endif // NIX_API_EXTERNAL_H

src/libexpr-c/nix_api_value.cc

@@ -0,0 +1,663 @@
+#include "attr-set.hh"
+#include "config.hh"
+#include "eval.hh"
+#include "globals.hh"
+#include "path.hh"
+#include "primops.hh"
+#include "value.hh"
+
+#include "nix_api_expr.h"
+#include "nix_api_expr_internal.h"
+#include "nix_api_util.h"
+#include "nix_api_util_internal.h"
+#include "nix_api_store_internal.h"
+#include "nix_api_value.h"
+#include "value/context.hh"
+
+#if HAVE_BOEHMGC
+#  include "gc/gc.h"
+#  define GC_INCLUDE_NEW 1
+#  include "gc_cpp.h"
+#endif
+
+// Internal helper functions to check [in] and [out] `Value *` parameters
+static const nix::Value & check_value_not_null(const nix_value * value)
+{
+    if (!value) {
+        throw std::runtime_error("nix_value is null");
+    }
+    return *((const nix::Value *) value);
+}
+
+static nix::Value & check_value_not_null(nix_value * value)
+{
+    if (!value) {
+        throw std::runtime_error("nix_value is null");
+    }
+    return value->value;
+}
+
+static const nix::Value & check_value_in(const nix_value * value)
+{
+    auto & v = check_value_not_null(value);
+    if (!v.isValid()) {
+        throw std::runtime_error("Uninitialized nix_value");
+    }
+    return v;
+}
+
+static nix::Value & check_value_in(nix_value * value)
+{
+    auto & v = check_value_not_null(value);
+    if (!v.isValid()) {
+        throw std::runtime_error("Uninitialized nix_value");
+    }
+    return v;
+}
+
+static nix::Value & check_value_out(nix_value * value)
+{
+    auto & v = check_value_not_null(value);
+    if (v.isValid()) {
+        throw std::runtime_error("nix_value already initialized. Variables are immutable");
+    }
+    return v;
+}
+
+static inline nix_value * as_nix_value_ptr(nix::Value * v)
+{
+    return reinterpret_cast<nix_value *>(v);
+}
+
+/**
+ * Helper function to convert calls from nix into C API.
+ *
+ * Deals with errors and converts arguments from C++ into C types.
+ */
+static void nix_c_primop_wrapper(
+    PrimOpFun f, void * userdata, nix::EvalState & state, const nix::PosIdx pos, nix::Value ** args, nix::Value & v)
+{
+    nix_c_context ctx;
+
+    // v currently has a thunk, but the C API initializers require an uninitialized value.
+    //
+    // We can't destroy the thunk, because that makes it impossible to retry,
+    // which is needed for tryEval and for evaluation drivers that evaluate more
+    // than one value (e.g. an attrset with two derivations, both of which
+    // reference v).
+    //
+    // Instead we create a temporary value, and then assign the result to v.
+    // This does not give the primop definition access to the thunk, but that's
+    // ok because we don't see a need for this yet (e.g. inspecting thunks,
+    // or maybe something to make blackholes work better; we don't know).
+    nix::Value vTmp;
+
+    f(userdata, &ctx, (EvalState *) &state, (nix_value **) args, (nix_value *) &vTmp);
+
+    if (ctx.last_err_code != NIX_OK) {
+        /* TODO: Throw different errors depending on the error code */
+        state.error<nix::EvalError>("Error from custom function: %s", *ctx.last_err).atPos(pos).debugThrow();
+    }
+
+    if (!vTmp.isValid()) {
+        state.error<nix::EvalError>("Implementation error in custom function: return value was not initialized")
+            .atPos(pos)
+            .debugThrow();
+    }
+
+    if (vTmp.type() == nix::nThunk) {
+        // We might allow this in the future if it makes sense for the evaluator
+        // e.g. implementing tail recursion by returning a thunk to the next
+        // "iteration". Until then, this is most likely a mistake or misunderstanding.
+        state.error<nix::EvalError>("Implementation error in custom function: return value must not be a thunk")
+            .atPos(pos)
+            .debugThrow();
+    }
+
+    v = vTmp;
+}
+
+PrimOp * nix_alloc_primop(
+    nix_c_context * context,
+    PrimOpFun fun,
+    int arity,
+    const char * name,
+    const char ** args,
+    const char * doc,
+    void * user_data)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        using namespace std::placeholders;
+        auto p = new
+#if HAVE_BOEHMGC
+            (GC)
+#endif
+                nix::PrimOp{
+                    .name = name,
+                    .args = {},
+                    .arity = (size_t) arity,
+                    .doc = doc,
+                    .fun = std::bind(nix_c_primop_wrapper, fun, user_data, _1, _2, _3, _4)};
+        if (args)
+            for (size_t i = 0; args[i]; i++)
+                p->args.emplace_back(*args);
+        nix_gc_incref(nullptr, p);
+        return (PrimOp *) p;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+nix_err nix_register_primop(nix_c_context * context, PrimOp * primOp)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        nix::RegisterPrimOp r(std::move(*((nix::PrimOp *) primOp)));
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_value * nix_alloc_value(nix_c_context * context, EvalState * state)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        nix_value * res = as_nix_value_ptr(state->state.allocValue());
+        nix_gc_incref(nullptr, res);
+        return res;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+ValueType nix_get_type(nix_c_context * context, const nix_value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_in(value);
+        using namespace nix;
+        switch (v.type()) {
+        case nThunk:
+            return NIX_TYPE_THUNK;
+        case nInt:
+            return NIX_TYPE_INT;
+        case nFloat:
+            return NIX_TYPE_FLOAT;
+        case nBool:
+            return NIX_TYPE_BOOL;
+        case nString:
+            return NIX_TYPE_STRING;
+        case nPath:
+            return NIX_TYPE_PATH;
+        case nNull:
+            return NIX_TYPE_NULL;
+        case nAttrs:
+            return NIX_TYPE_ATTRS;
+        case nList:
+            return NIX_TYPE_LIST;
+        case nFunction:
+            return NIX_TYPE_FUNCTION;
+        case nExternal:
+            return NIX_TYPE_EXTERNAL;
+        }
+        return NIX_TYPE_NULL;
+    }
+    NIXC_CATCH_ERRS_RES(NIX_TYPE_NULL);
+}
+
+const char * nix_get_typename(nix_c_context * context, const nix_value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_in(value);
+        auto s = nix::showType(v);
+        return strdup(s.c_str());
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+bool nix_get_bool(nix_c_context * context, const nix_value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_in(value);
+        assert(v.type() == nix::nBool);
+        return v.boolean();
+    }
+    NIXC_CATCH_ERRS_RES(false);
+}
+
+nix_err
+nix_get_string(nix_c_context * context, const nix_value * value, nix_get_string_callback callback, void * user_data)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_in(value);
+        assert(v.type() == nix::nString);
+        call_nix_get_string_callback(v.c_str(), callback, user_data);
+    }
+    NIXC_CATCH_ERRS
+}
+
+const char * nix_get_path_string(nix_c_context * context, const nix_value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_in(value);
+        assert(v.type() == nix::nPath);
+        // NOTE (from @yorickvP)
+        // v._path.path should work but may not be how Eelco intended it.
+        // Long-term this function should be rewritten to copy some data into a
+        // user-allocated string.
+        // We could use v.path().to_string().c_str(), but I'm concerned this
+        // crashes. Looks like .path() allocates a CanonPath with a copy of the
+        // string, then it gets the underlying data from that.
+        return v.payload.path.path;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+unsigned int nix_get_list_size(nix_c_context * context, const nix_value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_in(value);
+        assert(v.type() == nix::nList);
+        return v.listSize();
+    }
+    NIXC_CATCH_ERRS_RES(0);
+}
+
+unsigned int nix_get_attrs_size(nix_c_context * context, const nix_value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_in(value);
+        assert(v.type() == nix::nAttrs);
+        return v.attrs()->size();
+    }
+    NIXC_CATCH_ERRS_RES(0);
+}
+
+double nix_get_float(nix_c_context * context, const nix_value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_in(value);
+        assert(v.type() == nix::nFloat);
+        return v.fpoint();
+    }
+    NIXC_CATCH_ERRS_RES(0.0);
+}
+
+int64_t nix_get_int(nix_c_context * context, const nix_value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_in(value);
+        assert(v.type() == nix::nInt);
+        return v.integer().value;
+    }
+    NIXC_CATCH_ERRS_RES(0);
+}
+
+ExternalValue * nix_get_external(nix_c_context * context, nix_value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_out(value);
+        assert(v.type() == nix::nExternal);
+        return (ExternalValue *) v.external();
+    }
+    NIXC_CATCH_ERRS_NULL;
+}
+
+nix_value * nix_get_list_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int ix)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_in(value);
+        assert(v.type() == nix::nList);
+        auto * p = v.listElems()[ix];
+        nix_gc_incref(nullptr, p);
+        if (p != nullptr)
+            state->state.forceValue(*p, nix::noPos);
+        return as_nix_value_ptr(p);
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+nix_value * nix_get_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_in(value);
+        assert(v.type() == nix::nAttrs);
+        nix::Symbol s = state->state.symbols.create(name);
+        auto attr = v.attrs()->get(s);
+        if (attr) {
+            nix_gc_incref(nullptr, attr->value);
+            state->state.forceValue(*attr->value, nix::noPos);
+            return as_nix_value_ptr(attr->value);
+        }
+        nix_set_err_msg(context, NIX_ERR_KEY, "missing attribute");
+        return nullptr;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+bool nix_has_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_in(value);
+        assert(v.type() == nix::nAttrs);
+        nix::Symbol s = state->state.symbols.create(name);
+        auto attr = v.attrs()->get(s);
+        if (attr)
+            return true;
+        return false;
+    }
+    NIXC_CATCH_ERRS_RES(false);
+}
+
+nix_value * nix_get_attr_byidx(
+    nix_c_context * context, const nix_value * value, EvalState * state, unsigned int i, const char ** name)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_in(value);
+        const nix::Attr & a = (*v.attrs())[i];
+        *name = state->state.symbols[a.name].c_str();
+        nix_gc_incref(nullptr, a.value);
+        state->state.forceValue(*a.value, nix::noPos);
+        return as_nix_value_ptr(a.value);
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+const char *
+nix_get_attr_name_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int i)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_in(value);
+        const nix::Attr & a = (*v.attrs())[i];
+        return state->state.symbols[a.name].c_str();
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+nix_err nix_init_bool(nix_c_context * context, nix_value * value, bool b)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_out(value);
+        v.mkBool(b);
+    }
+    NIXC_CATCH_ERRS
+}
+
+// todo string context
+nix_err nix_init_string(nix_c_context * context, nix_value * value, const char * str)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_out(value);
+        v.mkString(std::string_view(str));
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_path_string(nix_c_context * context, EvalState * s, nix_value * value, const char * str)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_out(value);
+        v.mkPath(s->state.rootPath(nix::CanonPath(str)));
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_float(nix_c_context * context, nix_value * value, double d)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_out(value);
+        v.mkFloat(d);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_int(nix_c_context * context, nix_value * value, int64_t i)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_out(value);
+        v.mkInt(i);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_null(nix_c_context * context, nix_value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_out(value);
+        v.mkNull();
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_apply(nix_c_context * context, nix_value * value, nix_value * fn, nix_value * arg)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        auto & f = check_value_not_null(fn);
+        auto & a = check_value_not_null(arg);
+        v.mkApp(&f, &a);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_external(nix_c_context * context, nix_value * value, ExternalValue * val)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_out(value);
+        auto r = (nix::ExternalValueBase *) val;
+        v.mkExternal(r);
+    }
+    NIXC_CATCH_ERRS
+}
+
+ListBuilder * nix_make_list_builder(nix_c_context * context, EvalState * state, size_t capacity)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto builder = state->state.buildList(capacity);
+        return new
+#if HAVE_BOEHMGC
+            (NoGC)
+#endif
+                ListBuilder{std::move(builder)};
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+nix_err
+nix_list_builder_insert(nix_c_context * context, ListBuilder * list_builder, unsigned int index, nix_value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & e = check_value_not_null(value);
+        list_builder->builder[index] = &e;
+    }
+    NIXC_CATCH_ERRS
+}
+
+void nix_list_builder_free(ListBuilder * list_builder)
+{
+#if HAVE_BOEHMGC
+    GC_FREE(list_builder);
+#else
+    delete list_builder;
+#endif
+}
+
+nix_err nix_make_list(nix_c_context * context, ListBuilder * list_builder, nix_value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_out(value);
+        v.mkList(list_builder->builder);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_primop(nix_c_context * context, nix_value * value, PrimOp * p)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_out(value);
+        v.mkPrimOp((nix::PrimOp *) p);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_copy_value(nix_c_context * context, nix_value * value, const nix_value * source)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_out(value);
+        auto & s = check_value_in(source);
+        v = s;
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_make_attrs(nix_c_context * context, nix_value * value, BindingsBuilder * b)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_out(value);
+        v.mkAttrs(b->builder);
+    }
+    NIXC_CATCH_ERRS
+}
+
+BindingsBuilder * nix_make_bindings_builder(nix_c_context * context, EvalState * state, size_t capacity)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto bb = state->state.buildBindings(capacity);
+        return new
+#if HAVE_BOEHMGC
+            (NoGC)
+#endif
+                BindingsBuilder{std::move(bb)};
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+nix_err nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * bb, const char * name, nix_value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        nix::Symbol s = bb->builder.state.symbols.create(name);
+        bb->builder.insert(s, &v);
+    }
+    NIXC_CATCH_ERRS
+}
+
+void nix_bindings_builder_free(BindingsBuilder * bb)
+{
+#if HAVE_BOEHMGC
+    GC_FREE((nix::BindingsBuilder *) bb);
+#else
+    delete (nix::BindingsBuilder *) bb;
+#endif
+}
+
+nix_realised_string * nix_string_realise(nix_c_context * context, EvalState * state, nix_value * value, bool isIFD)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_in(value);
+        nix::NixStringContext stringContext;
+        auto rawStr = state->state.coerceToString(nix::noPos, v, stringContext, "while realising a string").toOwned();
+        nix::StorePathSet storePaths;
+        auto rewrites = state->state.realiseContext(stringContext, &storePaths);
+
+        auto s = nix::rewriteStrings(rawStr, rewrites);
+
+        // Convert to the C API StorePath type and convert to vector for index-based access
+        std::vector<StorePath> vec;
+        for (auto & sp : storePaths) {
+            vec.push_back(StorePath{sp});
+        }
+
+        return new nix_realised_string{.str = s, .storePaths = vec};
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+void nix_realised_string_free(nix_realised_string * s)
+{
+    delete s;
+}
+
+size_t nix_realised_string_get_buffer_size(nix_realised_string * s)
+{
+    return s->str.size();
+}
+
+const char * nix_realised_string_get_buffer_start(nix_realised_string * s)
+{
+    return s->str.data();
+}
+
+size_t nix_realised_string_get_store_path_count(nix_realised_string * s)
+{
+    return s->storePaths.size();
+}
+
+const StorePath * nix_realised_string_get_store_path(nix_realised_string * s, size_t i)
+{
+    return &s->storePaths[i];
+}

src/libexpr-c/nix_api_value.h

@@ -0,0 +1,537 @@
+#ifndef NIX_API_VALUE_H
+#define NIX_API_VALUE_H
+
+/** @addtogroup libexpr
+ * @{
+ */
+/** @file
+ * @brief libexpr C bindings dealing with values
+ */
+
+#include "nix_api_util.h"
+#include "nix_api_store.h"
+#include "stdbool.h"
+#include "stddef.h"
+#include "stdint.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+// cffi start
+
+// Type definitions
+typedef enum {
+    NIX_TYPE_THUNK,
+    NIX_TYPE_INT,
+    NIX_TYPE_FLOAT,
+    NIX_TYPE_BOOL,
+    NIX_TYPE_STRING,
+    NIX_TYPE_PATH,
+    NIX_TYPE_NULL,
+    NIX_TYPE_ATTRS,
+    NIX_TYPE_LIST,
+    NIX_TYPE_FUNCTION,
+    NIX_TYPE_EXTERNAL
+} ValueType;
+
+// forward declarations
+typedef struct nix_value nix_value;
+typedef struct EvalState EvalState;
+
+[[deprecated("use nix_value instead")]] typedef nix_value Value;
+
+// type defs
+/** @brief Stores an under-construction set of bindings
+ * @ingroup value_manip
+ *
+ * Do not reuse.
+ * @see nix_make_bindings_builder, nix_bindings_builder_free, nix_make_attrs
+ * @see nix_bindings_builder_insert
+ */
+typedef struct BindingsBuilder BindingsBuilder;
+
+/** @brief Stores an under-construction list
+ * @ingroup value_manip
+ *
+ * Do not reuse.
+ * @see nix_make_list_builder, nix_list_builder_free, nix_make_list
+ * @see nix_list_builder_insert
+ */
+typedef struct ListBuilder ListBuilder;
+
+/** @brief PrimOp function
+ * @ingroup primops
+ *
+ * Owned by the GC
+ * @see nix_alloc_primop, nix_init_primop
+ */
+typedef struct PrimOp PrimOp;
+/** @brief External Value
+ * @ingroup Externals
+ *
+ * Owned by the GC
+ */
+typedef struct ExternalValue ExternalValue;
+
+/** @brief String without placeholders, and realised store paths
+ */
+typedef struct nix_realised_string nix_realised_string;
+
+/** @defgroup primops
+ * @brief Create your own primops
+ * @{
+ */
+/** @brief Function pointer for primops
+ *
+ * When you want to return an error, call nix_set_err_msg(context, NIX_ERR_UNKNOWN, "your error message here").
+ *
+ * @param[in] user_data Arbitrary data that was initially supplied to nix_alloc_primop
+ * @param[out] context Stores error information.
+ * @param[in] state Evaluator state
+ * @param[in] args list of arguments. Note that these can be thunks and should be forced using nix_value_force before
+ * use.
+ * @param[out] ret return value
+ * @see nix_alloc_primop, nix_init_primop
+ */
+typedef void (*PrimOpFun)(
+    void * user_data, nix_c_context * context, EvalState * state, nix_value ** args, nix_value * ret);
+
+/** @brief Allocate a PrimOp
+ *
+ * Owned by the garbage collector.
+ * Use nix_gc_decref() when you're done with the returned PrimOp.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] fun callback
+ * @param[in] arity expected number of function arguments
+ * @param[in] name function name
+ * @param[in] args array of argument names, NULL-terminated
+ * @param[in] doc optional, documentation for this primop
+ * @param[in] user_data optional, arbitrary data, passed to the callback when it's called
+ * @return primop, or null in case of errors
+ * @see nix_init_primop
+ */
+PrimOp * nix_alloc_primop(
+    nix_c_context * context,
+    PrimOpFun fun,
+    int arity,
+    const char * name,
+    const char ** args,
+    const char * doc,
+    void * user_data);
+
+/** @brief add a primop to the `builtins` attribute set
+ *
+ * Only applies to States created after this call.
+ *
+ * Moves your PrimOp content into the global evaluator
+ * registry, meaning your input PrimOp pointer is no longer usable.
+ * You are free to remove your references to it,
+ * after which it will be garbage collected.
+ *
+ * @param[out] context Optional, stores error information
+ * @return primop, or null in case of errors
+ *
+ */
+nix_err nix_register_primop(nix_c_context * context, PrimOp * primOp);
+/** @} */
+
+// Function prototypes
+
+/** @brief Allocate a Nix value
+ *
+ * Owned by the GC. Use nix_gc_decref() when you're done with the pointer
+ * @param[out] context Optional, stores error information
+ * @param[in] state nix evaluator state
+ * @return value, or null in case of errors
+ *
+ */
+nix_value * nix_alloc_value(nix_c_context * context, EvalState * state);
+
+/**
+ * @brief Increment the garbage collector reference counter for the given `nix_value`.
+ *
+ * The Nix language evaluator C API keeps track of alive objects by reference counting.
+ * When you're done with a refcounted pointer, call nix_value_decref().
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] value The object to keep alive
+ */
+nix_err nix_value_incref(nix_c_context * context, nix_value * value);
+
+/**
+ * @brief Decrement the garbage collector reference counter for the given object
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] value The object to stop referencing
+ */
+nix_err nix_value_decref(nix_c_context * context, nix_value * value);
+
+/** @addtogroup value_manip Manipulating values
+ * @brief Functions to inspect and change Nix language values, represented by nix_value.
+ * @{
+ */
+/** @anchor getters
+ * @name Getters
+ */
+/**@{*/
+/** @brief Get value type
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return type of nix value
+ */
+ValueType nix_get_type(nix_c_context * context, const nix_value * value);
+
+/** @brief Get type name of value as defined in the evaluator
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return type name, owned string
+ * @todo way to free the result
+ */
+const char * nix_get_typename(nix_c_context * context, const nix_value * value);
+
+/** @brief Get boolean value
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return true or false, error info via context
+ */
+bool nix_get_bool(nix_c_context * context, const nix_value * value);
+
+/** @brief Get the raw string
+ *
+ * This may contain placeholders.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @param[in] callback Called with the string value.
+ * @param[in] user_data optional, arbitrary data, passed to the callback when it's called.
+ * @return string
+ * @return error code, NIX_OK on success.
+ */
+nix_err
+nix_get_string(nix_c_context * context, const nix_value * value, nix_get_string_callback callback, void * user_data);
+
+/** @brief Get path as string
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return string
+ * @return NULL in case of error.
+ */
+const char * nix_get_path_string(nix_c_context * context, const nix_value * value);
+
+/** @brief Get the length of a list
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return length of list, error info via context
+ */
+unsigned int nix_get_list_size(nix_c_context * context, const nix_value * value);
+
+/** @brief Get the element count of an attrset
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return attrset element count, error info via context
+ */
+unsigned int nix_get_attrs_size(nix_c_context * context, const nix_value * value);
+
+/** @brief Get float value in 64 bits
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return float contents, error info via context
+ */
+double nix_get_float(nix_c_context * context, const nix_value * value);
+
+/** @brief Get int value
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return int contents, error info via context
+ */
+int64_t nix_get_int(nix_c_context * context, const nix_value * value);
+
+/** @brief Get external reference
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return reference to external, NULL in case of error
+ */
+ExternalValue * nix_get_external(nix_c_context * context, nix_value *);
+
+/** @brief Get the ix'th element of a list
+ *
+ * Owned by the GC. Use nix_gc_decref when you're done with the pointer
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @param[in] state nix evaluator state
+ * @param[in] ix list element to get
+ * @return value, NULL in case of errors
+ */
+nix_value * nix_get_list_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int ix);
+
+/** @brief Get an attr by name
+ *
+ * Owned by the GC. Use nix_gc_decref when you're done with the pointer
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @param[in] state nix evaluator state
+ * @param[in] name attribute name
+ * @return value, NULL in case of errors
+ */
+nix_value * nix_get_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name);
+
+/** @brief Check if an attribute name exists on a value
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @param[in] state nix evaluator state
+ * @param[in] name attribute name
+ * @return value, error info via context
+ */
+bool nix_has_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name);
+
+/** @brief Get an attribute by index in the sorted bindings
+ *
+ * Also gives you the name.
+ *
+ * Owned by the GC. Use nix_gc_decref when you're done with the pointer
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @param[in] state nix evaluator state
+ * @param[in] i attribute index
+ * @param[out] name will store a pointer to the attribute name
+ * @return value, NULL in case of errors
+ */
+nix_value * nix_get_attr_byidx(
+    nix_c_context * context, const nix_value * value, EvalState * state, unsigned int i, const char ** name);
+
+/** @brief Get an attribute name by index in the sorted bindings
+ *
+ * Useful when you want the name but want to avoid evaluation.
+ *
+ * Owned by the nix EvalState
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @param[in] state nix evaluator state
+ * @param[in] i attribute index
+ * @return name, NULL in case of errors
+ */
+const char *
+nix_get_attr_name_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int i);
+
+/**@}*/
+/** @name Initializers
+ *
+ * Values are typically "returned" by initializing already allocated memory that serves as the return value.
+ * For this reason, the construction of values is not tied their allocation.
+ * Nix is a language with immutable values. Respect this property by only initializing Values once; and only initialize
+ * Values that are meant to be initialized by you. Failing to adhere to these rules may lead to undefined behavior.
+ */
+/**@{*/
+/** @brief Set boolean value
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] b the boolean value
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_init_bool(nix_c_context * context, nix_value * value, bool b);
+
+/** @brief Set a string
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] str the string, copied
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_init_string(nix_c_context * context, nix_value * value, const char * str);
+
+/** @brief Set a path
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] str the path string, copied
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_init_path_string(nix_c_context * context, EvalState * s, nix_value * value, const char * str);
+
+/** @brief Set a float
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] d the float, 64-bits
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_init_float(nix_c_context * context, nix_value * value, double d);
+
+/** @brief Set an int
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] i the int
+ * @return error code, NIX_OK on success.
+ */
+
+nix_err nix_init_int(nix_c_context * context, nix_value * value, int64_t i);
+/** @brief Set null
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_init_null(nix_c_context * context, nix_value * value);
+
+/** @brief Set the value to a thunk that will perform a function application when needed.
+ *
+ * Thunks may be put into attribute sets and lists to perform some computation lazily; on demand.
+ * However, note that in some places, a thunk must not be returned, such as in the return value of a PrimOp.
+ * In such cases, you may use nix_value_call() instead (but note the different argument order).
+ *
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] fn function to call
+ * @param[in] arg argument to pass
+ * @return error code, NIX_OK on successful initialization.
+ * @see nix_value_call() for a similar function that performs the call immediately and only stores the return value.
+ *      Note the different argument order.
+ */
+nix_err nix_init_apply(nix_c_context * context, nix_value * value, nix_value * fn, nix_value * arg);
+
+/** @brief Set an external value
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] val the external value to set. Will be GC-referenced by the value.
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_init_external(nix_c_context * context, nix_value * value, ExternalValue * val);
+
+/** @brief Create a list from a list builder
+ * @param[out] context Optional, stores error information
+ * @param[in] list_builder list builder to use. Make sure to unref this afterwards.
+ * @param[out] value Nix value to modify
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_make_list(nix_c_context * context, ListBuilder * list_builder, nix_value * value);
+
+/** @brief Create a list builder
+ * @param[out] context Optional, stores error information
+ * @param[in] state nix evaluator state
+ * @param[in] capacity how many bindings you'll add. Don't exceed.
+ * @return owned reference to a list builder. Make sure to unref when you're done.
+ */
+ListBuilder * nix_make_list_builder(nix_c_context * context, EvalState * state, size_t capacity);
+
+/** @brief Insert bindings into a builder
+ * @param[out] context Optional, stores error information
+ * @param[in] list_builder ListBuilder to insert into
+ * @param[in] index index to manipulate
+ * @param[in] value value to insert
+ * @return error code, NIX_OK on success.
+ */
+nix_err
+nix_list_builder_insert(nix_c_context * context, ListBuilder * list_builder, unsigned int index, nix_value * value);
+
+/** @brief Free a list builder
+ *
+ * Does not fail.
+ * @param[in] builder the builder to free
+ */
+void nix_list_builder_free(ListBuilder * list_builder);
+
+/** @brief Create an attribute set from a bindings builder
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] b bindings builder to use. Make sure to unref this afterwards.
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_make_attrs(nix_c_context * context, nix_value * value, BindingsBuilder * b);
+
+/** @brief Set primop
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] op primop, will be gc-referenced by the value
+ * @see nix_alloc_primop
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_init_primop(nix_c_context * context, nix_value * value, PrimOp * op);
+/** @brief Copy from another value
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] source value to copy from
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_copy_value(nix_c_context * context, nix_value * value, const nix_value * source);
+/**@}*/
+
+/** @brief Create a bindings builder
+* @param[out] context Optional, stores error information
+* @param[in] state nix evaluator state
+* @param[in] capacity how many bindings you'll add. Don't exceed.
+* @return owned reference to a bindings builder. Make sure to unref when you're
+done.
+*/
+BindingsBuilder * nix_make_bindings_builder(nix_c_context * context, EvalState * state, size_t capacity);
+
+/** @brief Insert bindings into a builder
+ * @param[out] context Optional, stores error information
+ * @param[in] builder BindingsBuilder to insert into
+ * @param[in] name attribute name, only used for the duration of the call.
+ * @param[in] value value to give the binding
+ * @return error code, NIX_OK on success.
+ */
+nix_err
+nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * builder, const char * name, nix_value * value);
+
+/** @brief Free a bindings builder
+ *
+ * Does not fail.
+ * @param[in] builder the builder to free
+ */
+void nix_bindings_builder_free(BindingsBuilder * builder);
+/**@}*/
+
+/** @brief Realise a string context.
+ *
+ * This will
+ *  - realise the store paths referenced by the string's context, and
+ *  - perform the replacement of placeholders.
+ *  - create temporary garbage collection roots for the store paths, for
+ *    the lifetime of the current process.
+ *  - log to stderr
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value, which must be a string
+ * @param[in] state Nix evaluator state
+ * @param[in] isIFD If true, disallow derivation outputs if setting `allow-import-from-derivation` is false.
+                    You should set this to true when this call is part of a primop.
+                    You should set this to false when building for your application's purpose.
+ * @return NULL if failed, are a new nix_realised_string, which must be freed with nix_realised_string_free
+ */
+nix_realised_string * nix_string_realise(nix_c_context * context, EvalState * state, nix_value * value, bool isIFD);
+
+/** @brief Start of the string
+ * @param[in] realised_string
+ * @return pointer to the start of the string. It may not be null-terminated.
+ */
+const char * nix_realised_string_get_buffer_start(nix_realised_string * realised_string);
+
+/** @brief Length of the string
+ * @param[in] realised_string
+ * @return length of the string in bytes
+ */
+size_t nix_realised_string_get_buffer_size(nix_realised_string * realised_string);
+
+/** @brief Number of realised store paths
+ * @param[in] realised_string
+ * @return number of realised store paths that were referenced by the string via its context
+ */
+size_t nix_realised_string_get_store_path_count(nix_realised_string * realised_string);
+
+/** @brief Get a store path. The store paths are stored in an arbitrary order.
+ * @param[in] realised_string
+ * @param[in] index index of the store path, must be less than the count
+ * @return store path
+ */
+const StorePath * nix_realised_string_get_store_path(nix_realised_string * realised_string, size_t index);
+
+/** @brief Free a realised string
+ * @param[in] realised_string
+ */
+void nix_realised_string_free(nix_realised_string * realised_string);
+
+// cffi end
+#ifdef __cplusplus
+}
+#endif
+
+/** @} */
+#endif // NIX_API_VALUE_H

src/libexpr-c/package.nix

@@ -0,0 +1,74 @@
+{ lib
+, stdenv
+, mkMesonDerivation
+
+, meson
+, ninja
+, pkg-config
+
+, nix-store-c
+, nix-expr
+
+# Configuration Options
+
+, version
+}:
+
+let
+  inherit (lib) fileset;
+in
+
+mkMesonDerivation (finalAttrs: {
+  pname = "nix-expr-c";
+  inherit version;
+
+  workDir = ./.;
+  fileset = fileset.unions [
+    ../../build-utils-meson
+    ./build-utils-meson
+    ../../.version
+    ./.version
+    ./meson.build
+    # ./meson.options
+    (fileset.fileFilter (file: file.hasExt "cc") ./.)
+    (fileset.fileFilter (file: file.hasExt "hh") ./.)
+    (fileset.fileFilter (file: file.hasExt "h") ./.)
+  ];
+
+  outputs = [ "out" "dev" ];
+
+  nativeBuildInputs = [
+    meson
+    ninja
+    pkg-config
+  ];
+
+  propagatedBuildInputs = [
+    nix-store-c
+    nix-expr
+  ];
+
+  preConfigure =
+    # "Inline" .version so it's not a symlink, and includes the suffix.
+    # Do the meson utils, without modification.
+    ''
+      chmod u+w ./.version
+      echo ${version} > ../../.version
+    '';
+
+  mesonFlags = [
+  ];
+
+  env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) {
+    LDFLAGS = "-fuse-ld=gold";
+  };
+
+  separateDebugInfo = !stdenv.hostPlatform.isStatic;
+
+  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
+
+  meta = {
+    platforms = lib.platforms.unix ++ lib.platforms.windows;
+  };
+
+})

src/libexpr/.version

@@ -0,0 +1 @@
+../../.version

src/libexpr/attr-path.cc

@@ -1,6 +1,5 @@
 #include "attr-path.hh"
 #include "eval-inline.hh"
-#include "util.hh"
 
 
 namespace nix {
@@ -66,18 +65,18 @@ std::pair<Value *, PosIdx> findAlongAttrPath(EvalState & state, const std::strin
         if (!attrIndex) {
 
             if (v->type() != nAttrs)
-                throw TypeError(
+                state.error<TypeError>(
                     "the expression selected by the selection path '%1%' should be a set but is %2%",
                     attrPath,
-                    showType(*v));
+                    showType(*v)).debugThrow();
             if (attr.empty())
                 throw Error("empty attribute name in selection path '%1%'", attrPath);
 
-            Bindings::iterator a = v->attrs->find(state.symbols.create(attr));
-            if (a == v->attrs->end()) {
+            auto a = v->attrs()->get(state.symbols.create(attr));
+            if (!a) {
                 std::set<std::string> attrNames;
-                for (auto & attr : *v->attrs)
-                    attrNames.insert(state.symbols[attr.name]);
+                for (auto & attr : *v->attrs())
+                    attrNames.insert(std::string(state.symbols[attr.name]));
 
                 auto suggestions = Suggestions::bestMatches(attrNames, attr);
                 throw AttrPathNotFound(suggestions, "attribute '%1%' in selection path '%2%' not found", attr, attrPath);
@@ -89,10 +88,10 @@ std::pair<Value *, PosIdx> findAlongAttrPath(EvalState & state, const std::strin
         else {
 
             if (!v->isList())
-                throw TypeError(
+                state.error<TypeError>(
                     "the expression selected by the selection path '%1%' should be a list but is %2%",
                     attrPath,
-                    showType(*v));
+                    showType(*v)).debugThrow();
             if (*attrIndex >= v->listSize())
                 throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", *attrIndex, attrPath);
 
@@ -106,7 +105,7 @@ std::pair<Value *, PosIdx> findAlongAttrPath(EvalState & state, const std::strin
 }
 
 
-std::pair<std::string, uint32_t> findPackageFilename(EvalState & state, Value & v, std::string what)
+std::pair<SourcePath, uint32_t> findPackageFilename(EvalState & state, Value & v, std::string what)
 {
     Value * v2;
     try {
@@ -118,21 +117,25 @@ std::pair<std::string, uint32_t> findPackageFilename(EvalState & state, Value &
 
     // FIXME: is it possible to extract the Pos object instead of doing this
     //        toString + parsing?
-    auto pos = state.forceString(*v2);
+    NixStringContext context;
+    auto path = state.coerceToPath(noPos, *v2, context, "while evaluating the 'meta.position' attribute of a derivation");
 
-    auto colon = pos.rfind(':');
-    if (colon == std::string::npos)
-        throw ParseError("cannot parse meta.position attribute '%s'", pos);
+    auto fn = path.path.abs();
+
+    auto fail = [fn]() {
+        throw ParseError("cannot parse 'meta.position' attribute '%s'", fn);
+    };
 
-    std::string filename(pos, 0, colon);
-    unsigned int lineno;
     try {
-        lineno = std::stoi(std::string(pos, colon + 1, std::string::npos));
+        auto colon = fn.rfind(':');
+        if (colon == std::string::npos) fail();
+        std::string filename(fn, 0, colon);
+        auto lineno = std::stoi(std::string(fn, colon + 1, std::string::npos));
+        return {SourcePath{path.accessor, CanonPath(fn.substr(0, colon))}, lineno};
     } catch (std::invalid_argument & e) {
-        throw ParseError("cannot parse line number '%s'", pos);
+        fail();
+        unreachable();
     }
-
-    return { std::move(filename), lineno };
 }
 
 

src/libexpr/attr-path.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "eval.hh"
 
@@ -16,8 +17,10 @@ std::pair<Value *, PosIdx> findAlongAttrPath(
     Bindings & autoArgs,
     Value & vIn);
 
-/* Heuristic to find the filename and lineno or a nix value. */
-std::pair<std::string, uint32_t> findPackageFilename(EvalState & state, Value & v, std::string what);
+/**
+ * Heuristic to find the filename and lineno or a nix value.
+ */
+std::pair<SourcePath, uint32_t> findPackageFilename(EvalState & state, Value & v, std::string what);
 
 std::vector<Symbol> parseAttrPath(EvalState & state, std::string_view s);
 

src/libexpr/attr-set.cc

@@ -23,23 +23,6 @@ Bindings * EvalState::allocBindings(size_t capacity)
 }
 
 
-/* Create a new attribute named 'name' on an existing attribute set stored
-   in 'vAttrs' and return the newly allocated Value which is associated with
-   this attribute. */
-Value * EvalState::allocAttr(Value & vAttrs, Symbol name)
-{
-    Value * v = allocValue();
-    vAttrs.attrs->push_back(Attr(name, v));
-    return v;
-}
-
-
-Value * EvalState::allocAttr(Value & vAttrs, std::string_view name)
-{
-    return allocAttr(vAttrs, symbols.create(name));
-}
-
-
 Value & BindingsBuilder::alloc(Symbol name, PosIdx pos)
 {
     auto value = state.allocValue();

src/libexpr/attr-set.hh

@@ -1,10 +1,10 @@
 #pragma once
+///@file
 
 #include "nixexpr.hh"
 #include "symbol-table.hh"
 
 #include <algorithm>
-#include <optional>
 
 namespace nix {
 
@@ -12,7 +12,9 @@ namespace nix {
 class EvalState;
 struct Value;
 
-/* Map one attribute name to its value. */
+/**
+ * Map one attribute name to its value.
+ */
 struct Attr
 {
     /* the placement of `name` and `pos` in this struct is important.
@@ -25,9 +27,9 @@ struct Attr
     Attr(Symbol name, Value * value, PosIdx pos = noPos)
         : name(name), pos(pos), value(value) { };
     Attr() { };
-    bool operator < (const Attr & a) const
+    auto operator <=> (const Attr & a) const
     {
-        return name < a.name;
+        return name <=> a.name;
     }
 };
 
@@ -36,10 +38,12 @@ static_assert(sizeof(Attr) == 2 * sizeof(uint32_t) + sizeof(Value *),
     "avoid introducing any padding into Attr if at all possible, and do not "
     "introduce new fields that need not be present for almost every instance.");
 
-/* Bindings contains all the attributes of an attribute set. It is defined
-   by its size and its capacity, the capacity being the number of Attr
-   elements allocated after this structure, while the size corresponds to
-   the number of elements already inserted in this structure. */
+/**
+ * Bindings contains all the attributes of an attribute set. It is defined
+ * by its size and its capacity, the capacity being the number of Attr
+ * elements allocated after this structure, while the size corresponds to
+ * the number of elements already inserted in this structure.
+ */
 class Bindings
 {
 public:
@@ -60,24 +64,26 @@ public:
 
     typedef Attr * iterator;
 
+    typedef const Attr * const_iterator;
+
     void push_back(const Attr & attr)
     {
         assert(size_ < capacity_);
         attrs[size_++] = attr;
     }
 
-    iterator find(Symbol name)
+    const_iterator find(Symbol name) const
     {
         Attr key(name, 0);
-        iterator i = std::lower_bound(begin(), end(), key);
+        const_iterator i = std::lower_bound(begin(), end(), key);
         if (i != end() && i->name == name) return i;
         return end();
     }
 
-    Attr * get(Symbol name)
+    const Attr * get(Symbol name) const
     {
         Attr key(name, 0);
-        iterator i = std::lower_bound(begin(), end(), key);
+        const_iterator i = std::lower_bound(begin(), end(), key);
         if (i != end() && i->name == name) return &*i;
         return nullptr;
     }
@@ -85,16 +91,26 @@ public:
     iterator begin() { return &attrs[0]; }
     iterator end() { return &attrs[size_]; }
 
+    const_iterator begin() const { return &attrs[0]; }
+    const_iterator end() const { return &attrs[size_]; }
+
     Attr & operator[](size_t pos)
     {
         return attrs[pos];
     }
 
+    const Attr & operator[](size_t pos) const
+    {
+        return attrs[pos];
+    }
+
     void sort();
 
-    size_t capacity() { return capacity_; }
+    size_t capacity() const { return capacity_; }
 
-    /* Returns the attributes in lexicographically sorted order. */
+    /**
+     * Returns the attributes in lexicographically sorted order.
+     */
     std::vector<const Attr *> lexicographicOrder(const SymbolTable & symbols) const
     {
         std::vector<const Attr *> res;
@@ -111,9 +127,11 @@ public:
     friend class EvalState;
 };
 
-/* A wrapper around Bindings that ensures that its always in sorted
-   order at the end. The only way to consume a BindingsBuilder is to
-   call finish(), which sorts the bindings. */
+/**
+ * A wrapper around Bindings that ensures that its always in sorted
+ * order at the end. The only way to consume a BindingsBuilder is to
+ * call finish(), which sorts the bindings.
+ */
 class BindingsBuilder
 {
     Bindings * bindings;
@@ -157,6 +175,20 @@ public:
     {
         return bindings;
     }
+
+    size_t capacity()
+    {
+        return bindings->capacity();
+    }
+
+    void grow(Bindings * newBindings)
+    {
+        for (auto & i : *bindings)
+            newBindings->push_back(i);
+        bindings = newBindings;
+    }
+
+    friend struct ExprAttrs;
 };
 
 }

src/libexpr/build-utils-meson

@@ -0,0 +1 @@
+../../build-utils-meson

src/libexpr/call-flake.nix

@@ -0,0 +1,85 @@
+# This is a helper to callFlake() to lazily fetch flake inputs.
+
+# The contents of the lock file, in JSON format.
+lockFileStr:
+
+# A mapping of lock file node IDs to { sourceInfo, subdir } attrsets,
+# with sourceInfo.outPath providing an SourceAccessor to a previously
+# fetched tree. This is necessary for possibly unlocked inputs, in
+# particular the root input, but also --override-inputs pointing to
+# unlocked trees.
+overrides:
+
+let
+
+  lockFile = builtins.fromJSON lockFileStr;
+
+  # Resolve a input spec into a node name. An input spec is
+  # either a node name, or a 'follows' path from the root
+  # node.
+  resolveInput = inputSpec:
+    if builtins.isList inputSpec
+    then getInputByPath lockFile.root inputSpec
+    else inputSpec;
+
+  # Follow an input path (e.g. ["dwarffs" "nixpkgs"]) from the
+  # root node, returning the final node.
+  getInputByPath = nodeName: path:
+    if path == []
+    then nodeName
+    else
+      getInputByPath
+        # Since this could be a 'follows' input, call resolveInput.
+        (resolveInput lockFile.nodes.${nodeName}.inputs.${builtins.head path})
+        (builtins.tail path);
+
+  allNodes =
+    builtins.mapAttrs
+      (key: node:
+        let
+
+          sourceInfo =
+            if overrides ? ${key}
+            then
+              overrides.${key}.sourceInfo
+            else
+              # FIXME: remove obsolete node.info.
+              fetchTree (node.info or {} // removeAttrs node.locked ["dir"]);
+
+          subdir = overrides.${key}.dir or node.locked.dir or "";
+
+          outPath = sourceInfo + ((if subdir == "" then "" else "/") + subdir);
+
+          flake = import (outPath + "/flake.nix");
+
+          inputs = builtins.mapAttrs
+            (inputName: inputSpec: allNodes.${resolveInput inputSpec})
+            (node.inputs or {});
+
+          outputs = flake.outputs (inputs // { self = result; });
+
+          result =
+            outputs
+            # We add the sourceInfo attribute for its metadata, as they are
+            # relevant metadata for the flake. However, the outPath of the
+            # sourceInfo does not necessarily match the outPath of the flake,
+            # as the flake may be in a subdirectory of a source.
+            # This is shadowed in the next //
+            // sourceInfo
+            // {
+              # This shadows the sourceInfo.outPath
+              inherit outPath;
+
+              inherit inputs; inherit outputs; inherit sourceInfo; _type = "flake";
+            };
+
+        in
+          if node.flake or true then
+            assert builtins.isFunction flake.outputs;
+            result
+          else
+            sourceInfo
+      )
+      lockFile.nodes;
+
+in allNodes.${lockFile.root}

src/libexpr/eval-cache.cc

@@ -1,11 +1,33 @@
+#include "users.hh"
 #include "eval-cache.hh"
 #include "sqlite.hh"
 #include "eval.hh"
 #include "eval-inline.hh"
 #include "store-api.hh"
+// Need specialization involving `SymbolStr` just in this one module.
+#include "strings-inline.hh"
 
 namespace nix::eval_cache {
 
+CachedEvalError::CachedEvalError(ref<AttrCursor> cursor, Symbol attr)
+    : EvalError(cursor->root->state, "cached failure of attribute '%s'", cursor->getAttrPathStr(attr))
+    , cursor(cursor), attr(attr)
+{ }
+
+void CachedEvalError::force()
+{
+    auto & v = cursor->forceValue();
+
+    if (v.type() == nAttrs) {
+        auto a = v.attrs()->get(this->attr);
+
+        state.forceValue(*a->value, a->pos);
+    }
+
+    // Shouldn't happen.
+    throw EvalError(state, "evaluation of cached failed attribute '%s' unexpectedly succeeded", cursor->getAttrPathStr(attr));
+}
+
 static const char * schema = R"sql(
 create table if not exists Attributes (
     parent      integer not null,
@@ -21,7 +43,7 @@ struct AttrDb
 {
     std::atomic_bool failed{false};
 
-    const Store & cfg;
+    const StoreDirConfig & cfg;
 
     struct State
     {
@@ -38,7 +60,7 @@ struct AttrDb
     SymbolTable & symbols;
 
     AttrDb(
-        const Store & cfg,
+        const StoreDirConfig & cfg,
         const Hash & fingerprint,
         SymbolTable & symbols)
         : cfg(cfg)
@@ -47,10 +69,10 @@ struct AttrDb
     {
         auto state(_state->lock());
 
-        Path cacheDir = getCacheDir() + "/nix/eval-cache-v4";
+        Path cacheDir = getCacheDir() + "/nix/eval-cache-v5";
         createDirs(cacheDir);
 
-        Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite";
+        Path dbPath = cacheDir + "/" + fingerprint.to_string(HashFormat::Base16, false) + ".sqlite";
 
         state->db = SQLite(dbPath);
         state->db.isCache();
@@ -75,7 +97,7 @@ struct AttrDb
     {
         try {
             auto state(_state->lock());
-            if (!failed)
+            if (!failed && state->txn->active)
                 state->txn->commit();
             state->txn.reset();
         } catch (...) {
@@ -205,7 +227,7 @@ struct AttrDb
                 (key.first)
                 (symbols[key.second])
                 (AttrType::ListOfStrings)
-                (concatStringsSep("\t", l)).exec();
+                (dropEmptyInitThenConcatStringsSep("\t", l)).exec();
 
             return state->db.getLastInsertedRowId();
         });
@@ -300,13 +322,13 @@ struct AttrDb
                 NixStringContext context;
                 if (!queryAttribute.isNull(3))
                     for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), ";"))
-                        context.push_back(decodeContext(cfg, s));
+                        context.insert(NixStringContextElem::parse(s));
                 return {{rowId, string_t{queryAttribute.getStr(2), context}}};
             }
             case AttrType::Bool:
                 return {{rowId, queryAttribute.getInt(2) != 0}};
             case AttrType::Int:
-                return {{rowId, int_t{queryAttribute.getInt(2)}}};
+                return {{rowId, int_t{NixInt{queryAttribute.getInt(2)}}}};
             case AttrType::ListOfStrings:
                 return {{rowId, tokenizeString<std::vector<std::string>>(queryAttribute.getStr(2), "\t")}};
             case AttrType::Missing:
@@ -322,7 +344,7 @@ struct AttrDb
 };
 
 static std::shared_ptr<AttrDb> makeAttrDb(
-    const Store & cfg,
+    const StoreDirConfig & cfg,
     const Hash & fingerprint,
     SymbolTable & symbols)
 {
@@ -385,8 +407,8 @@ Value & AttrCursor::getValue()
     if (!_value) {
         if (parent) {
             auto & vParent = parent->first->getValue();
-            root->state.forceAttrs(vParent, noPos);
-            auto attr = vParent.attrs->get(parent->second);
+            root->state.forceAttrs(vParent, noPos, "while searching for an attribute");
+            auto attr = vParent.attrs()->get(parent->second);
             if (!attr)
                 throw Error("attribute '%s' is unexpectedly missing", getAttrPathStr());
             _value = allocRootValue(attr->value);
@@ -415,12 +437,12 @@ std::vector<Symbol> AttrCursor::getAttrPath(Symbol name) const
 
 std::string AttrCursor::getAttrPathStr() const
 {
-    return concatStringsSep(".", root->state.symbols.resolve(getAttrPath()));
+    return dropEmptyInitThenConcatStringsSep(".", root->state.symbols.resolve(getAttrPath()));
 }
 
 std::string AttrCursor::getAttrPathStr(Symbol name) const
 {
-    return concatStringsSep(".", root->state.symbols.resolve(getAttrPath(name)));
+    return dropEmptyInitThenConcatStringsSep(".", root->state.symbols.resolve(getAttrPath(name)));
 }
 
 Value & AttrCursor::forceValue()
@@ -440,14 +462,16 @@ Value & AttrCursor::forceValue()
 
     if (root->db && (!cachedValue || std::get_if<placeholder_t>(&cachedValue->second))) {
         if (v.type() == nString)
-            cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context),
-                           string_t{v.string.s, {}}};
-        else if (v.type() == nPath)
-            cachedValue = {root->db->setString(getKey(), v.path), string_t{v.path, {}}};
+            cachedValue = {root->db->setString(getKey(), v.c_str(), v.context()),
+                           string_t{v.c_str(), {}}};
+        else if (v.type() == nPath) {
+            auto path = v.path().path;
+            cachedValue = {root->db->setString(getKey(), path.abs()), string_t{path.abs(), {}}};
+        }
         else if (v.type() == nBool)
-            cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean};
+            cachedValue = {root->db->setBool(getKey(), v.boolean()), v.boolean()};
         else if (v.type() == nInt)
-            cachedValue = {root->db->setInt(getKey(), v.integer), int_t{v.integer}};
+            cachedValue = {root->db->setInt(getKey(), v.integer().value), int_t{v.integer()}};
         else if (v.type() == nAttrs)
             ; // FIXME: do something?
         else
@@ -462,12 +486,12 @@ Suggestions AttrCursor::getSuggestionsForAttr(Symbol name)
     auto attrNames = getAttrs();
     std::set<std::string> strAttrNames;
     for (auto & name : attrNames)
-        strAttrNames.insert(root->state.symbols[name]);
+        strAttrNames.insert(std::string(root->state.symbols[name]));
 
     return Suggestions::bestMatches(strAttrNames, root->state.symbols[name]);
 }
 
-std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErrors)
+std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name)
 {
     if (root->db) {
         if (!cachedValue)
@@ -484,12 +508,9 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErro
                 if (attr) {
                     if (std::get_if<missing_t>(&attr->second))
                         return nullptr;
-                    else if (std::get_if<failed_t>(&attr->second)) {
-                        if (forceErrors)
-                            debug("reevaluating failed cached attribute '%s'", getAttrPathStr(name));
-                        else
-                            throw CachedEvalError("cached failure of attribute '%s'", getAttrPathStr(name));
-                    } else
+                    else if (std::get_if<failed_t>(&attr->second))
+                        throw CachedEvalError(ref(shared_from_this()), name);
+                    else
                         return std::make_shared<AttrCursor>(root,
                             std::make_pair(shared_from_this(), name), nullptr, std::move(attr));
                 }
@@ -497,7 +518,7 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErro
                 // evaluate to see whether 'name' exists
             } else
                 return nullptr;
-                //throw TypeError("'%s' is not an attribute set", getAttrPathStr());
+                //error<TypeError>("'%s' is not an attribute set", getAttrPathStr()).debugThrow();
         }
     }
 
@@ -505,9 +526,9 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErro
 
     if (v.type() != nAttrs)
         return nullptr;
-        //throw TypeError("'%s' is not an attribute set", getAttrPathStr());
+        //error<TypeError>("'%s' is not an attribute set", getAttrPathStr()).debugThrow();
 
-    auto attr = v.attrs->get(name);
+    auto attr = v.attrs()->get(name);
 
     if (!attr) {
         if (root->db) {
@@ -534,9 +555,9 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(std::string_view name)
     return maybeGetAttr(root->state.symbols.create(name));
 }
 
-ref<AttrCursor> AttrCursor::getAttr(Symbol name, bool forceErrors)
+ref<AttrCursor> AttrCursor::getAttr(Symbol name)
 {
-    auto p = maybeGetAttr(name, forceErrors);
+    auto p = maybeGetAttr(name);
     if (!p)
         throw Error("attribute '%s' does not exist", getAttrPathStr(name));
     return ref(p);
@@ -547,11 +568,11 @@ ref<AttrCursor> AttrCursor::getAttr(std::string_view name)
     return getAttr(root->state.symbols.create(name));
 }
 
-OrSuggestions<ref<AttrCursor>> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force)
+OrSuggestions<ref<AttrCursor>> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath)
 {
     auto res = shared_from_this();
     for (auto & attr : attrPath) {
-        auto child = res->maybeGetAttr(attr, force);
+        auto child = res->maybeGetAttr(attr);
         if (!child) {
             auto suggestions = res->getSuggestionsForAttr(attr);
             return OrSuggestions<ref<AttrCursor>>::failed(suggestions);
@@ -571,16 +592,16 @@ std::string AttrCursor::getString()
                 debug("using cached string attribute '%s'", getAttrPathStr());
                 return s->first;
             } else
-                root->state.debugThrowLastTrace(TypeError("'%s' is not a string", getAttrPathStr()));
+                root->state.error<TypeError>("'%s' is not a string", getAttrPathStr()).debugThrow();
         }
     }
 
     auto & v = forceValue();
 
     if (v.type() != nString && v.type() != nPath)
-        root->state.debugThrowLastTrace(TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type())));
+        root->state.error<TypeError>("'%s' is not a string but %s", getAttrPathStr(), showType(v)).debugThrow();
 
-    return v.type() == nString ? v.string.s : v.path;
+    return v.type() == nString ? v.c_str() : v.path().to_string();
 }
 
 string_t AttrCursor::getStringWithContext()
@@ -592,7 +613,18 @@ string_t AttrCursor::getStringWithContext()
             if (auto s = std::get_if<string_t>(&cachedValue->second)) {
                 bool valid = true;
                 for (auto & c : s->second) {
-                    if (!root->state.store->isValidPath(c.first)) {
+                    const StorePath & path = std::visit(overloaded {
+                        [&](const NixStringContextElem::DrvDeep & d) -> const StorePath & {
+                            return d.drvPath;
+                        },
+                        [&](const NixStringContextElem::Built & b) -> const StorePath & {
+                            return b.drvPath->getBaseStorePath();
+                        },
+                        [&](const NixStringContextElem::Opaque & o) -> const StorePath & {
+                            return o.path;
+                        },
+                    }, c.raw);
+                    if (!root->state.store->isValidPath(path)) {
                         valid = false;
                         break;
                     }
@@ -602,18 +634,21 @@ string_t AttrCursor::getStringWithContext()
                     return *s;
                 }
             } else
-                root->state.debugThrowLastTrace(TypeError("'%s' is not a string", getAttrPathStr()));
+                root->state.error<TypeError>("'%s' is not a string", getAttrPathStr()).debugThrow();
         }
     }
 
     auto & v = forceValue();
 
-    if (v.type() == nString)
-        return {v.string.s, v.getContext(*root->state.store)};
+    if (v.type() == nString) {
+        NixStringContext context;
+        copyContext(v, context);
+        return {v.c_str(), std::move(context)};
+    }
     else if (v.type() == nPath)
-        return {v.path, {}};
+        return {v.path().to_string(), {}};
     else
-        root->state.debugThrowLastTrace(TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type())));
+        root->state.error<TypeError>("'%s' is not a string but %s", getAttrPathStr(), showType(v)).debugThrow();
 }
 
 bool AttrCursor::getBool()
@@ -626,16 +661,16 @@ bool AttrCursor::getBool()
                 debug("using cached Boolean attribute '%s'", getAttrPathStr());
                 return *b;
             } else
-                root->state.debugThrowLastTrace(TypeError("'%s' is not a Boolean", getAttrPathStr()));
+                root->state.error<TypeError>("'%s' is not a Boolean", getAttrPathStr()).debugThrow();
         }
     }
 
     auto & v = forceValue();
 
     if (v.type() != nBool)
-        root->state.debugThrowLastTrace(TypeError("'%s' is not a Boolean", getAttrPathStr()));
+        root->state.error<TypeError>("'%s' is not a Boolean", getAttrPathStr()).debugThrow();
 
-    return v.boolean;
+    return v.boolean();
 }
 
 NixInt AttrCursor::getInt()
@@ -648,16 +683,16 @@ NixInt AttrCursor::getInt()
                 debug("using cached integer attribute '%s'", getAttrPathStr());
                 return i->x;
             } else
-                throw TypeError("'%s' is not an integer", getAttrPathStr());
+                root->state.error<TypeError>("'%s' is not an integer", getAttrPathStr()).debugThrow();
         }
     }
 
     auto & v = forceValue();
 
     if (v.type() != nInt)
-        throw TypeError("'%s' is not an integer", getAttrPathStr());
+        root->state.error<TypeError>("'%s' is not an integer", getAttrPathStr()).debugThrow();
 
-    return v.integer;
+    return v.integer();
 }
 
 std::vector<std::string> AttrCursor::getListOfStrings()
@@ -670,7 +705,7 @@ std::vector<std::string> AttrCursor::getListOfStrings()
                 debug("using cached list of strings attribute '%s'", getAttrPathStr());
                 return *l;
             } else
-                throw TypeError("'%s' is not a list of strings", getAttrPathStr());
+                root->state.error<TypeError>("'%s' is not a list of strings", getAttrPathStr()).debugThrow();
         }
     }
 
@@ -680,12 +715,12 @@ std::vector<std::string> AttrCursor::getListOfStrings()
     root->state.forceValue(v, noPos);
 
     if (v.type() != nList)
-        throw TypeError("'%s' is not a list", getAttrPathStr());
+        root->state.error<TypeError>("'%s' is not a list", getAttrPathStr()).debugThrow();
 
     std::vector<std::string> res;
 
     for (auto & elem : v.listItems())
-        res.push_back(std::string(root->state.forceStringNoCtx(*elem)));
+        res.push_back(std::string(root->state.forceStringNoCtx(*elem, noPos, "while evaluating an attribute for caching")));
 
     if (root->db)
         cachedValue = {root->db->setListOfStrings(getKey(), res), res};
@@ -703,17 +738,17 @@ std::vector<Symbol> AttrCursor::getAttrs()
                 debug("using cached attrset attribute '%s'", getAttrPathStr());
                 return *attrs;
             } else
-                root->state.debugThrowLastTrace(TypeError("'%s' is not an attribute set", getAttrPathStr()));
+                root->state.error<TypeError>("'%s' is not an attribute set", getAttrPathStr()).debugThrow();
         }
     }
 
     auto & v = forceValue();
 
     if (v.type() != nAttrs)
-        root->state.debugThrowLastTrace(TypeError("'%s' is not an attribute set", getAttrPathStr()));
+        root->state.error<TypeError>("'%s' is not an attribute set", getAttrPathStr()).debugThrow();
 
     std::vector<Symbol> attrs;
-    for (auto & attr : *getValue().attrs)
+    for (auto & attr : *getValue().attrs())
         attrs.push_back(attr.name);
     std::sort(attrs.begin(), attrs.end(), [&](Symbol a, Symbol b) {
         std::string_view sa = root->state.symbols[a], sb = root->state.symbols[b];
@@ -734,8 +769,9 @@ bool AttrCursor::isDerivation()
 
 StorePath AttrCursor::forceDerivation()
 {
-    auto aDrvPath = getAttr(root->state.sDrvPath, true);
+    auto aDrvPath = getAttr(root->state.sDrvPath);
     auto drvPath = root->state.store->parseStorePath(aDrvPath->getString());
+    drvPath.requireDerivation();
     if (!root->state.store->isValidPath(drvPath) && !settings.readOnlyMode) {
         /* The eval cache contains 'drvPath', but the actual path has
            been garbage-collected. So force it to be regenerated. */

src/libexpr/eval-cache.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "sync.hh"
 #include "hash.hh"
@@ -9,14 +10,28 @@
 
 namespace nix::eval_cache {
 
-MakeError(CachedEvalError, EvalError);
-
 struct AttrDb;
 class AttrCursor;
 
+struct CachedEvalError : EvalError
+{
+    const ref<AttrCursor> cursor;
+    const Symbol attr;
+
+    CachedEvalError(ref<AttrCursor> cursor, Symbol attr);
+
+    /**
+     * Evaluate this attribute, which should result in a regular
+     * `EvalError` exception being thrown.
+     */
+    [[noreturn]]
+    void force();
+};
+
 class EvalCache : public std::enable_shared_from_this<EvalCache>
 {
     friend class AttrCursor;
+    friend struct CachedEvalError;
 
     std::shared_ptr<AttrDb> db;
     EvalState & state;
@@ -72,6 +87,7 @@ typedef std::variant<
 class AttrCursor : public std::enable_shared_from_this<AttrCursor>
 {
     friend class EvalCache;
+    friend struct CachedEvalError;
 
     ref<EvalCache> root;
     typedef std::optional<std::pair<std::shared_ptr<AttrCursor>, Symbol>> Parent;
@@ -101,17 +117,19 @@ public:
 
     Suggestions getSuggestionsForAttr(Symbol name);
 
-    std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name, bool forceErrors = false);
+    std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name);
 
     std::shared_ptr<AttrCursor> maybeGetAttr(std::string_view name);
 
-    ref<AttrCursor> getAttr(Symbol name, bool forceErrors = false);
+    ref<AttrCursor> getAttr(Symbol name);
 
     ref<AttrCursor> getAttr(std::string_view name);
 
-    /* Get an attribute along a chain of attrsets. Note that this does
-       not auto-call functors or functions. */
-    OrSuggestions<ref<AttrCursor>> findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force = false);
+    /**
+     * Get an attribute along a chain of attrsets. Note that this does
+     * not auto-call functors or functions.
+     */
+    OrSuggestions<ref<AttrCursor>> findAlongAttrPath(const std::vector<Symbol> & attrPath);
 
     std::string getString();
 
@@ -129,7 +147,9 @@ public:
 
     Value & forceValue();
 
-    /* Force creation of the .drv file in the Nix store. */
+    /**
+     * Force creation of the .drv file in the Nix store.
+     */
     StorePath forceDerivation();
 };
 

src/libexpr/eval-error.cc

@@ -0,0 +1,114 @@
+#include "eval-error.hh"
+#include "eval.hh"
+#include "value.hh"
+
+namespace nix {
+
+template<class T>
+EvalErrorBuilder<T> & EvalErrorBuilder<T>::withExitStatus(unsigned int exitStatus)
+{
+    error.withExitStatus(exitStatus);
+    return *this;
+}
+
+template<class T>
+EvalErrorBuilder<T> & EvalErrorBuilder<T>::atPos(PosIdx pos)
+{
+    error.err.pos = error.state.positions[pos];
+    return *this;
+}
+
+template<class T>
+EvalErrorBuilder<T> & EvalErrorBuilder<T>::atPos(Value & value, PosIdx fallback)
+{
+    return atPos(value.determinePos(fallback));
+}
+
+template<class T>
+EvalErrorBuilder<T> & EvalErrorBuilder<T>::withTrace(PosIdx pos, const std::string_view text)
+{
+    error.addTrace(error.state.positions[pos], text);
+    return *this;
+}
+
+template<class T>
+EvalErrorBuilder<T> & EvalErrorBuilder<T>::withSuggestions(Suggestions & s)
+{
+    error.err.suggestions = s;
+    return *this;
+}
+
+template<class T>
+EvalErrorBuilder<T> & EvalErrorBuilder<T>::withFrame(const Env & env, const Expr & expr)
+{
+    // NOTE: This is abusing side-effects.
+    // TODO: check compatibility with nested debugger calls.
+    // TODO: What side-effects??
+    error.state.debugTraces.push_front(DebugTrace{
+        .pos = error.state.positions[expr.getPos()],
+        .expr = expr,
+        .env = env,
+        .hint = HintFmt("Fake frame for debugging purposes"),
+        .isError = true});
+    return *this;
+}
+
+template<class T>
+EvalErrorBuilder<T> & EvalErrorBuilder<T>::addTrace(PosIdx pos, HintFmt hint)
+{
+    error.addTrace(error.state.positions[pos], hint);
+    return *this;
+}
+
+template<class T>
+template<typename... Args>
+EvalErrorBuilder<T> &
+EvalErrorBuilder<T>::addTrace(PosIdx pos, std::string_view formatString, const Args &... formatArgs)
+{
+
+    addTrace(error.state.positions[pos], HintFmt(std::string(formatString), formatArgs...));
+    return *this;
+}
+
+template<class T>
+EvalErrorBuilder<T> & EvalErrorBuilder<T>::setIsFromExpr()
+{
+    error.err.isFromExpr = true;
+    return *this;
+}
+
+template<class T>
+void EvalErrorBuilder<T>::debugThrow()
+{
+    error.state.runDebugRepl(&error);
+
+    // `EvalState` is the only class that can construct an `EvalErrorBuilder`,
+    // and it does so in dynamic storage. This is the final method called on
+    // any such instance and must delete itself before throwing the underlying
+    // error.
+    auto error = std::move(this->error);
+    delete this;
+
+    throw error;
+}
+
+template<class T>
+void EvalErrorBuilder<T>::panic()
+{
+    logError(error.info());
+    printError("This is a bug! An unexpected condition occurred, causing the Nix evaluator to have to stop. If you could share a reproducible example or a core dump, please open an issue at https://github.com/NixOS/nix/issues");
+    abort();
+}
+
+template class EvalErrorBuilder<EvalBaseError>;
+template class EvalErrorBuilder<EvalError>;
+template class EvalErrorBuilder<AssertionError>;
+template class EvalErrorBuilder<ThrownError>;
+template class EvalErrorBuilder<Abort>;
+template class EvalErrorBuilder<TypeError>;
+template class EvalErrorBuilder<UndefinedVarError>;
+template class EvalErrorBuilder<MissingArgumentError>;
+template class EvalErrorBuilder<InfiniteRecursionError>;
+template class EvalErrorBuilder<InvalidPathError>;
+
+}

src/libexpr/eval-error.hh

@@ -0,0 +1,121 @@
+#pragma once
+
+#include "error.hh"
+#include "pos-idx.hh"
+
+namespace nix {
+
+struct Env;
+struct Expr;
+struct Value;
+
+class EvalState;
+template<class T>
+class EvalErrorBuilder;
+
+/**
+ * Base class for all errors that occur during evaluation.
+ *
+ * Most subclasses should inherit from `EvalError` instead of this class.
+ */
+class EvalBaseError : public Error
+{
+    template<class T>
+    friend class EvalErrorBuilder;
+public:
+    EvalState & state;
+
+    EvalBaseError(EvalState & state, ErrorInfo && errorInfo)
+        : Error(errorInfo)
+        , state(state)
+    {
+    }
+
+    template<typename... Args>
+    explicit EvalBaseError(EvalState & state, const std::string & formatString, const Args &... formatArgs)
+        : Error(formatString, formatArgs...)
+        , state(state)
+    {
+    }
+};
+
+/**
+ * `EvalError` is the base class for almost all errors that occur during evaluation.
+ *
+ * All instances of `EvalError` should show a degree of purity that allows them to be
+ * cached in pure mode. This means that they should not depend on the configuration or the overall environment.
+ */
+MakeError(EvalError, EvalBaseError);
+MakeError(ParseError, Error);
+MakeError(AssertionError, EvalError);
+MakeError(ThrownError, AssertionError);
+MakeError(Abort, EvalError);
+MakeError(TypeError, EvalError);
+MakeError(UndefinedVarError, EvalError);
+MakeError(MissingArgumentError, EvalError);
+MakeError(InfiniteRecursionError, EvalError);
+
+struct InvalidPathError : public EvalError
+{
+public:
+    Path path;
+    InvalidPathError(EvalState & state, const Path & path)
+        : EvalError(state, "path '%s' is not valid", path)
+    {
+    }
+};
+
+/**
+ * `EvalErrorBuilder`s may only be constructed by `EvalState`. The `debugThrow`
+ * method must be the final method in any such `EvalErrorBuilder` usage, and it
+ * handles deleting the object.
+ */
+template<class T>
+class EvalErrorBuilder final
+{
+    friend class EvalState;
+
+    template<typename... Args>
+    explicit EvalErrorBuilder(EvalState & state, const Args &... args)
+        : error(T(state, args...))
+    {
+    }
+
+public:
+    T error;
+
+    [[nodiscard, gnu::noinline]] EvalErrorBuilder<T> & withExitStatus(unsigned int exitStatus);
+
+    [[nodiscard, gnu::noinline]] EvalErrorBuilder<T> & atPos(PosIdx pos);
+
+    [[nodiscard, gnu::noinline]] EvalErrorBuilder<T> & atPos(Value & value, PosIdx fallback = noPos);
+
+    [[nodiscard, gnu::noinline]] EvalErrorBuilder<T> & withTrace(PosIdx pos, const std::string_view text);
+
+    [[nodiscard, gnu::noinline]] EvalErrorBuilder<T> & withFrameTrace(PosIdx pos, const std::string_view text);
+
+    [[nodiscard, gnu::noinline]] EvalErrorBuilder<T> & withSuggestions(Suggestions & s);
+
+    [[nodiscard, gnu::noinline]] EvalErrorBuilder<T> & withFrame(const Env & e, const Expr & ex);
+
+    [[nodiscard, gnu::noinline]] EvalErrorBuilder<T> & addTrace(PosIdx pos, HintFmt hint);
+
+    [[nodiscard, gnu::noinline]] EvalErrorBuilder<T> & setIsFromExpr();
+
+    template<typename... Args>
+    [[nodiscard, gnu::noinline]] EvalErrorBuilder<T> &
+    addTrace(PosIdx pos, std::string_view formatString, const Args &... formatArgs);
+
+    /**
+     * Delete the `EvalErrorBuilder` and throw the underlying exception.
+     */
+    [[gnu::noinline, gnu::noreturn]] void debugThrow();
+
+    /**
+     * A programming error or fatal condition occurred. Abort the process for core dump and debugging.
+     * This does not print a proper backtrace, because unwinding the stack is destructive.
+     */
+    [[gnu::noinline, gnu::noreturn]] void panic();
+};
+
+}

src/libexpr/eval-gc.cc

@@ -0,0 +1,117 @@
+#include "error.hh"
+#include "environment-variables.hh"
+#include "eval-settings.hh"
+#include "config-global.hh"
+#include "serialise.hh"
+#include "eval-gc.hh"
+
+#if HAVE_BOEHMGC
+
+#  include <pthread.h>
+#  if __FreeBSD__
+#    include <pthread_np.h>
+#  endif
+
+#  include <gc/gc.h>
+#  include <gc/gc_cpp.h>
+#  include <gc/gc_allocator.h>
+
+#  include <boost/coroutine2/coroutine.hpp>
+#  include <boost/coroutine2/protected_fixedsize_stack.hpp>
+#  include <boost/context/stack_context.hpp>
+
+#endif
+
+namespace nix {
+
+#if HAVE_BOEHMGC
+/* Called when the Boehm GC runs out of memory. */
+static void * oomHandler(size_t requested)
+{
+    /* Convert this to a proper C++ exception. */
+    throw std::bad_alloc();
+}
+
+static inline void initGCReal()
+{
+    /* Initialise the Boehm garbage collector. */
+
+    /* Don't look for interior pointers. This reduces the odds of
+       misdetection a bit. */
+    GC_set_all_interior_pointers(0);
+
+    /* We don't have any roots in data segments, so don't scan from
+       there. */
+    GC_set_no_dls(1);
+
+    /* Enable perf measurements. This is just a setting; not much of a
+       start of something. */
+    GC_start_performance_measurement();
+
+    GC_INIT();
+
+    GC_set_oom_fn(oomHandler);
+
+    /* Set the initial heap size to something fairly big (25% of
+       physical RAM, up to a maximum of 384 MiB) so that in most cases
+       we don't need to garbage collect at all.  (Collection has a
+       fairly significant overhead.)  The heap size can be overridden
+       through libgc's GC_INITIAL_HEAP_SIZE environment variable.  We
+       should probably also provide a nix.conf setting for this.  Note
+       that GC_expand_hp() causes a lot of virtual, but not physical
+       (resident) memory to be allocated.  This might be a problem on
+       systems that don't overcommit. */
+    if (!getEnv("GC_INITIAL_HEAP_SIZE")) {
+        size_t size = 32 * 1024 * 1024;
+#  if HAVE_SYSCONF && defined(_SC_PAGESIZE) && defined(_SC_PHYS_PAGES)
+        size_t maxSize = 384 * 1024 * 1024;
+        long pageSize = sysconf(_SC_PAGESIZE);
+        long pages = sysconf(_SC_PHYS_PAGES);
+        if (pageSize != -1)
+            size = (pageSize * pages) / 4; // 25% of RAM
+        if (size > maxSize)
+            size = maxSize;
+#  endif
+        debug("setting initial heap size to %1% bytes", size);
+        GC_expand_hp(size);
+    }
+}
+
+static size_t gcCyclesAfterInit = 0;
+
+size_t getGCCycles()
+{
+    assertGCInitialized();
+    return static_cast<size_t>(GC_get_gc_no()) - gcCyclesAfterInit;
+}
+
+#endif
+
+static bool gcInitialised = false;
+
+void initGC()
+{
+    if (gcInitialised)
+        return;
+
+#if HAVE_BOEHMGC
+    initGCReal();
+
+    gcCyclesAfterInit = GC_get_gc_no();
+#endif
+
+    // NIX_PATH must override the regular setting
+    // See the comment in applyConfig
+    if (auto nixPathEnv = getEnv("NIX_PATH")) {
+        globalConfig.set("nix-path", concatStringsSep(" ", EvalSettings::parseNixPath(nixPathEnv.value())));
+    }
+
+    gcInitialised = true;
+}
+
+void assertGCInitialized()
+{
+    assert(gcInitialised);
+}
+
+} // namespace nix

src/libexpr/eval-gc.hh

@@ -0,0 +1,25 @@
+#pragma once
+///@file
+
+#include <cstddef>
+
+namespace nix {
+
+/**
+ * Initialise the Boehm GC, if applicable.
+ */
+void initGC();
+
+/**
+ * Make sure `initGC` has already been called.
+ */
+void assertGCInitialized();
+
+#ifdef HAVE_BOEHMGC
+/**
+ * The number of GC cycles since initGC().
+ */
+size_t getGCCycles();
+#endif
+
+} // namespace nix

src/libexpr/eval-inline.hh

@@ -1,10 +1,15 @@
 #pragma once
+///@file
 
+#include "print.hh"
 #include "eval.hh"
+#include "eval-error.hh"
 
 namespace nix {
 
-/* Note: Various places expect the allocated memory to be zeroed. */
+/**
+ * Note: Various places expect the allocated memory to be zeroed.
+ */
 [[gnu::always_inline]]
 inline void * allocBytes(size_t n)
 {
@@ -70,8 +75,6 @@ Env & EvalState::allocEnv(size_t size)
 #endif
         env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *));
 
-    env->type = Env::Plain;
-
     /* We assume that env->values has been cleared by the allocator; maybeThunk() and lookupVar fromWith expect this. */
 
     return *env;
@@ -80,56 +83,59 @@ Env & EvalState::allocEnv(size_t size)
 
 [[gnu::always_inline]]
 void EvalState::forceValue(Value & v, const PosIdx pos)
-{
-    forceValue(v, [&]() { return pos; });
-}
-
-
-template<typename Callable>
-void EvalState::forceValue(Value & v, Callable getPos)
 {
     if (v.isThunk()) {
-        Env * env = v.thunk.env;
-        Expr * expr = v.thunk.expr;
+        Env * env = v.payload.thunk.env;
+        Expr * expr = v.payload.thunk.expr;
         try {
             v.mkBlackhole();
             //checkInterrupt();
             expr->eval(*this, *env, v);
         } catch (...) {
             v.mkThunk(env, expr);
+            tryFixupBlackHolePos(v, pos);
             throw;
         }
     }
     else if (v.isApp())
-        callFunction(*v.app.left, *v.app.right, v, noPos);
-    else if (v.isBlackhole())
-        throwEvalError(getPos(), "infinite recursion encountered");
+        callFunction(*v.payload.app.left, *v.payload.app.right, v, pos);
 }
 
 
 [[gnu::always_inline]]
-inline void EvalState::forceAttrs(Value & v, const PosIdx pos)
+inline void EvalState::forceAttrs(Value & v, const PosIdx pos, std::string_view errorCtx)
 {
-    forceAttrs(v, [&]() { return pos; });
+    forceAttrs(v, [&]() { return pos; }, errorCtx);
 }
 
 
 template <typename Callable>
 [[gnu::always_inline]]
-inline void EvalState::forceAttrs(Value & v, Callable getPos)
+inline void EvalState::forceAttrs(Value & v, Callable getPos, std::string_view errorCtx)
 {
-    forceValue(v, getPos);
-    if (v.type() != nAttrs)
-        throwTypeError(getPos(), "value is %1% while a set was expected", v);
+    PosIdx pos = getPos();
+    forceValue(v, pos);
+    if (v.type() != nAttrs) {
+        error<TypeError>(
+            "expected a set but found %1%: %2%",
+            showType(v),
+            ValuePrinter(*this, v, errorPrintOptions)
+        ).withTrace(pos, errorCtx).debugThrow();
+    }
 }
 
 
 [[gnu::always_inline]]
-inline void EvalState::forceList(Value & v, const PosIdx pos)
+inline void EvalState::forceList(Value & v, const PosIdx pos, std::string_view errorCtx)
 {
     forceValue(v, pos);
-    if (!v.isList())
-        throwTypeError(pos, "value is %1% while a list was expected", v);
+    if (!v.isList()) {
+        error<TypeError>(
+            "expected a list but found %1%: %2%",
+            showType(v),
+            ValuePrinter(*this, v, errorPrintOptions)
+        ).withTrace(pos, errorCtx).debugThrow();
+    }
 }
 
 

src/libexpr/eval-settings.cc

@@ -0,0 +1,106 @@
+#include "users.hh"
+#include "globals.hh"
+#include "profiles.hh"
+#include "eval.hh"
+#include "eval-settings.hh"
+
+namespace nix {
+
+/* Very hacky way to parse $NIX_PATH, which is colon-separated, but
+   can contain URLs (e.g. "nixpkgs=https://bla...:foo=https://"). */
+Strings EvalSettings::parseNixPath(const std::string & s)
+{
+    Strings res;
+
+    auto p = s.begin();
+
+    while (p != s.end()) {
+        auto start = p;
+        auto start2 = p;
+
+        while (p != s.end() && *p != ':') {
+            if (*p == '=') start2 = p + 1;
+            ++p;
+        }
+
+        if (p == s.end()) {
+            if (p != start) res.push_back(std::string(start, p));
+            break;
+        }
+
+        if (*p == ':') {
+            auto prefix = std::string(start2, s.end());
+            if (EvalSettings::isPseudoUrl(prefix) || hasPrefix(prefix, "flake:")) {
+                ++p;
+                while (p != s.end() && *p != ':') ++p;
+            }
+            res.push_back(std::string(start, p));
+            if (p == s.end()) break;
+        }
+
+        ++p;
+    }
+
+    return res;
+}
+
+EvalSettings::EvalSettings(bool & readOnlyMode, EvalSettings::LookupPathHooks lookupPathHooks)
+    : readOnlyMode{readOnlyMode}
+    , lookupPathHooks{lookupPathHooks}
+{
+    auto var = getEnv("NIX_ABORT_ON_WARN");
+    if (var && (var == "1" || var == "yes" || var == "true"))
+        builtinsAbortOnWarn = true;
+}
+
+Strings EvalSettings::getDefaultNixPath()
+{
+    Strings res;
+    auto add = [&](const Path & p, const std::string & s = std::string()) {
+        if (pathAccessible(p)) {
+            if (s.empty()) {
+                res.push_back(p);
+            } else {
+                res.push_back(s + "=" + p);
+            }
+        }
+    };
+
+    add(getNixDefExpr() + "/channels");
+    add(rootChannelsDir() + "/nixpkgs", "nixpkgs");
+    add(rootChannelsDir());
+
+    return res;
+}
+
+bool EvalSettings::isPseudoUrl(std::string_view s)
+{
+    if (s.compare(0, 8, "channel:") == 0) return true;
+    size_t pos = s.find("://");
+    if (pos == std::string::npos) return false;
+    std::string scheme(s, 0, pos);
+    return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh";
+}
+
+std::string EvalSettings::resolvePseudoUrl(std::string_view url)
+{
+    if (hasPrefix(url, "channel:"))
+        return "https://nixos.org/channels/" + std::string(url.substr(8)) + "/nixexprs.tar.xz";
+    else
+        return std::string(url);
+}
+
+const std::string & EvalSettings::getCurrentSystem() const
+{
+    const auto & evalSystem = currentSystem.get();
+    return evalSystem != "" ? evalSystem : settings.thisSystem.get();
+}
+
+Path getNixDefExpr()
+{
+    return settings.useXDGBaseDirectories
+        ? getStateDir() + "/nix/defexpr"
+        : getHome() + "/.nix-defexpr";
+}
+
+}

src/libexpr/eval-settings.hh

@@ -0,0 +1,248 @@
+#pragma once
+///@file
+
+#include "config.hh"
+#include "ref.hh"
+
+namespace nix {
+
+class Store;
+
+struct EvalSettings : Config
+{
+    /**
+     * Function used to interpet look path entries of a given scheme.
+     *
+     * The argument is the non-scheme part of the lookup path entry (see
+     * `LookupPathHooks` below).
+     *
+     * The return value is (a) whether the entry was valid, and, if so,
+     * what does it map to.
+     *
+     * @todo Return (`std::optional` of) `SourceAccssor` or something
+     * more structured instead of mere `std::string`?
+     */
+    using LookupPathHook = std::optional<std::string>(ref<Store> store, std::string_view);
+
+    /**
+     * Map from "scheme" to a `LookupPathHook`.
+     *
+     * Given a lookup path value (i.e. either the whole thing, or after
+     * the `<key>=`) in the form of:
+     *
+     * ```
+     * <scheme>:<arbitrary string>
+     * ```
+     *
+     * if `<scheme>` is a key in this map, then `<arbitrary string>` is
+     * passed to the hook that is the value in this map.
+     */
+    using LookupPathHooks = std::map<std::string, std::function<LookupPathHook>>;
+
+    EvalSettings(bool & readOnlyMode, LookupPathHooks lookupPathHooks = {});
+
+    bool & readOnlyMode;
+
+    static Strings getDefaultNixPath();
+
+    static bool isPseudoUrl(std::string_view s);
+
+    static Strings parseNixPath(const std::string & s);
+
+    static std::string resolvePseudoUrl(std::string_view url);
+
+    LookupPathHooks lookupPathHooks;
+
+    Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation", R"(
+        Enable built-in functions that allow executing native code.
+
+        In particular, this adds:
+        - `builtins.importNative` *path* *symbol*
+
+          Opens dynamic shared object (DSO) at *path*, loads the function with the symbol name *symbol* from it and runs it.
+          The loaded function must have the following signature:
+          ```cpp
+          extern "C" typedef void (*ValueInitialiser) (EvalState & state, Value & v);
+          ```
+
+          The [Nix C++ API documentation](@docroot@/development/documentation.md#api-documentation) has more details on evaluator internals.
+
+        - `builtins.exec` *arguments*
+
+          Execute a program, where *arguments* are specified as a list of strings, and parse its output as a Nix expression.
+    )"};
+
+    Setting<Strings> nixPath{
+        this, {}, "nix-path",
+        R"(
+          List of search paths to use for [lookup path](@docroot@/language/constructs/lookup-path.md) resolution.
+          This setting determines the value of
+          [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath) and can be used with [`builtins.findFile`](@docroot@/language/builtins.md#builtins-findFile).
+
+          - The configuration setting is overridden by the [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH)
+          environment variable.
+          - `NIX_PATH` is overridden by [specifying the setting as the command line flag](@docroot@/command-ref/conf-file.md#command-line-flags) `--nix-path`.
+          - Any current value is extended by the [`-I` option](@docroot@/command-ref/opt-common.md#opt-I) or `--extra-nix-path`.
+
+          If the respective paths are accessible, the default values are:
+
+          - `$HOME/.nix-defexpr/channels`
+          - `nixpkgs=$NIX_STATE_DIR/profiles/per-user/root/channels/nixpkgs`
+          - `$NIX_STATE_DIR/profiles/per-user/root/channels`
+
+          See [`NIX_STATE_DIR`](@docroot@/command-ref/env-common.md#env-NIX_STATE_DIR) for details.
+
+          > **Note**
+          >
+          > If [restricted evaluation](@docroot@/command-ref/conf-file.md#conf-restrict-eval) is enabled, the default value is empty.
+          >
+          > If [pure evaluation](#conf-pure-eval) is enabled, `builtins.nixPath` *always* evaluates to the empty list `[ ]`.
+        )", {}, false};
+
+    Setting<std::string> currentSystem{
+        this, "", "eval-system",
+        R"(
+          This option defines
+          [`builtins.currentSystem`](@docroot@/language/builtins.md#builtins-currentSystem)
+          in the Nix language if it is set as a non-empty string.
+          Otherwise, if it is defined as the empty string (the default), the value of the
+          [`system` ](#conf-system)
+          configuration setting is used instead.
+
+          Unlike `system`, this setting does not change what kind of derivations can be built locally.
+          This is useful for evaluating Nix code on one system to produce derivations to be built on another type of system.
+        )"};
+
+    /**
+     * Implements the `eval-system` vs `system` defaulting logic
+     * described for `eval-system`.
+     */
+    const std::string & getCurrentSystem() const;
+
+    Setting<bool> restrictEval{
+        this, false, "restrict-eval",
+        R"(
+          If set to `true`, the Nix evaluator will not allow access to any
+          files outside of
+          [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath),
+          or to URIs outside of
+          [`allowed-uris`](@docroot@/command-ref/conf-file.md#conf-allowed-uris).
+        )"};
+
+    Setting<bool> pureEval{this, false, "pure-eval",
+        R"(
+          Pure evaluation mode ensures that the result of Nix expressions is fully determined by explicitly declared inputs, and not influenced by external state:
+
+          - Restrict file system and network access to files specified by cryptographic hash
+          - Disable impure constants:
+            - [`builtins.currentSystem`](@docroot@/language/builtins.md#builtins-currentSystem)
+            - [`builtins.currentTime`](@docroot@/language/builtins.md#builtins-currentTime)
+            - [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath)
+            - [`builtins.storePath`](@docroot@/language/builtins.md#builtins-storePath)
+        )"
+        };
+
+    Setting<bool> enableImportFromDerivation{
+        this, true, "allow-import-from-derivation",
+        R"(
+          By default, Nix allows [Import from Derivation](@docroot@/language/import-from-derivation.md).
+
+          With this option set to `false`, Nix will throw an error when evaluating an expression that uses this feature,
+          even when the required store object is readily available.
+          This ensures that evaluation will not require any builds to take place,
+          regardless of the state of the store.
+        )"};
+
+    Setting<Strings> allowedUris{this, {}, "allowed-uris",
+        R"(
+          A list of URI prefixes to which access is allowed in restricted
+          evaluation mode. For example, when set to
+          `https://github.com/NixOS`, builtin functions such as `fetchGit` are
+          allowed to access `https://github.com/NixOS/patchelf.git`.
+
+          Access is granted when
+          - the URI is equal to the prefix,
+          - or the URI is a subpath of the prefix,
+          - or the prefix is a URI scheme ended by a colon `:` and the URI has the same scheme.
+        )"};
+
+    Setting<bool> traceFunctionCalls{this, false, "trace-function-calls",
+        R"(
+          If set to `true`, the Nix evaluator will trace every function call.
+          Nix will print a log message at the "vomit" level for every function
+          entrance and function exit.
+
+              function-trace entered undefined position at 1565795816999559622
+              function-trace exited undefined position at 1565795816999581277
+              function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150
+              function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684
+
+          The `undefined position` means the function call is a builtin.
+
+          Use the `contrib/stack-collapse.py` script distributed with the Nix
+          source code to convert the trace logs in to a format suitable for
+          `flamegraph.pl`.
+        )"};
+
+    Setting<bool> useEvalCache{this, true, "eval-cache",
+        R"(
+            Whether to use the flake evaluation cache.
+            Certain commands won't have to evaluate when invoked for the second time with a particular version of a flake.
+            Intermediate results are not cached.
+        )"};
+
+    Setting<bool> ignoreExceptionsDuringTry{this, false, "ignore-try",
+        R"(
+          If set to true, ignore exceptions inside 'tryEval' calls when evaluating nix expressions in
+          debug mode (using the --debugger flag). By default the debugger will pause on all exceptions.
+        )"};
+
+    Setting<bool> traceVerbose{this, false, "trace-verbose",
+        "Whether `builtins.traceVerbose` should trace its first argument when evaluated."};
+
+    Setting<unsigned int> maxCallDepth{this, 10000, "max-call-depth",
+        "The maximum function call depth to allow before erroring."};
+
+    Setting<bool> builtinsTraceDebugger{this, false, "debugger-on-trace",
+        R"(
+          If set to true and the `--debugger` flag is given, the following functions
+          will enter the debugger like [`builtins.break`](@docroot@/language/builtins.md#builtins-break).
+
+          * [`builtins.trace`](@docroot@/language/builtins.md#builtins-trace)
+          * [`builtins.traceVerbose`](@docroot@/language/builtins.md#builtins-traceVerbose)
+            if [`trace-verbose`](#conf-trace-verbose) is set to true.
+          * [`builtins.warn`](@docroot@/language/builtins.md#builtins-warn)
+
+          This is useful for debugging warnings in third-party Nix code.
+        )"};
+
+    Setting<bool> builtinsDebuggerOnWarn{this, false, "debugger-on-warn",
+        R"(
+          If set to true and the `--debugger` flag is given, [`builtins.warn`](@docroot@/language/builtins.md#builtins-warn)
+          will enter the debugger like [`builtins.break`](@docroot@/language/builtins.md#builtins-break).
+
+          This is useful for debugging warnings in third-party Nix code.
+
+          Use [`debugger-on-trace`](#conf-debugger-on-trace) to also enter the debugger on legacy warnings that are logged with [`builtins.trace`](@docroot@/language/builtins.md#builtins-trace).
+        )"};
+
+    Setting<bool> builtinsAbortOnWarn{this, false, "abort-on-warn",
+        R"(
+          If set to true, [`builtins.warn`](@docroot@/language/builtins.md#builtins-warn) will throw an error when logging a warning.
+
+          This will give you a stack trace that leads to the location of the warning.
+
+          This is useful for finding information about warnings in third-party Nix code when you can not start the interactive debugger, such as when Nix is called from a non-interactive script. See [`debugger-on-warn`](#conf-debugger-on-warn).
+
+          Currently, a stack trace can only be produced when the debugger is enabled, or when evaluation is aborted.
+
+          This option can be enabled by setting `NIX_ABORT_ON_WARN=1` in the environment.
+        )"};
+};
+
+/**
+ * Conventionally part of the default nix path in impure mode.
+ */
+Path getNixDefExpr();
+
+}

src/libexpr/fetchurl.nix

@@ -28,11 +28,8 @@ derivation ({
   # No need to double the amount of network traffic
   preferLocalBuild = true;
 
+  # This attribute does nothing; it's here to avoid changing evaluation results.
   impureEnvVars = [
-    # We borrow these environment variables from the caller to allow
-    # easy proxy configuration.  This is impure, but a fixed-output
-    # derivation like fetchurl is allowed to do so since its result is
-    # by definition pure.
     "http_proxy" "https_proxy" "ftp_proxy" "all_proxy" "no_proxy"
   ];
 

src/libexpr/flake/call-flake.nix

@@ -1,56 +0,0 @@
-lockFileStr: rootSrc: rootSubdir:
-
-let
-
-  lockFile = builtins.fromJSON lockFileStr;
-
-  allNodes =
-    builtins.mapAttrs
-      (key: node:
-        let
-
-          sourceInfo =
-            if key == lockFile.root
-            then rootSrc
-            else fetchTree (node.info or {} // removeAttrs node.locked ["dir"]);
-
-          subdir = if key == lockFile.root then rootSubdir else node.locked.dir or "";
-
-          flake = import (sourceInfo + (if subdir != "" then "/" else "") + subdir + "/flake.nix");
-
-          inputs = builtins.mapAttrs
-            (inputName: inputSpec: allNodes.${resolveInput inputSpec})
-            (node.inputs or {});
-
-          # Resolve a input spec into a node name. An input spec is
-          # either a node name, or a 'follows' path from the root
-          # node.
-          resolveInput = inputSpec:
-              if builtins.isList inputSpec
-              then getInputByPath lockFile.root inputSpec
-              else inputSpec;
-
-          # Follow an input path (e.g. ["dwarffs" "nixpkgs"]) from the
-          # root node, returning the final node.
-          getInputByPath = nodeName: path:
-            if path == []
-            then nodeName
-            else
-              getInputByPath
-                # Since this could be a 'follows' input, call resolveInput.
-                (resolveInput lockFile.nodes.${nodeName}.inputs.${builtins.head path})
-                (builtins.tail path);
-
-          outputs = flake.outputs (inputs // { self = result; });
-
-          result = outputs // sourceInfo // { inherit inputs; inherit outputs; inherit sourceInfo; _type = "flake"; };
-        in
-          if node.flake or true then
-            assert builtins.isFunction flake.outputs;
-            result
-          else
-            sourceInfo
-      )
-      lockFile.nodes;
-
-in allNodes.${lockFile.root}

src/libexpr/flake/flake.hh

@@ -1,148 +0,0 @@
-#pragma once
-
-#include "types.hh"
-#include "flakeref.hh"
-#include "lockfile.hh"
-#include "value.hh"
-
-namespace nix {
-
-class EvalState;
-
-namespace fetchers { struct Tree; }
-
-namespace flake {
-
-struct FlakeInput;
-
-typedef std::map<FlakeId, FlakeInput> FlakeInputs;
-
-/* FlakeInput is the 'Flake'-level parsed form of the "input" entries
- * in the flake file.
- *
- * A FlakeInput is normally constructed by the 'parseFlakeInput'
- * function which parses the input specification in the '.flake' file
- * to create a 'FlakeRef' (a fetcher, the fetcher-specific
- * representation of the input specification, and possibly the fetched
- * local store path result) and then creating this FlakeInput to hold
- * that FlakeRef, along with anything that might override that
- * FlakeRef (like command-line overrides or "follows" specifications).
- *
- * A FlakeInput is also sometimes constructed directly from a FlakeRef
- * instead of starting at the flake-file input specification
- * (e.g. overrides, follows, and implicit inputs).
- *
- * A FlakeInput will usually have one of either "ref" or "follows"
- * set.  If not otherwise specified, a "ref" will be generated to a
- * 'type="indirect"' flake, which is treated as simply the name of a
- * flake to be resolved in the registry.
- */
-
-struct FlakeInput
-{
-    std::optional<FlakeRef> ref;
-    bool isFlake = true;  // true = process flake to get outputs, false = (fetched) static source path
-    std::optional<InputPath> follows;
-    FlakeInputs overrides;
-};
-
-struct ConfigFile
-{
-    using ConfigValue = std::variant<std::string, int64_t, Explicit<bool>, std::vector<std::string>>;
-
-    std::map<std::string, ConfigValue> settings;
-
-    void apply();
-};
-
-/* The contents of a flake.nix file. */
-struct Flake
-{
-    FlakeRef originalRef; // the original flake specification (by the user)
-    FlakeRef resolvedRef; // registry references and caching resolved to the specific underlying flake
-    FlakeRef lockedRef; // the specific local store result of invoking the fetcher
-    bool forceDirty = false; // pretend that 'lockedRef' is dirty
-    std::optional<std::string> description;
-    std::shared_ptr<const fetchers::Tree> sourceInfo;
-    FlakeInputs inputs;
-    ConfigFile config; // 'nixConfig' attribute
-    ~Flake();
-};
-
-Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool allowLookup);
-
-/* Fingerprint of a locked flake; used as a cache key. */
-typedef Hash Fingerprint;
-
-struct LockedFlake
-{
-    Flake flake;
-    LockFile lockFile;
-
-    Fingerprint getFingerprint() const;
-};
-
-struct LockFlags
-{
-    /* Whether to ignore the existing lock file, creating a new one
-       from scratch. */
-    bool recreateLockFile = false;
-
-    /* Whether to update the lock file at all. If set to false, if any
-       change to the lock file is needed (e.g. when an input has been
-       added to flake.nix), you get a fatal error. */
-    bool updateLockFile = true;
-
-    /* Whether to write the lock file to disk. If set to true, if the
-       any changes to the lock file are needed and the flake is not
-       writable (i.e. is not a local Git working tree or similar), you
-       get a fatal error. If set to false, Nix will use the modified
-       lock file in memory only, without writing it to disk. */
-    bool writeLockFile = true;
-
-    /* Whether to use the registries to lookup indirect flake
-       references like 'nixpkgs'. */
-    std::optional<bool> useRegistries = std::nullopt;
-
-    /* Whether to apply flake's nixConfig attribute to the configuration */
-
-    bool applyNixConfig = false;
-
-    /* Whether unlocked flake references (i.e. those without a Git
-       revision or similar) without a corresponding lock are
-       allowed. Unlocked flake references with a lock are always
-       allowed. */
-    bool allowUnlocked = true;
-
-    /* Whether to commit changes to flake.lock. */
-    bool commitLockFile = false;
-
-    /* Flake inputs to be overridden. */
-    std::map<InputPath, FlakeRef> inputOverrides;
-
-    /* Flake inputs to be updated. This means that any existing lock
-       for those inputs will be ignored. */
-    std::set<InputPath> inputUpdates;
-};
-
-LockedFlake lockFlake(
-    EvalState & state,
-    const FlakeRef & flakeRef,
-    const LockFlags & lockFlags);
-
-void callFlake(
-    EvalState & state,
-    const LockedFlake & lockedFlake,
-    Value & v);
-
-}
-
-void emitTreeAttrs(
-    EvalState & state,
-    const fetchers::Tree & tree,
-    const fetchers::Input & input,
-    Value & v,
-    bool emptyRevFallback = false,
-    bool forceDirty = false);
-
-}

src/libexpr/flake/flakeref.cc

@@ -1,252 +0,0 @@
-#include "flakeref.hh"
-#include "store-api.hh"
-#include "url.hh"
-#include "url-parts.hh"
-#include "fetchers.hh"
-#include "registry.hh"
-
-namespace nix {
-
-#if 0
-// 'dir' path elements cannot start with a '.'. We also reject
-// potentially dangerous characters like ';'.
-const static std::string subDirElemRegex = "(?:[a-zA-Z0-9_-]+[a-zA-Z0-9._-]*)";
-const static std::string subDirRegex = subDirElemRegex + "(?:/" + subDirElemRegex + ")*";
-#endif
-
-std::string FlakeRef::to_string() const
-{
-    std::map<std::string, std::string> extraQuery;
-    if (subdir != "")
-        extraQuery.insert_or_assign("dir", subdir);
-    return input.toURLString(extraQuery);
-}
-
-fetchers::Attrs FlakeRef::toAttrs() const
-{
-    auto attrs = input.toAttrs();
-    if (subdir != "")
-        attrs.emplace("dir", subdir);
-    return attrs;
-}
-
-std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef)
-{
-    str << flakeRef.to_string();
-    return str;
-}
-
-bool FlakeRef::operator ==(const FlakeRef & other) const
-{
-    return input == other.input && subdir == other.subdir;
-}
-
-FlakeRef FlakeRef::resolve(ref<Store> store) const
-{
-    auto [input2, extraAttrs] = lookupInRegistries(store, input);
-    return FlakeRef(std::move(input2), fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir));
-}
-
-FlakeRef parseFlakeRef(
-    const std::string & url,
-    const std::optional<Path> & baseDir,
-    bool allowMissing,
-    bool isFlake)
-{
-    auto [flakeRef, fragment] = parseFlakeRefWithFragment(url, baseDir, allowMissing, isFlake);
-    if (fragment != "")
-        throw Error("unexpected fragment '%s' in flake reference '%s'", fragment, url);
-    return flakeRef;
-}
-
-std::optional<FlakeRef> maybeParseFlakeRef(
-    const std::string & url, const std::optional<Path> & baseDir)
-{
-    try {
-        return parseFlakeRef(url, baseDir);
-    } catch (Error &) {
-        return {};
-    }
-}
-
-std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
-    const std::string & url,
-    const std::optional<Path> & baseDir,
-    bool allowMissing,
-    bool isFlake)
-{
-    using namespace fetchers;
-
-    static std::string fnRegex = "[0-9a-zA-Z-._~!$&'\"()*+,;=]+";
-
-    static std::regex pathUrlRegex(
-        "(/?" + fnRegex + "(?:/" + fnRegex + ")*/?)"
-        + "(?:\\?(" + queryRegex + "))?"
-        + "(?:#(" + queryRegex + "))?",
-        std::regex::ECMAScript);
-
-    static std::regex flakeRegex(
-        "((" + flakeIdRegexS + ")(?:/(?:" + refAndOrRevRegex + "))?)"
-        + "(?:#(" + queryRegex + "))?",
-        std::regex::ECMAScript);
-
-    std::smatch match;
-
-    /* Check if 'url' is a flake ID. This is an abbreviated syntax for
-       'flake:<flake-id>?ref=<ref>&rev=<rev>'. */
-
-    if (std::regex_match(url, match, flakeRegex)) {
-        auto parsedURL = ParsedURL{
-            .url = url,
-            .base = "flake:" + match.str(1),
-            .scheme = "flake",
-            .authority = "",
-            .path = match[1],
-        };
-
-        return std::make_pair(
-            FlakeRef(Input::fromURL(parsedURL), ""),
-            percentDecode(match.str(6)));
-    }
-
-    else if (std::regex_match(url, match, pathUrlRegex)) {
-        std::string path = match[1];
-        std::string fragment = percentDecode(match.str(3));
-
-        if (baseDir) {
-            /* Check if 'url' is a path (either absolute or relative
-               to 'baseDir'). If so, search upward to the root of the
-               repo (i.e. the directory containing .git). */
-
-            path = absPath(path, baseDir);
-
-            if (isFlake) {
-
-                if (!allowMissing && !pathExists(path + "/flake.nix")){
-                    notice("path '%s' does not contain a 'flake.nix', searching up",path);
-
-                    // Save device to detect filesystem boundary
-                    dev_t device = lstat(path).st_dev;
-                    bool found = false;
-                    while (path != "/") {
-                        if (pathExists(path + "/flake.nix")) {
-                            found = true;
-                            break;
-                        } else if (pathExists(path + "/.git"))
-                            throw Error("path '%s' is not part of a flake (neither it nor its parent directories contain a 'flake.nix' file)", path);
-                        else {
-                            if (lstat(path).st_dev != device)
-                                throw Error("unable to find a flake before encountering filesystem boundary at '%s'", path);
-                        }
-                        path = dirOf(path);
-                    }
-                    if (!found)
-                        throw BadURL("could not find a flake.nix file");
-                }
-
-                if (!S_ISDIR(lstat(path).st_mode))
-                    throw BadURL("path '%s' is not a flake (because it's not a directory)", path);
-
-                if (!allowMissing && !pathExists(path + "/flake.nix"))
-                    throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path);
-
-                auto flakeRoot = path;
-                std::string subdir;
-
-                while (flakeRoot != "/") {
-                    if (pathExists(flakeRoot + "/.git")) {
-                        auto base = std::string("git+file://") + flakeRoot;
-
-                        auto parsedURL = ParsedURL{
-                            .url = base, // FIXME
-                            .base = base,
-                            .scheme = "git+file",
-                            .authority = "",
-                            .path = flakeRoot,
-                            .query = decodeQuery(match[2]),
-                        };
-
-                        if (subdir != "") {
-                            if (parsedURL.query.count("dir"))
-                                throw Error("flake URL '%s' has an inconsistent 'dir' parameter", url);
-                            parsedURL.query.insert_or_assign("dir", subdir);
-                        }
-
-                        if (pathExists(flakeRoot + "/.git/shallow"))
-                            parsedURL.query.insert_or_assign("shallow", "1");
-
-                        return std::make_pair(
-                            FlakeRef(Input::fromURL(parsedURL), getOr(parsedURL.query, "dir", "")),
-                            fragment);
-                    }
-
-                    subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir);
-                    flakeRoot = dirOf(flakeRoot);
-                }
-            }
-
-        } else {
-            if (!hasPrefix(path, "/"))
-                throw BadURL("flake reference '%s' is not an absolute path", url);
-            auto query = decodeQuery(match[2]);
-            path = canonPath(path + "/" + getOr(query, "dir", ""));
-        }
-
-        fetchers::Attrs attrs;
-        attrs.insert_or_assign("type", "path");
-        attrs.insert_or_assign("path", path);
-
-        return std::make_pair(FlakeRef(Input::fromAttrs(std::move(attrs)), ""), fragment);
-    }
-
-    else {
-        auto parsedURL = parseURL(url);
-        std::string fragment;
-        std::swap(fragment, parsedURL.fragment);
-
-        auto input = Input::fromURL(parsedURL);
-        input.parent = baseDir;
-
-        return std::make_pair(
-            FlakeRef(std::move(input), getOr(parsedURL.query, "dir", "")),
-            fragment);
-    }
-}
-
-std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment(
-    const std::string & url, const std::optional<Path> & baseDir)
-{
-    try {
-        return parseFlakeRefWithFragment(url, baseDir);
-    } catch (Error & e) {
-        return {};
-    }
-}
-
-FlakeRef FlakeRef::fromAttrs(const fetchers::Attrs & attrs)
-{
-    auto attrs2(attrs);
-    attrs2.erase("dir");
-    return FlakeRef(
-        fetchers::Input::fromAttrs(std::move(attrs2)),
-        fetchers::maybeGetStrAttr(attrs, "dir").value_or(""));
-}
-
-std::pair<fetchers::Tree, FlakeRef> FlakeRef::fetchTree(ref<Store> store) const
-{
-    auto [tree, lockedInput] = input.fetch(store);
-    return {std::move(tree), FlakeRef(std::move(lockedInput), subdir)};
-}
-
-std::tuple<FlakeRef, std::string, OutputsSpec> parseFlakeRefWithFragmentAndOutputsSpec(
-    const std::string & url,
-    const std::optional<Path> & baseDir,
-    bool allowMissing,
-    bool isFlake)
-{
-    auto [prefix, outputsSpec] = parseOutputsSpec(url);
-    auto [flakeRef, fragment] = parseFlakeRefWithFragment(prefix, baseDir, allowMissing, isFlake);
-    return {std::move(flakeRef), fragment, outputsSpec};
-}
-
-}

src/libexpr/function-trace.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "eval.hh"
 

src/libexpr/gc-small-vector.hh

@@ -0,0 +1,42 @@
+#pragma once
+
+#include <boost/container/small_vector.hpp>
+
+#if HAVE_BOEHMGC
+
+#include <gc/gc.h>
+#include <gc/gc_cpp.h>
+#include <gc/gc_allocator.h>
+
+#endif
+
+namespace nix {
+
+struct Value;
+
+/**
+ * A GC compatible vector that may used a reserved portion of `nItems` on the stack instead of allocating on the heap.
+ */
+#if HAVE_BOEHMGC
+template <typename T, size_t nItems>
+using SmallVector = boost::container::small_vector<T, nItems, traceable_allocator<T>>;
+#else
+template <typename T, size_t nItems>
+using SmallVector = boost::container::small_vector<T, nItems>;
+#endif
+
+/**
+ * A vector of value pointers. See `SmallVector`.
+ */
+template <size_t nItems>
+using SmallValueVector = SmallVector<Value *, nItems>;
+
+/**
+ * A vector of values that must not be referenced after the vector is destroyed.
+ *
+ * See also `SmallValueVector`.
+ */
+template <size_t nItems>
+using SmallTemporaryValueVector = SmallVector<Value, nItems>;
+
+}

src/libexpr/get-drvs.cc

@@ -1,5 +1,4 @@
 #include "get-drvs.hh"
-#include "util.hh"
 #include "eval-inline.hh"
 #include "derivations.hh"
 #include "store-api.hh"
@@ -12,13 +11,13 @@
 namespace nix {
 
 
-DrvInfo::DrvInfo(EvalState & state, std::string attrPath, Bindings * attrs)
+PackageInfo::PackageInfo(EvalState & state, std::string attrPath, const Bindings * attrs)
     : state(&state), attrs(attrs), attrPath(std::move(attrPath))
 {
 }
 
 
-DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs)
+PackageInfo::PackageInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs)
     : state(&state), attrs(nullptr), attrPath("")
 {
     auto [drvPath, selectedOutputs] = parsePathWithOutputs(*store, drvPathWithOutputs);
@@ -46,42 +45,48 @@ DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPat
 }
 
 
-std::string DrvInfo::queryName() const
+std::string PackageInfo::queryName() const
 {
     if (name == "" && attrs) {
         auto i = attrs->find(state->sName);
-        if (i == attrs->end()) throw TypeError("derivation name missing");
-        name = state->forceStringNoCtx(*i->value);
+        if (i == attrs->end()) state->error<TypeError>("derivation name missing").debugThrow();
+        name = state->forceStringNoCtx(*i->value, noPos, "while evaluating the 'name' attribute of a derivation");
     }
     return name;
 }
 
 
-std::string DrvInfo::querySystem() const
+std::string PackageInfo::querySystem() const
 {
     if (system == "" && attrs) {
         auto i = attrs->find(state->sSystem);
-        system = i == attrs->end() ? "unknown" : state->forceStringNoCtx(*i->value, i->pos);
+        system = i == attrs->end() ? "unknown" : state->forceStringNoCtx(*i->value, i->pos, "while evaluating the 'system' attribute of a derivation");
     }
     return system;
 }
 
 
-std::optional<StorePath> DrvInfo::queryDrvPath() const
+std::optional<StorePath> PackageInfo::queryDrvPath() const
 {
     if (!drvPath && attrs) {
-        Bindings::iterator i = attrs->find(state->sDrvPath);
-        PathSet context;
-        if (i == attrs->end())
+        if (auto i = attrs->get(state->sDrvPath)) {
+            NixStringContext context;
+            auto found = state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the 'drvPath' attribute of a derivation");
+            try {
+                found.requireDerivation();
+            } catch (Error & e) {
+                e.addTrace(state->positions[i->pos], "while evaluating the 'drvPath' attribute of a derivation");
+                throw;
+            }
+            drvPath = {std::move(found)};
+        } else
             drvPath = {std::nullopt};
-        else
-            drvPath = {state->coerceToStorePath(i->pos, *i->value, context)};
     }
     return drvPath.value_or(std::nullopt);
 }
 
 
-StorePath DrvInfo::requireDrvPath() const
+StorePath PackageInfo::requireDrvPath() const
 {
     if (auto drvPath = queryDrvPath())
         return *drvPath;
@@ -89,13 +94,13 @@ StorePath DrvInfo::requireDrvPath() const
 }
 
 
-StorePath DrvInfo::queryOutPath() const
+StorePath PackageInfo::queryOutPath() const
 {
     if (!outPath && attrs) {
-        Bindings::iterator i = attrs->find(state->sOutPath);
-        PathSet context;
+        auto i = attrs->find(state->sOutPath);
+        NixStringContext context;
         if (i != attrs->end())
-            outPath = state->coerceToStorePath(i->pos, *i->value, context);
+            outPath = state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the output path of a derivation");
     }
     if (!outPath)
         throw UnimplementedError("CA derivations are not yet supported");
@@ -103,29 +108,29 @@ StorePath DrvInfo::queryOutPath() const
 }
 
 
-DrvInfo::Outputs DrvInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall)
+PackageInfo::Outputs PackageInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall)
 {
     if (outputs.empty()) {
         /* Get the ‘outputs’ list. */
-        Bindings::iterator i;
-        if (attrs && (i = attrs->find(state->sOutputs)) != attrs->end()) {
-            state->forceList(*i->value, i->pos);
+        const Attr * i;
+        if (attrs && (i = attrs->get(state->sOutputs))) {
+            state->forceList(*i->value, i->pos, "while evaluating the 'outputs' attribute of a derivation");
 
             /* For each output... */
             for (auto elem : i->value->listItems()) {
-                std::string output(state->forceStringNoCtx(*elem, i->pos));
+                std::string output(state->forceStringNoCtx(*elem, i->pos, "while evaluating the name of an output of a derivation"));
 
                 if (withPaths) {
                     /* Evaluate the corresponding set. */
-                    Bindings::iterator out = attrs->find(state->symbols.create(output));
-                    if (out == attrs->end()) continue; // FIXME: throw error?
-                    state->forceAttrs(*out->value, i->pos);
+                    auto out = attrs->get(state->symbols.create(output));
+                    if (!out) continue; // FIXME: throw error?
+                    state->forceAttrs(*out->value, i->pos, "while evaluating an output of a derivation");
 
                     /* And evaluate its ‘outPath’ attribute. */
-                    Bindings::iterator outPath = out->value->attrs->find(state->sOutPath);
-                    if (outPath == out->value->attrs->end()) continue; // FIXME: throw error?
-                    PathSet context;
-                    outputs.emplace(output, state->coerceToStorePath(outPath->pos, *outPath->value, context));
+                    auto outPath = out->value->attrs()->get(state->sOutPath);
+                    if (!outPath) continue; // FIXME: throw error?
+                    NixStringContext context;
+                    outputs.emplace(output, state->coerceToStorePath(outPath->pos, *outPath->value, context, "while evaluating an output path of a derivation"));
                 } else
                     outputs.emplace(output, std::nullopt);
             }
@@ -136,8 +141,8 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall
     if (!onlyOutputsToInstall || !attrs)
         return outputs;
 
-    Bindings::iterator i;
-    if (attrs && (i = attrs->find(state->sOutputSpecified)) != attrs->end() && state->forceBool(*i->value, i->pos)) {
+    const Attr * i;
+    if (attrs && (i = attrs->get(state->sOutputSpecified)) && state->forceBool(*i->value, i->pos, "while evaluating the 'outputSpecified' attribute of a derivation")) {
         Outputs result;
         auto out = outputs.find(queryOutputName());
         if (out == outputs.end())
@@ -156,7 +161,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall
         Outputs result;
         for (auto elem : outTI->listItems()) {
             if (elem->type() != nString) throw errMsg;
-            auto out = outputs.find(elem->string.s);
+            auto out = outputs.find(elem->c_str());
             if (out == outputs.end()) throw errMsg;
             result.insert(*out);
         }
@@ -165,29 +170,29 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall
 }
 
 
-std::string DrvInfo::queryOutputName() const
+std::string PackageInfo::queryOutputName() const
 {
     if (outputName == "" && attrs) {
-        Bindings::iterator i = attrs->find(state->sOutputName);
-        outputName = i != attrs->end() ? state->forceStringNoCtx(*i->value) : "";
+        auto i = attrs->get(state->sOutputName);
+        outputName = i ? state->forceStringNoCtx(*i->value, noPos, "while evaluating the output name of a derivation") : "";
     }
     return outputName;
 }
 
 
-Bindings * DrvInfo::getMeta()
+const Bindings * PackageInfo::getMeta()
 {
     if (meta) return meta;
     if (!attrs) return 0;
-    Bindings::iterator a = attrs->find(state->sMeta);
-    if (a == attrs->end()) return 0;
-    state->forceAttrs(*a->value, a->pos);
-    meta = a->value->attrs;
+    auto a = attrs->get(state->sMeta);
+    if (!a) return 0;
+    state->forceAttrs(*a->value, a->pos, "while evaluating the 'meta' attribute of a derivation");
+    meta = a->value->attrs();
     return meta;
 }
 
 
-StringSet DrvInfo::queryMetaNames()
+StringSet PackageInfo::queryMetaNames()
 {
     StringSet res;
     if (!getMeta()) return res;
@@ -197,18 +202,17 @@ StringSet DrvInfo::queryMetaNames()
 }
 
 
-bool DrvInfo::checkMeta(Value & v)
+bool PackageInfo::checkMeta(Value & v)
 {
-    state->forceValue(v, [&]() { return v.determinePos(noPos); });
+    state->forceValue(v, v.determinePos(noPos));
     if (v.type() == nList) {
         for (auto elem : v.listItems())
             if (!checkMeta(*elem)) return false;
         return true;
     }
     else if (v.type() == nAttrs) {
-        Bindings::iterator i = v.attrs->find(state->sOutPath);
-        if (i != v.attrs->end()) return false;
-        for (auto & i : *v.attrs)
+        if (v.attrs()->get(state->sOutPath)) return false;
+        for (auto & i : *v.attrs())
             if (!checkMeta(*i.value)) return false;
         return true;
     }
@@ -217,68 +221,68 @@ bool DrvInfo::checkMeta(Value & v)
 }
 
 
-Value * DrvInfo::queryMeta(const std::string & name)
+Value * PackageInfo::queryMeta(const std::string & name)
 {
     if (!getMeta()) return 0;
-    Bindings::iterator a = meta->find(state->symbols.create(name));
-    if (a == meta->end() || !checkMeta(*a->value)) return 0;
+    auto a = meta->get(state->symbols.create(name));
+    if (!a || !checkMeta(*a->value)) return 0;
     return a->value;
 }
 
 
-std::string DrvInfo::queryMetaString(const std::string & name)
+std::string PackageInfo::queryMetaString(const std::string & name)
 {
     Value * v = queryMeta(name);
     if (!v || v->type() != nString) return "";
-    return v->string.s;
+    return v->c_str();
 }
 
 
-NixInt DrvInfo::queryMetaInt(const std::string & name, NixInt def)
+NixInt PackageInfo::queryMetaInt(const std::string & name, NixInt def)
 {
     Value * v = queryMeta(name);
     if (!v) return def;
-    if (v->type() == nInt) return v->integer;
+    if (v->type() == nInt) return v->integer();
     if (v->type() == nString) {
         /* Backwards compatibility with before we had support for
            integer meta fields. */
-        if (auto n = string2Int<NixInt>(v->string.s))
-            return *n;
+        if (auto n = string2Int<NixInt::Inner>(v->c_str()))
+            return NixInt{*n};
     }
     return def;
 }
 
-NixFloat DrvInfo::queryMetaFloat(const std::string & name, NixFloat def)
+NixFloat PackageInfo::queryMetaFloat(const std::string & name, NixFloat def)
 {
     Value * v = queryMeta(name);
     if (!v) return def;
-    if (v->type() == nFloat) return v->fpoint;
+    if (v->type() == nFloat) return v->fpoint();
     if (v->type() == nString) {
         /* Backwards compatibility with before we had support for
            float meta fields. */
-        if (auto n = string2Float<NixFloat>(v->string.s))
+        if (auto n = string2Float<NixFloat>(v->c_str()))
             return *n;
     }
     return def;
 }
 
 
-bool DrvInfo::queryMetaBool(const std::string & name, bool def)
+bool PackageInfo::queryMetaBool(const std::string & name, bool def)
 {
     Value * v = queryMeta(name);
     if (!v) return def;
-    if (v->type() == nBool) return v->boolean;
+    if (v->type() == nBool) return v->boolean();
     if (v->type() == nString) {
         /* Backwards compatibility with before we had support for
            Boolean meta fields. */
-        if (strcmp(v->string.s, "true") == 0) return true;
-        if (strcmp(v->string.s, "false") == 0) return false;
+        if (v->string_view() == "true") return true;
+        if (v->string_view() == "false") return false;
     }
     return def;
 }
 
 
-void DrvInfo::setMeta(const std::string & name, Value * v)
+void PackageInfo::setMeta(const std::string & name, Value * v)
 {
     getMeta();
     auto attrs = state->buildBindings(1 + (meta ? meta->size() : 0));
@@ -293,7 +297,7 @@ void DrvInfo::setMeta(const std::string & name, Value * v)
 
 
 /* Cache for already considered attrsets. */
-typedef std::set<Bindings *> Done;
+typedef std::set<const Bindings *> Done;
 
 
 /* Evaluate value `v'.  If it evaluates to a set of type `derivation',
@@ -301,18 +305,18 @@ typedef std::set<Bindings *> Done;
    The result boolean indicates whether it makes sense
    for the caller to recursively search for derivations in `v'. */
 static bool getDerivation(EvalState & state, Value & v,
-    const std::string & attrPath, DrvInfos & drvs, Done & done,
+    const std::string & attrPath, PackageInfos & drvs, Done & done,
     bool ignoreAssertionFailures)
 {
     try {
-        state.forceValue(v, [&]() { return v.determinePos(noPos); });
+        state.forceValue(v, v.determinePos(noPos));
         if (!state.isDerivation(v)) return true;
 
         /* Remove spurious duplicates (e.g., a set like `rec { x =
            derivation {...}; y = x;}'. */
-        if (!done.insert(v.attrs).second) return false;
+        if (!done.insert(v.attrs()).second) return false;
 
-        DrvInfo drv(state, attrPath, v.attrs);
+        PackageInfo drv(state, attrPath, v.attrs());
 
         drv.queryName();
 
@@ -327,20 +331,20 @@ static bool getDerivation(EvalState & state, Value & v,
 }
 
 
-std::optional<DrvInfo> getDerivation(EvalState & state, Value & v,
+std::optional<PackageInfo> getDerivation(EvalState & state, Value & v,
     bool ignoreAssertionFailures)
 {
     Done done;
-    DrvInfos drvs;
+    PackageInfos drvs;
     getDerivation(state, v, "", drvs, done, ignoreAssertionFailures);
     if (drvs.size() != 1) return {};
     return std::move(drvs.front());
 }
 
 
-static std::string addToPath(const std::string & s1, const std::string & s2)
+static std::string addToPath(const std::string & s1, std::string_view s2)
 {
-    return s1.empty() ? s2 : s1 + "." + s2;
+    return s1.empty() ? std::string(s2) : s1 + "." + s2;
 }
 
 
@@ -349,7 +353,7 @@ static std::regex attrRegex("[A-Za-z_][A-Za-z0-9-_+]*");
 
 static void getDerivations(EvalState & state, Value & vIn,
     const std::string & pathPrefix, Bindings & autoArgs,
-    DrvInfos & drvs, Done & done,
+    PackageInfos & drvs, Done & done,
     bool ignoreAssertionFailures)
 {
     Value v;
@@ -362,29 +366,34 @@ static void getDerivations(EvalState & state, Value & vIn,
 
         /* !!! undocumented hackery to support combining channels in
            nix-env.cc. */
-        bool combineChannels = v.attrs->find(state.symbols.create("_combineChannels")) != v.attrs->end();
+        bool combineChannels = v.attrs()->get(state.symbols.create("_combineChannels"));
 
         /* Consider the attributes in sorted order to get more
            deterministic behaviour in nix-env operations (e.g. when
            there are names clashes between derivations, the derivation
            bound to the attribute with the "lower" name should take
            precedence). */
-        for (auto & i : v.attrs->lexicographicOrder(state.symbols)) {
-            debug("evaluating attribute '%1%'", state.symbols[i->name]);
-            if (!std::regex_match(std::string(state.symbols[i->name]), attrRegex))
-                continue;
-            std::string pathPrefix2 = addToPath(pathPrefix, state.symbols[i->name]);
-            if (combineChannels)
-                getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
-            else if (getDerivation(state, *i->value, pathPrefix2, drvs, done, ignoreAssertionFailures)) {
-                /* If the value of this attribute is itself a set,
-                   should we recurse into it?  => Only if it has a
-                   `recurseForDerivations = true' attribute. */
-                if (i->value->type() == nAttrs) {
-                    Bindings::iterator j = i->value->attrs->find(state.sRecurseForDerivations);
-                    if (j != i->value->attrs->end() && state.forceBool(*j->value, j->pos))
-                        getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
+        for (auto & i : v.attrs()->lexicographicOrder(state.symbols)) {
+            try {
+                debug("evaluating attribute '%1%'", state.symbols[i->name]);
+                if (!std::regex_match(std::string(state.symbols[i->name]), attrRegex))
+                    continue;
+                std::string pathPrefix2 = addToPath(pathPrefix, state.symbols[i->name]);
+                if (combineChannels)
+                    getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
+                else if (getDerivation(state, *i->value, pathPrefix2, drvs, done, ignoreAssertionFailures)) {
+                    /* If the value of this attribute is itself a set,
+                    should we recurse into it?  => Only if it has a
+                    `recurseForDerivations = true' attribute. */
+                    if (i->value->type() == nAttrs) {
+                        auto j = i->value->attrs()->get(state.sRecurseForDerivations);
+                        if (j && state.forceBool(*j->value, j->pos, "while evaluating the attribute `recurseForDerivations`"))
+                            getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
+                    }
                 }
+            } catch (Error & e) {
+                e.addTrace(state.positions[i->pos], "while evaluating the attribute '%s'", state.symbols[i->name]);
+                throw;
             }
         }
     }
@@ -397,12 +406,13 @@ static void getDerivations(EvalState & state, Value & vIn,
         }
     }
 
-    else throw TypeError("expression does not evaluate to a derivation (or a set or list of those)");
+    else
+        state.error<TypeError>("expression does not evaluate to a derivation (or a set or list of those)").debugThrow();
 }
 
 
 void getDerivations(EvalState & state, Value & v, const std::string & pathPrefix,
-    Bindings & autoArgs, DrvInfos & drvs, bool ignoreAssertionFailures)
+    Bindings & autoArgs, PackageInfos & drvs, bool ignoreAssertionFailures)
 {
     Done done;
     getDerivations(state, v, pathPrefix, autoArgs, drvs, done, ignoreAssertionFailures);

src/libexpr/get-drvs.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "eval.hh"
 #include "path.hh"
@@ -9,8 +10,10 @@
 
 namespace nix {
 
-
-struct DrvInfo
+/**
+ * A "parsed" package attribute set.
+ */
+struct PackageInfo
 {
 public:
     typedef std::map<std::string, std::optional<StorePath>> Outputs;
@@ -25,20 +28,26 @@ private:
     mutable std::string outputName;
     Outputs outputs;
 
-    bool failed = false; // set if we get an AssertionError
+    /**
+     * Set if we get an AssertionError
+     */
+    bool failed = false;
 
-    Bindings * attrs = nullptr, * meta = nullptr;
+    const Bindings * attrs = nullptr, * meta = nullptr;
 
-    Bindings * getMeta();
+    const Bindings * getMeta();
 
     bool checkMeta(Value & v);
 
 public:
-    std::string attrPath; /* path towards the derivation */
+    /**
+     * path towards the derivation
+     */
+    std::string attrPath;
 
-    DrvInfo(EvalState & state) : state(&state) { };
-    DrvInfo(EvalState & state, std::string attrPath, Bindings * attrs);
-    DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs);
+    PackageInfo(EvalState & state) : state(&state) { };
+    PackageInfo(EvalState & state, std::string attrPath, const Bindings * attrs);
+    PackageInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs);
 
     std::string queryName() const;
     std::string querySystem() const;
@@ -46,8 +55,10 @@ public:
     StorePath requireDrvPath() const;
     StorePath queryOutPath() const;
     std::string queryOutputName() const;
-    /** Return the unordered map of output names to (optional) output paths.
-     * The "outputs to install" are determined by `meta.outputsToInstall`. */
+    /**
+     * Return the unordered map of output names to (optional) output paths.
+     * The "outputs to install" are determined by `meta.outputsToInstall`.
+     */
     Outputs queryOutputs(bool withPaths = true, bool onlyOutputsToInstall = false);
 
     StringSet queryMetaNames();
@@ -73,19 +84,21 @@ public:
 
 
 #if HAVE_BOEHMGC
-typedef std::list<DrvInfo, traceable_allocator<DrvInfo>> DrvInfos;
+typedef std::list<PackageInfo, traceable_allocator<PackageInfo>> PackageInfos;
 #else
-typedef std::list<DrvInfo> DrvInfos;
+typedef std::list<PackageInfo> PackageInfos;
 #endif
 
 
-/* If value `v' denotes a derivation, return a DrvInfo object
-   describing it. Otherwise return nothing. */
-std::optional<DrvInfo> getDerivation(EvalState & state,
+/**
+ * If value `v` denotes a derivation, return a PackageInfo object
+ * describing it. Otherwise return nothing.
+ */
+std::optional<PackageInfo> getDerivation(EvalState & state,
     Value & v, bool ignoreAssertionFailures);
 
 void getDerivations(EvalState & state, Value & v, const std::string & pathPrefix,
-    Bindings & autoArgs, DrvInfos & drvs,
+    Bindings & autoArgs, PackageInfos & drvs,
     bool ignoreAssertionFailures);
 
 

src/libexpr/json-to-value.cc

@@ -1,5 +1,8 @@
 #include "json-to-value.hh"
+#include "value.hh"
+#include "eval.hh"
 
+#include <limits>
 #include <variant>
 #include <nlohmann/json.hpp>
 
@@ -40,7 +43,7 @@ class JSONSax : nlohmann::json_sax<json> {
             auto attrs2 = state.buildBindings(attrs.size());
             for (auto & i : attrs)
                 attrs2.insert(i.first, i.second);
-            parent->value(state).mkAttrs(attrs2.alreadySorted());
+            parent->value(state).mkAttrs(attrs2);
             return std::move(parent);
         }
         void add() override { v = nullptr; }
@@ -55,11 +58,10 @@ class JSONSax : nlohmann::json_sax<json> {
         ValueVector values;
         std::unique_ptr<JSONState> resolve(EvalState & state) override
         {
-            Value & v = parent->value(state);
-            state.mkList(v, values.size());
-            for (size_t n = 0; n < values.size(); ++n) {
-                v.listElems()[n] = values[n];
-            }
+            auto list = state.buildList(values.size());
+            for (const auto & [n, v2] : enumerate(list))
+                v2 = values[n];
+            parent->value(state).mkList(list);
             return std::move(parent);
         }
         void add() override {
@@ -79,42 +81,46 @@ class JSONSax : nlohmann::json_sax<json> {
 public:
     JSONSax(EvalState & state, Value & v) : state(state), rs(new JSONState(&v)) {};
 
-    bool null()
+    bool null() override
     {
         rs->value(state).mkNull();
         rs->add();
         return true;
     }
 
-    bool boolean(bool val)
+    bool boolean(bool val) override
     {
         rs->value(state).mkBool(val);
         rs->add();
         return true;
     }
 
-    bool number_integer(number_integer_t val)
+    bool number_integer(number_integer_t val) override
     {
         rs->value(state).mkInt(val);
         rs->add();
         return true;
     }
 
-    bool number_unsigned(number_unsigned_t val)
+    bool number_unsigned(number_unsigned_t val_) override
     {
+        if (val_ > std::numeric_limits<NixInt::Inner>::max()) {
+            throw Error("unsigned json number %1% outside of Nix integer range", val_);
+        }
+        NixInt::Inner val = val_;
         rs->value(state).mkInt(val);
         rs->add();
         return true;
     }
 
-    bool number_float(number_float_t val, const string_t & s)
+    bool number_float(number_float_t val, const string_t & s) override
     {
         rs->value(state).mkFloat(val);
         rs->add();
         return true;
     }
 
-    bool string(string_t & val)
+    bool string(string_t & val) override
     {
         rs->value(state).mkString(val);
         rs->add();
@@ -122,7 +128,7 @@ public:
     }
 
 #if NLOHMANN_JSON_VERSION_MAJOR >= 3 && NLOHMANN_JSON_VERSION_MINOR >= 8
-    bool binary(binary_t&)
+    bool binary(binary_t&) override
     {
         // This function ought to be unreachable
         assert(false);
@@ -130,36 +136,36 @@ public:
     }
 #endif
 
-    bool start_object(std::size_t len)
+    bool start_object(std::size_t len) override
     {
         rs = std::make_unique<JSONObjectState>(std::move(rs));
         return true;
     }
 
-    bool key(string_t & name)
+    bool key(string_t & name) override
     {
         dynamic_cast<JSONObjectState*>(rs.get())->key(name, state);
         return true;
     }
 
-    bool end_object() {
+    bool end_object() override {
         rs = rs->resolve(state);
         rs->add();
         return true;
     }
 
-    bool end_array() {
+    bool end_array() override {
         return end_object();
     }
 
-    bool start_array(size_t len) {
+    bool start_array(size_t len) override {
         rs = std::make_unique<JSONListState>(std::move(rs),
             len != std::numeric_limits<size_t>::max() ? len : 128);
         return true;
     }
 
-    bool parse_error(std::size_t, const std::string&, const nlohmann::detail::exception& ex) {
-        throw JSONParseError(ex.what());
+    bool parse_error(std::size_t, const std::string&, const nlohmann::detail::exception& ex) override {
+        throw JSONParseError("%s", ex.what());
     }
 };
 

src/libexpr/json-to-value.hh

@@ -1,12 +1,16 @@
 #pragma once
+///@file
 
-#include "eval.hh"
+#include "error.hh"
 
 #include <string>
 
 namespace nix {
 
-MakeError(JSONParseError, EvalError);
+class EvalState;
+struct Value;
+
+MakeError(JSONParseError, Error);
 
 void parseJSON(EvalState & state, const std::string_view & s, Value & v);
 

src/libexpr/lexer-helpers.cc

@@ -0,0 +1,30 @@
+#include "lexer-tab.hh"
+#include "lexer-helpers.hh"
+#include "parser-tab.hh"
+
+void nix::lexer::internal::initLoc(YYLTYPE * loc)
+{
+    loc->beginOffset = loc->endOffset = 0;
+}
+
+void nix::lexer::internal::adjustLoc(yyscan_t yyscanner, YYLTYPE * loc, const char * s, size_t len)
+{
+    loc->stash();
+
+    LexerState & lexerState = *yyget_extra(yyscanner);
+
+    if (lexerState.docCommentDistance == 1) {
+        // Preceding token was a doc comment.
+        ParserLocation doc;
+        doc.beginOffset = lexerState.lastDocCommentLoc.beginOffset;
+        ParserLocation docEnd;
+        docEnd.beginOffset = lexerState.lastDocCommentLoc.endOffset;
+        DocComment docComment{lexerState.at(doc), lexerState.at(docEnd)};
+        PosIdx locPos = lexerState.at(*loc);
+        lexerState.positionToDocComment.emplace(locPos, docComment);
+    }
+    lexerState.docCommentDistance++;
+
+    loc->beginOffset = loc->endOffset;
+    loc->endOffset += len;
+}

src/libexpr/lexer-helpers.hh

@@ -0,0 +1,9 @@
+#pragma once
+
+namespace nix::lexer::internal {
+
+void initLoc(YYLTYPE * loc);
+
+void adjustLoc(yyscan_t yyscanner, YYLTYPE * loc, const char * s, size_t len);
+
+} // namespace nix::lexer

src/libexpr/lexer.l

@@ -1,10 +1,11 @@
 %option reentrant bison-bridge bison-locations
+%option align
 %option noyywrap
 %option never-interactive
 %option stack
 %option nodefault
 %option nounput noyy_top_state
-
+%option extra-type="::nix::LexerState *"
 
 %s DEFAULT
 %x STRING
@@ -13,62 +14,30 @@
 %x INPATH_SLASH
 %x PATH_START
 
+%top {
+#include "parser-tab.hh" // YYSTYPE
+#include "parser-state.hh"
+}
 
 %{
 #ifdef __clang__
 #pragma clang diagnostic ignored "-Wunneeded-internal-declaration"
 #endif
 
-#include <boost/lexical_cast.hpp>
-
 #include "nixexpr.hh"
 #include "parser-tab.hh"
+#include "lexer-helpers.hh"
+
+namespace nix {
+  struct LexerState;
+}
 
 using namespace nix;
+using namespace nix::lexer::internal;
 
 namespace nix {
 
-static inline PosIdx makeCurPos(const YYLTYPE & loc, ParseData * data)
-{
-    return data->state.positions.add(data->origin, loc.first_line, loc.first_column);
-}
-
-#define CUR_POS makeCurPos(*yylloc, data)
-
-// backup to recover from yyless(0)
-YYLTYPE prev_yylloc;
-
-static void initLoc(YYLTYPE * loc)
-{
-    loc->first_line = loc->last_line = 1;
-    loc->first_column = loc->last_column = 1;
-}
-
-static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
-{
-    prev_yylloc = *loc;
-
-    loc->first_line = loc->last_line;
-    loc->first_column = loc->last_column;
-
-    for (size_t i = 0; i < len; i++) {
-       switch (*s++) {
-       case '\r':
-           if (*s == '\n') { /* cr/lf */
-               i++;
-               s++;
-           }
-           /* fall through */
-       case '\n':
-           ++loc->last_line;
-           loc->last_column = 1;
-           break;
-       default:
-           ++loc->last_column;
-       }
-    }
-}
-
+#define CUR_POS state->at(*yylloc)
 
 // we make use of the fact that the parser receives a private copy of the input
 // string and can munge around in it.
@@ -98,11 +67,22 @@ static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length)
     return {result, size_t(t - result)};
 }
 
+static void requireExperimentalFeature(const ExperimentalFeature & feature, const Pos & pos)
+{
+    if (!experimentalFeatureSettings.isEnabled(feature))
+        throw ParseError(ErrorInfo{
+            .msg = HintFmt("experimental Nix feature '%1%' is disabled; add '--extra-experimental-features %1%' to enable it", showExperimentalFeature(feature)),
+            .pos = pos,
+        });
+}
 
 }
 
+// yacc generates code that uses unannotated fallthrough.
+#pragma GCC diagnostic ignored "-Wimplicit-fallthrough"
+
 #define YY_USER_INIT initLoc(yylloc)
-#define YY_USER_ACTION adjustLoc(yylloc, yytext, yyleng);
+#define YY_USER_ACTION adjustLoc(yyscanner, yylloc, yytext, yyleng);
 
 #define PUSH_STATE(state) yy_push_state(state, yyscanner)
 #define POP_STATE() yy_pop_state(yyscanner)
@@ -132,7 +112,7 @@ else        { return ELSE; }
 assert      { return ASSERT; }
 with        { return WITH; }
 let         { return LET; }
-in          { return IN; }
+in          { return IN_KW; }
 rec         { return REC; }
 inherit     { return INHERIT; }
 or          { return OR_KW; }
@@ -147,27 +127,34 @@ or          { return OR_KW; }
 \-\>        { return IMPL; }
 \/\/        { return UPDATE; }
 \+\+        { return CONCAT; }
+\<\|        { requireExperimentalFeature(Xp::PipeOperators, state->positions[CUR_POS]);
+              return PIPE_FROM;
+            }
+\|\>        { requireExperimentalFeature(Xp::PipeOperators, state->positions[CUR_POS]);
+              return PIPE_INTO;
+            }
 
 {ID}        { yylval->id = {yytext, (size_t) yyleng}; return ID; }
 {INT}       { errno = 0;
-              try {
-                  yylval->n = boost::lexical_cast<int64_t>(yytext);
-              } catch (const boost::bad_lexical_cast &) {
-                  throw ParseError({
-                      .msg = hintfmt("invalid integer '%1%'", yytext),
-                      .errPos = data->state.positions[CUR_POS],
+              std::optional<int64_t> numMay = string2Int<int64_t>(yytext);
+              if (numMay.has_value()) {
+                  yylval->n = NixInt{*numMay};
+              } else {
+                  throw ParseError(ErrorInfo{
+                      .msg = HintFmt("invalid integer '%1%'", yytext),
+                      .pos = state->positions[CUR_POS],
                   });
               }
-              return INT;
+              return INT_LIT;
             }
 {FLOAT}     { errno = 0;
               yylval->nf = strtod(yytext, 0);
               if (errno != 0)
-                  throw ParseError({
-                      .msg = hintfmt("invalid float '%1%'", yytext),
-                      .errPos = data->state.positions[CUR_POS],
+                  throw ParseError(ErrorInfo{
+                      .msg = HintFmt("invalid float '%1%'", yytext),
+                      .pos = state->positions[CUR_POS],
                   });
-              return FLOAT;
+              return FLOAT_LIT;
             }
 
 \$\{        { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
@@ -188,7 +175,7 @@ or          { return OR_KW; }
                 /* It is impossible to match strings ending with '$' with one
                    regex because trailing contexts are only valid at the end
                    of a rule. (A sane but undocumented limitation.) */
-                yylval->str = unescapeStr(data->symbols, yytext, yyleng);
+                yylval->str = unescapeStr(state->symbols, yytext, yyleng);
                 return STR;
               }
 <STRING>\$\{  { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
@@ -216,7 +203,7 @@ or          { return OR_KW; }
                    return IND_STR;
                  }
 <IND_STRING>\'\'\\{ANY} {
-                   yylval->str = unescapeStr(data->symbols, yytext + 2, yyleng - 2);
+                   yylval->str = unescapeStr(state->symbols, yytext + 2, yyleng - 2);
                    return IND_STR;
                  }
 <IND_STRING>\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
@@ -230,7 +217,7 @@ or          { return OR_KW; }
 {HPATH_START}\$\{ {
   PUSH_STATE(PATH_START);
   yyless(0);
-  *yylloc = prev_yylloc;
+  yylloc->unstash();
 }
 
 <PATH_START>{PATH_SEG} {
@@ -286,24 +273,47 @@ or          { return OR_KW; }
      context (it may be ')', ';', or something of that sort) */
   POP_STATE();
   yyless(0);
-  *yylloc = prev_yylloc;
+  yylloc->unstash();
   return PATH_END;
 }
 
 <INPATH_SLASH>{ANY} |
 <INPATH_SLASH><<EOF>> {
-  throw ParseError({
-      .msg = hintfmt("path has a trailing slash"),
-      .errPos = data->state.positions[CUR_POS],
+  throw ParseError(ErrorInfo{
+      .msg = HintFmt("path has a trailing slash"),
+      .pos = state->positions[CUR_POS],
   });
 }
 
 {SPATH}     { yylval->path = {yytext, (size_t) yyleng}; return SPATH; }
 {URI}       { yylval->uri = {yytext, (size_t) yyleng}; return URI; }
 
-[ \t\r\n]+    /* eat up whitespace */
-\#[^\r\n]*    /* single-line comments */
-\/\*([^*]|\*+[^*/])*\*+\/  /* long comments */
+%{
+// Doc comment rule
+//
+//   \/\*\*                            /**
+//         [^/*]                       reject /**/ (empty comment) and /***
+//              ([^*]|\*+[^*/])*\*+\/  same as the long comment rule
+//              (             )*       zero or more non-ending sequences
+//                              \*     end(1)
+//                                 \/  end(2)
+%}
+\/\*\*[^/*]([^*]|\*+[^*/])*\*+\/  /* doc comments */ {
+    LexerState & lexerState = *yyget_extra(yyscanner);
+    lexerState.docCommentDistance = 0;
+    lexerState.lastDocCommentLoc.beginOffset = yylloc->beginOffset;
+    lexerState.lastDocCommentLoc.endOffset = yylloc->endOffset;
+}
+
+
+%{
+// The following rules have docCommentDistance--
+// This compensates for the docCommentDistance++ which happens by default to
+// make all the other rules invalidate the doc comment.
+%}
+[ \t\r\n]+    /* eat up whitespace */ { yyget_extra(yyscanner)->docCommentDistance--; }
+\#[^\r\n]*    /* single-line comments */ { yyget_extra(yyscanner)->docCommentDistance--; }
+\/\*([^*]|\*+[^*/])*\*+\/  /* long comments */ { yyget_extra(yyscanner)->docCommentDistance--; }
 
 {ANY}       {
               /* Don't return a negative number, as this will cause

src/libexpr/local.mk

@@ -6,18 +6,23 @@ libexpr_DIR := $(d)
 
 libexpr_SOURCES := \
   $(wildcard $(d)/*.cc) \
+  $(wildcard $(d)/value/*.cc) \
   $(wildcard $(d)/primops/*.cc) \
-  $(wildcard $(d)/flake/*.cc) \
   $(d)/lexer-tab.cc \
   $(d)/parser-tab.cc
+# Not just for this library itself, but also for downstream libraries using this library
 
-libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libmain -I src/libexpr
+INCLUDE_libexpr := -I $(d)
+
+libexpr_CXXFLAGS += \
+	$(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) \
+	-DGC_THREADS
 
 libexpr_LIBS = libutil libstore libfetchers
 
-libexpr_LDFLAGS += -lboost_context -pthread
+libexpr_LDFLAGS += -lboost_context $(THREAD_LDFLAGS)
 ifdef HOST_LINUX
- libexpr_LDFLAGS += -ldl
+  libexpr_LDFLAGS += -ldl
 endif
 
 # The dependency on libgc must be propagated (i.e. meaning that
@@ -35,11 +40,11 @@ $(d)/lexer-tab.cc $(d)/lexer-tab.hh: $(d)/lexer.l
 
 clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh
 
-$(eval $(call install-file-in, $(d)/nix-expr.pc, $(libdir)/pkgconfig, 0644))
+$(eval $(call install-file-in, $(buildprefix)$(d)/nix-expr.pc, $(libdir)/pkgconfig, 0644))
 
-$(foreach i, $(wildcard src/libexpr/flake/*.hh), \
-  $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644)))
+$(foreach i, $(wildcard src/libexpr/value/*.hh), \
+  $(eval $(call install-file-in, $(i), $(includedir)/nix/value, 0644)))
 
-$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh
+$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh
 
-$(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh
+$(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh $(d)/call-flake.nix.gen.hh

src/libexpr/meson.build

@@ -0,0 +1,210 @@
+project('nix-expr', 'cpp',
+  version : files('.version'),
+  default_options : [
+    'cpp_std=c++2a',
+    # TODO(Qyriad): increase the warning level
+    'warning_level=1',
+    'debug=true',
+    'optimization=2',
+    'errorlogs=true', # Please print logs for tests that fail
+  ],
+  meson_version : '>= 1.1',
+  license : 'LGPL-2.1-or-later',
+)
+
+cxx = meson.get_compiler('cpp')
+
+subdir('build-utils-meson/deps-lists')
+
+configdata = configuration_data()
+
+deps_private_maybe_subproject = [
+]
+deps_public_maybe_subproject = [
+  dependency('nix-util'),
+  dependency('nix-store'),
+  dependency('nix-fetchers'),
+]
+subdir('build-utils-meson/subprojects')
+
+subdir('build-utils-meson/threads')
+
+boost = dependency(
+  'boost',
+  modules : ['container', 'context'],
+  include_type: 'system',
+)
+# boost is a public dependency, but not a pkg-config dependency unfortunately, so we
+# put in `deps_other`.
+deps_other += boost
+
+nlohmann_json = dependency('nlohmann_json', version : '>= 3.9')
+deps_public += nlohmann_json
+
+bdw_gc = dependency('bdw-gc', required : get_option('gc'))
+if bdw_gc.found()
+  deps_public += bdw_gc
+  foreach funcspec : [
+    'pthread_attr_get_np',
+    'pthread_getattr_np',
+  ]
+    define_name = 'HAVE_' + funcspec.underscorify().to_upper()
+    define_value = cxx.has_function(funcspec).to_int()
+    configdata.set(define_name, define_value)
+  endforeach
+  configdata.set('GC_THREADS', 1)
+endif
+configdata.set('HAVE_BOEHMGC', bdw_gc.found().to_int())
+
+toml11 = dependency(
+  'toml11',
+  version : '>=3.7.0',
+  method : 'cmake',
+  include_type: 'system',
+)
+deps_other += toml11
+
+config_h = configure_file(
+  configuration : configdata,
+  output : 'config-expr.hh',
+)
+
+add_project_arguments(
+  # TODO(Qyriad): Yes this is how the autoconf+Make system did it.
+  # It would be nice for our headers to be idempotent instead.
+  '-include', 'config-util.hh',
+  '-include', 'config-store.hh',
+  # '-include', 'config-fetchers.h',
+  '-include', 'config-expr.hh',
+  language : 'cpp',
+)
+
+subdir('build-utils-meson/diagnostics')
+
+parser_tab = custom_target(
+  input : 'parser.y',
+  output : [
+    'parser-tab.cc',
+    'parser-tab.hh',
+  ],
+  command : [
+    'bison',
+    '-v',
+    '-o',
+    '@OUTPUT0@',
+    '@INPUT@',
+    '-d',
+  ],
+  # NOTE(Qyriad): Meson doesn't support installing only part of a custom target, so we add
+  # an install script below which removes parser-tab.cc.
+  install : true,
+  install_dir : get_option('includedir') / 'nix',
+)
+
+lexer_tab = custom_target(
+  input : [
+    'lexer.l',
+    parser_tab,
+  ],
+  output : [
+    'lexer-tab.cc',
+    'lexer-tab.hh',
+  ],
+  command : [
+    'flex',
+    '--outfile',
+    '@OUTPUT0@',
+    '--header-file=' + '@OUTPUT1@',
+    '@INPUT0@',
+  ],
+  # NOTE(Qyriad): Meson doesn't support installing only part of a custom target, so we add
+  # an install script below which removes lexer-tab.cc.
+  install : true,
+  install_dir : get_option('includedir') / 'nix',
+)
+
+subdir('build-utils-meson/generate-header')
+
+generated_headers = []
+foreach header : [
+  'imported-drv-to-derivation.nix',
+  'fetchurl.nix',
+  'call-flake.nix',
+]
+  generated_headers += gen_header.process(header)
+endforeach
+
+sources = files(
+  'attr-path.cc',
+  'attr-set.cc',
+  'eval-cache.cc',
+  'eval-error.cc',
+  'eval-gc.cc',
+  'eval-settings.cc',
+  'eval.cc',
+  'function-trace.cc',
+  'get-drvs.cc',
+  'json-to-value.cc',
+  'lexer-helpers.cc',
+  'nixexpr.cc',
+  'paths.cc',
+  'primops.cc',
+  'print-ambiguous.cc',
+  'print.cc',
+  'search-path.cc',
+  'value-to-json.cc',
+  'value-to-xml.cc',
+  'value/context.cc',
+)
+
+include_dirs = [include_directories('.')]
+
+headers = [config_h] + files(
+  'attr-path.hh',
+  'attr-set.hh',
+  'eval-cache.hh',
+  'eval-error.hh',
+  'eval-gc.hh',
+  'eval-inline.hh',
+  'eval-settings.hh',
+  'eval.hh',
+  'function-trace.hh',
+  'gc-small-vector.hh',
+  'get-drvs.hh',
+  'json-to-value.hh',
+  # internal: 'lexer-helpers.hh',
+  'nixexpr.hh',
+  'parser-state.hh',
+  'pos-idx.hh',
+  'pos-table.hh',
+  'primops.hh',
+  'print-ambiguous.hh',
+  'print-options.hh',
+  'print.hh',
+  'repl-exit-status.hh',
+  'search-path.hh',
+  'symbol-table.hh',
+  'value-to-json.hh',
+  'value-to-xml.hh',
+  'value.hh',
+  'value/context.hh',
+)
+
+subdir('primops')
+
+this_library = library(
+  'nixexpr',
+  sources,
+  parser_tab,
+  lexer_tab,
+  generated_headers,
+  dependencies : deps_public + deps_private + deps_other,
+  prelink : true, # For C++ static initializers
+  install : true,
+)
+
+install_headers(headers, subdir : 'nix', preserve_path : true)
+
+libraries_private = []
+
+subdir('build-utils-meson/export')

src/libexpr/meson.options

@@ -0,0 +1,3 @@
+option('gc', type : 'feature',
+  description : 'enable garbage collection in the Nix expression evaluator (requires Boehm GC)',
+)

src/libexpr/nix-expr.pc.in

@@ -7,4 +7,4 @@ Description: Nix Package Manager
 Version: @PACKAGE_VERSION@
 Requires: nix-store bdw-gc
 Libs: -L${libdir} -lnixexpr
-Cflags: -I${includedir}/nix -std=c++17
+Cflags: -I${includedir}/nix -std=c++2a

src/libexpr/nixexpr.cc

@@ -1,124 +1,46 @@
 #include "nixexpr.hh"
-#include "derivations.hh"
 #include "eval.hh"
 #include "symbol-table.hh"
 #include "util.hh"
+#include "print.hh"
 
 #include <cstdlib>
+#include <sstream>
+
+#include "strings-inline.hh"
 
 namespace nix {
 
-struct PosAdapter : AbstractPos
-{
-    Pos::Origin origin;
+unsigned long Expr::nrExprs = 0;
 
-    PosAdapter(Pos::Origin origin)
-        : origin(std::move(origin))
-    {
-    }
-
-    std::optional<std::string> getSource() const override
-    {
-        return std::visit(overloaded {
-            [](const Pos::none_tag &) -> std::optional<std::string> {
-                return std::nullopt;
-            },
-            [](const Pos::Stdin & s) -> std::optional<std::string> {
-                // Get rid of the null terminators added by the parser.
-                return std::string(s.source->c_str());
-            },
-            [](const Pos::String & s) -> std::optional<std::string> {
-                // Get rid of the null terminators added by the parser.
-                return std::string(s.source->c_str());
-            },
-            [](const Path & path) -> std::optional<std::string> {
-                try {
-                    return readFile(path);
-                } catch (Error &) {
-                    return std::nullopt;
-                }
-            }
-        }, origin);
-    }
-
-    void print(std::ostream & out) const override
-    {
-        std::visit(overloaded {
-            [&](const Pos::none_tag &) { out << "«none»"; },
-            [&](const Pos::Stdin &) { out << "«stdin»"; },
-            [&](const Pos::String & s) { out << "«string»"; },
-            [&](const Path & path) { out << path; }
-        }, origin);
-    }
-};
-
-Pos::operator std::shared_ptr<AbstractPos>() const
-{
-    auto pos = std::make_shared<PosAdapter>(origin);
-    pos->line = line;
-    pos->column = column;
-    return pos;
-}
-
-/* Displaying abstract syntax trees. */
-
-static void showString(std::ostream & str, std::string_view s)
-{
-    str << '"';
-    for (auto c : s)
-        if (c == '"' || c == '\\' || c == '$') str << "\\" << c;
-        else if (c == '\n') str << "\\n";
-        else if (c == '\r') str << "\\r";
-        else if (c == '\t') str << "\\t";
-        else str << c;
-    str << '"';
-}
+ExprBlackHole eBlackHole;
 
+// FIXME: remove, because *symbols* are abstract and do not have a single
+//        textual representation; see printIdentifier()
 std::ostream & operator <<(std::ostream & str, const SymbolStr & symbol)
 {
     std::string_view s = symbol;
-
-    if (s.empty())
-        str << "\"\"";
-    else if (s == "if") // FIXME: handle other keywords
-        str << '"' << s << '"';
-    else {
-        char c = s[0];
-        if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')) {
-            showString(str, s);
-            return str;
-        }
-        for (auto c : s)
-            if (!((c >= 'a' && c <= 'z') ||
-                  (c >= 'A' && c <= 'Z') ||
-                  (c >= '0' && c <= '9') ||
-                  c == '_' || c == '\'' || c == '-')) {
-                showString(str, s);
-                return str;
-            }
-        str << s;
-    }
-    return str;
+    return printIdentifier(str, s);
 }
 
 void Expr::show(const SymbolTable & symbols, std::ostream & str) const
 {
-    abort();
+    unreachable();
 }
 
 void ExprInt::show(const SymbolTable & symbols, std::ostream & str) const
 {
-    str << n;
+    str << v.integer();
 }
 
 void ExprFloat::show(const SymbolTable & symbols, std::ostream & str) const
 {
-    str << nf;
+    str << v.fpoint();
 }
 
 void ExprString::show(const SymbolTable & symbols, std::ostream & str) const
 {
-    showString(str, s);
+    printLiteralString(str, s);
 }
 
 void ExprPath::show(const SymbolTable & symbols, std::ostream & str) const
@@ -150,10 +72,8 @@ void ExprOpHasAttr::show(const SymbolTable & symbols, std::ostream & str) const
     str << ") ? " << showAttrPath(symbols, attrPath) << ")";
 }
 
-void ExprAttrs::show(const SymbolTable & symbols, std::ostream & str) const
+void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) const
 {
-    if (recursive) str << "rec ";
-    str << "{ ";
     typedef const decltype(attrs)::value_type * Attr;
     std::vector<Attr> sorted;
     for (auto & i : attrs) sorted.push_back(&i);
@@ -161,10 +81,39 @@ void ExprAttrs::show(const SymbolTable & symbols, std::ostream & str) const
         std::string_view sa = symbols[a->first], sb = symbols[b->first];
         return sa < sb;
     });
+    std::vector<Symbol> inherits;
+    // We can use the displacement as a proxy for the order in which the symbols were parsed.
+    // The assignment of displacements should be deterministic, so that showBindings is deterministic.
+    std::map<Displacement, std::vector<Symbol>> inheritsFrom;
     for (auto & i : sorted) {
-        if (i->second.inherited)
-            str << "inherit " << symbols[i->first] << " " << "; ";
-        else {
+        switch (i->second.kind) {
+        case AttrDef::Kind::Plain:
+            break;
+        case AttrDef::Kind::Inherited:
+            inherits.push_back(i->first);
+            break;
+        case AttrDef::Kind::InheritedFrom: {
+            auto & select = dynamic_cast<ExprSelect &>(*i->second.e);
+            auto & from = dynamic_cast<ExprInheritFrom &>(*select.e);
+            inheritsFrom[from.displ].push_back(i->first);
+            break;
+        }
+        }
+    }
+    if (!inherits.empty()) {
+        str << "inherit";
+        for (auto sym : inherits) str << " " << symbols[sym];
+        str << "; ";
+    }
+    for (const auto & [from, syms] : inheritsFrom) {
+        str << "inherit (";
+        (*inheritFromExprs)[from]->show(symbols, str);
+        str << ")";
+        for (auto sym : syms) str << " " << symbols[sym];
+        str << "; ";
+    }
+    for (auto & i : sorted) {
+        if (i->second.kind == AttrDef::Kind::Plain) {
             str << symbols[i->first] << " = ";
             i->second.e->show(symbols, str);
             str << "; ";
@@ -177,6 +126,13 @@ void ExprAttrs::show(const SymbolTable & symbols, std::ostream & str) const
         i.valueExpr->show(symbols, str);
         str << "; ";
     }
+}
+
+void ExprAttrs::show(const SymbolTable & symbols, std::ostream & str) const
+{
+    if (recursive) str << "rec ";
+    str << "{ ";
+    showBindings(symbols, str);
     str << "}";
 }
 
@@ -197,7 +153,10 @@ void ExprLambda::show(const SymbolTable & symbols, std::ostream & str) const
     if (hasFormals()) {
         str << "{ ";
         bool first = true;
-        for (auto & i : formals->formals) {
+        // the natural Symbol ordering is by creation time, which can lead to the
+        // same expression being printed in two different ways depending on its
+        // context. always use lexicographic ordering to avoid this.
+        for (auto & i : formals->lexicographicOrder(symbols)) {
             if (first) first = false; else str << ", ";
             str << symbols[i.name];
             if (i.def) {
@@ -232,15 +191,7 @@ void ExprCall::show(const SymbolTable & symbols, std::ostream & str) const
 void ExprLet::show(const SymbolTable & symbols, std::ostream & str) const
 {
     str << "(let ";
-    for (auto & i : attrs->attrs)
-        if (i.second.inherited) {
-            str << "inherit " << symbols[i.first] << "; ";
-        }
-        else {
-            str << symbols[i.first] << " = ";
-            i.second.e->show(symbols, str);
-            str << "; ";
-        }
+    attrs->showBindings(symbols, str);
     str << "in ";
     body->show(symbols, str);
     str << ")";
@@ -298,17 +249,6 @@ void ExprPos::show(const SymbolTable & symbols, std::ostream & str) const
 }
 
 
-std::ostream & operator << (std::ostream & str, const Pos & pos)
-{
-    if (auto pos2 = (std::shared_ptr<AbstractPos>) pos) {
-        str << *pos2;
-    } else
-        str << "undefined position";
-
-    return str;
-}
-
-
 std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath)
 {
     std::ostringstream out;
@@ -331,7 +271,7 @@ std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath)
 
 void Expr::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
 {
-    abort();
+    unreachable();
 }
 
 void ExprInt::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
@@ -363,6 +303,8 @@ void ExprVar::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
     if (es.debugRepl)
         es.exprEnvs.insert(std::make_pair(this, env));
 
+    fromWith = nullptr;
+
     /* Check whether the variable appears in the environment.  If so,
        set its level and displacement. */
     const StaticEnv * curEnv;
@@ -374,7 +316,6 @@ void ExprVar::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
         } else {
             auto i = curEnv->find(name);
             if (i != curEnv->vars.end()) {
-                fromWith = false;
                 this->level = level;
                 displ = i->second;
                 return;
@@ -386,14 +327,21 @@ void ExprVar::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
        enclosing `with'.  If there is no `with', then we can issue an
        "undefined variable" error now. */
     if (withLevel == -1)
-        throw UndefinedVarError({
-            .msg = hintfmt("undefined variable '%1%'", es.symbols[name]),
-            .errPos = es.positions[pos]
-        });
-    fromWith = true;
+        es.error<UndefinedVarError>(
+            "undefined variable '%1%'",
+            es.symbols[name]
+        ).atPos(pos).debugThrow();
+    for (auto * e = env.get(); e && !fromWith; e = e->up)
+        fromWith = e->isWith;
     this->level = withLevel;
 }
 
+void ExprInheritFrom::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
+{
+    if (es.debugRepl)
+        es.exprEnvs.insert(std::make_pair(this, env));
+}
+
 void ExprSelect::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
 {
     if (es.debugRepl)
@@ -417,22 +365,47 @@ void ExprOpHasAttr::bindVars(EvalState & es, const std::shared_ptr<const StaticE
             i.expr->bindVars(es, env);
 }
 
+std::shared_ptr<const StaticEnv> ExprAttrs::bindInheritSources(
+    EvalState & es, const std::shared_ptr<const StaticEnv> & env)
+{
+    if (!inheritFromExprs)
+        return nullptr;
+
+    // the inherit (from) source values are inserted into an env of its own, which
+    // does not introduce any variable names.
+    // analysis must see an empty env, or an env that contains only entries with
+    // otherwise unused names to not interfere with regular names. the parser
+    // has already filled all exprs that access this env with appropriate level
+    // and displacement, and nothing else is allowed to access it. ideally we'd
+    // not even *have* an expr that grabs anything from this env since it's fully
+    // invisible, but the evaluator does not allow for this yet.
+    auto inner = std::make_shared<StaticEnv>(nullptr, env.get(), 0);
+    for (auto from : *inheritFromExprs)
+        from->bindVars(es, env);
+
+    return inner;
+}
+
 void ExprAttrs::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
 {
     if (es.debugRepl)
         es.exprEnvs.insert(std::make_pair(this, env));
 
     if (recursive) {
-        auto newEnv = std::make_shared<StaticEnv>(false, env.get(), recursive ? attrs.size() : 0);
+        auto newEnv = [&] () -> std::shared_ptr<const StaticEnv> {
+            auto newEnv = std::make_shared<StaticEnv>(nullptr, env.get(), attrs.size());
 
-        Displacement displ = 0;
-        for (auto & i : attrs)
-            newEnv->vars.emplace_back(i.first, i.second.displ = displ++);
+            Displacement displ = 0;
+            for (auto & i : attrs)
+                newEnv->vars.emplace_back(i.first, i.second.displ = displ++);
+            return newEnv;
+        }();
 
         // No need to sort newEnv since attrs is in sorted order.
 
+        auto inheritFromEnv = bindInheritSources(es, newEnv);
         for (auto & i : attrs)
-            i.second.e->bindVars(es, i.second.inherited ? env : newEnv);
+            i.second.e->bindVars(es, i.second.chooseByKind(newEnv, env, inheritFromEnv));
 
         for (auto & i : dynamicAttrs) {
             i.nameExpr->bindVars(es, newEnv);
@@ -440,8 +413,10 @@ void ExprAttrs::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv>
         }
     }
     else {
+        auto inheritFromEnv = bindInheritSources(es, env);
+
         for (auto & i : attrs)
-            i.second.e->bindVars(es, env);
+            i.second.e->bindVars(es, i.second.chooseByKind(env, env, inheritFromEnv));
 
         for (auto & i : dynamicAttrs) {
             i.nameExpr->bindVars(es, env);
@@ -465,7 +440,7 @@ void ExprLambda::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv>
         es.exprEnvs.insert(std::make_pair(this, env));
 
     auto newEnv = std::make_shared<StaticEnv>(
-        false, env.get(),
+        nullptr, env.get(),
         (hasFormals() ? formals->formals.size() : 0) +
         (!arg ? 0 : 1));
 
@@ -498,19 +473,23 @@ void ExprCall::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
 
 void ExprLet::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
 {
-    if (es.debugRepl)
-        es.exprEnvs.insert(std::make_pair(this, env));
+    auto newEnv = [&] () -> std::shared_ptr<const StaticEnv> {
+        auto newEnv = std::make_shared<StaticEnv>(nullptr, env.get(), attrs->attrs.size());
 
-    auto newEnv = std::make_shared<StaticEnv>(false, env.get(), attrs->attrs.size());
-
-    Displacement displ = 0;
-    for (auto & i : attrs->attrs)
-        newEnv->vars.emplace_back(i.first, i.second.displ = displ++);
+        Displacement displ = 0;
+        for (auto & i : attrs->attrs)
+            newEnv->vars.emplace_back(i.first, i.second.displ = displ++);
+        return newEnv;
+    }();
 
     // No need to sort newEnv since attrs->attrs is in sorted order.
 
+    auto inheritFromEnv = attrs->bindInheritSources(es, newEnv);
     for (auto & i : attrs->attrs)
-        i.second.e->bindVars(es, i.second.inherited ? env : newEnv);
+        i.second.e->bindVars(es, i.second.chooseByKind(newEnv, env, inheritFromEnv));
+
+    if (es.debugRepl)
+        es.exprEnvs.insert(std::make_pair(this, newEnv));
 
     body->bindVars(es, newEnv);
 }
@@ -520,6 +499,10 @@ void ExprWith::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
     if (es.debugRepl)
         es.exprEnvs.insert(std::make_pair(this, env));
 
+    parentWith = nullptr;
+    for (auto * e = env.get(); e && !parentWith; e = e->up)
+        parentWith = e->isWith;
+
     /* Does this `with' have an enclosing `with'?  If so, record its
        level so that `lookupVar' can look up variables in the previous
        `with' if this one doesn't contain the desired attribute. */
@@ -532,11 +515,8 @@ void ExprWith::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
             break;
         }
 
-    if (es.debugRepl)
-        es.exprEnvs.insert(std::make_pair(this, env));
-
     attrs->bindVars(es, env);
-    auto newEnv = std::make_shared<StaticEnv>(true, env.get());
+    auto newEnv = std::make_shared<StaticEnv>(this, env.get());
     body->bindVars(es, newEnv);
 }
 
@@ -605,6 +585,55 @@ std::string ExprLambda::showNamePos(const EvalState & state) const
     return fmt("%1% at %2%", id, state.positions[pos]);
 }
 
+void ExprLambda::setDocComment(DocComment docComment) {
+    // RFC 145 specifies that the innermost doc comment wins.
+    // See https://github.com/NixOS/rfcs/blob/master/rfcs/0145-doc-strings.md#ambiguous-placement
+    if (!this->docComment) {
+        this->docComment = docComment;
+
+        // Curried functions are defined by putting a function directly
+        // in the body of another function. To render docs for those, we
+        // need to propagate the doc comment to the innermost function.
+        //
+        // If we have our own comment, we've already propagated it, so this
+        // belongs in the same conditional.
+        body->setDocComment(docComment);
+    }
+};
+
+
+
+/* Position table. */
+
+Pos PosTable::operator[](PosIdx p) const
+{
+    auto origin = resolve(p);
+    if (!origin)
+        return {};
+
+    const auto offset = origin->offsetOf(p);
+
+    Pos result{0, 0, origin->origin};
+    auto lines = this->lines.lock();
+    auto linesForInput = (*lines)[origin->offset];
+
+    if (linesForInput.empty()) {
+        auto source = result.getSource().value_or("");
+        const char * begin = source.data();
+        for (Pos::LinesIterator it(source), end; it != end; it++)
+            linesForInput.push_back(it->data() - begin);
+        if (linesForInput.empty())
+            linesForInput.push_back(0);
+    }
+    // as above: the first line starts at byte 0 and is always present
+    auto lineStartOffset = std::prev(
+        std::upper_bound(linesForInput.begin(), linesForInput.end(), offset));
+
+    result.line = 1 + (lineStartOffset - linesForInput.begin());
+    result.column = 1 + (offset - *lineStartOffset);
+    return result;
+}
+
 
 
 /* Symbol table. */
@@ -616,4 +645,22 @@ size_t SymbolTable::totalSize() const
     return n;
 }
 
+std::string DocComment::getInnerText(const PosTable & positions) const {
+    auto beginPos = positions[begin];
+    auto endPos = positions[end];
+    auto docCommentStr = beginPos.getSnippetUpTo(endPos).value_or("");
+
+    // Strip "/**" and "*/"
+    constexpr size_t prefixLen = 3;
+    constexpr size_t suffixLen = 2;
+    std::string docStr = docCommentStr.substr(prefixLen, docCommentStr.size() - prefixLen - suffixLen);
+    if (docStr.empty())
+        return {};
+    // Turn the now missing "/**" into indentation
+    docStr = "   " + docStr;
+    // Strip indentation (for the whole, potentially multi-line string)
+    docStr = stripIndentation(docStr);
+    return docStr;
+}
+
 }