Merge remote-tracking branch 'upstream/master' into overlayfs-store

.clang-format

@@ -0,0 +1,30 @@
+BasedOnStyle: LLVM
+IndentWidth: 4
+BreakBeforeBraces: Custom
+BraceWrapping:
+  AfterStruct: true
+  AfterClass: true
+  AfterFunction: true
+  AfterUnion: true
+  SplitEmptyRecord: false
+PointerAlignment: Middle
+FixNamespaceComments: false
+SortIncludes: Never
+#IndentPPDirectives: BeforeHash
+SpaceAfterCStyleCast: true
+SpaceAfterTemplateKeyword: false
+AccessModifierOffset: -4
+AlignAfterOpenBracket: AlwaysBreak
+AlignEscapedNewlines: DontAlign
+ColumnLimit: 120
+BreakStringLiterals: false
+BitFieldColonSpacing: None
+AllowShortFunctionsOnASingleLine: Empty
+AlwaysBreakTemplateDeclarations: Yes
+BinPackParameters: false
+BreakConstructorInitializers: BeforeComma
+EmptyLineAfterAccessModifier: Leave # change to always/never later?
+EmptyLineBeforeAccessModifier: Leave
+#PackConstructorInitializers: BinPack
+BreakBeforeBinaryOperators: NonAssignment
+AlwaysBreakBeforeMultilineStrings: true

.clang-tidy

@@ -0,0 +1,3 @@
+# We use pointers to aggregates in a couple of places, intentionally.
+# void * would look weird.
+Checks: '-bugprone-sizeof-expression'

.github/CODEOWNERS

@@ -10,16 +10,8 @@
 # This file
 .github/CODEOWNERS @edolstra
 
-# Public documentation
-/doc @fricklerhandwerk
-*.md @fricklerhandwerk
-
 # Documentation of built-in functions
-src/libexpr/primops.cc @fricklerhandwerk @roberth
+src/libexpr/primops.cc @roberth
-# Documentation on experimental features
-src/libutil/experimental-features.cc @fricklerhandwerk
-# Documentation on configuration settings
-src/libstore/globals.hh @fricklerhandwerk
 
 # Libstore layer
 /src/libstore @thufschmitt

.github/PULL_REQUEST_TEMPLATE.md

@@ -10,6 +10,8 @@
 
 <!-- Large change: Provide instructions to reviewers how to read the diff. -->
 
-# Priorities
+# Priorities and Process
 
 Add :+1: to [pull requests you find important](https://github.com/NixOS/nix/pulls?q=is%3Aopen+sort%3Areactions-%2B1-desc).
+
+The Nix maintainer team uses a [GitHub project board](https://github.com/orgs/NixOS/projects/19) to [schedule and track reviews](https://github.com/NixOS/nix/tree/master/maintainers#project-board-protocol). 

.github/workflows/backport.yml

@@ -21,7 +21,7 @@ jobs:
           fetch-depth: 0
       - name: Create backport PRs
         # should be kept in sync with `version`
-        uses: zeebe-io/backport-action@v2.2.0
+        uses: zeebe-io/backport-action@v2.4.1
         with:
           # Config README: https://github.com/zeebe-io/backport-action#backport-action
           github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci.yml

@@ -20,12 +20,12 @@ jobs:
     - uses: actions/checkout@v4
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@v24
+    - uses: cachix/install-nix-action@v25
       with:
         # The sandbox would otherwise be disabled by default on Darwin
         extra_nix_config: "sandbox = true"
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/cachix-action@v13
+    - uses: cachix/cachix-action@v14
       if: needs.check_secrets.outputs.cachix == 'true'
       with:
         name: '${{ env.CACHIX_NAME }}'
@@ -62,10 +62,10 @@ jobs:
       with:
         fetch-depth: 0
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@v24
+    - uses: cachix/install-nix-action@v25
       with:
         install_url: https://releases.nixos.org/nix/nix-2.13.3/install
-    - uses: cachix/cachix-action@v13
+    - uses: cachix/cachix-action@v14
       with:
         name: '${{ env.CACHIX_NAME }}'
         signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
@@ -84,7 +84,7 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@v24
+    - uses: cachix/install-nix-action@v25
       with:
         install_url: '${{needs.installer.outputs.installerURL}}'
         install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve"
@@ -114,12 +114,12 @@ jobs:
     - uses: actions/checkout@v4
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@v24
+    - uses: cachix/install-nix-action@v25
       with:
         install_url: https://releases.nixos.org/nix/nix-2.13.3/install
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
     - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#default.version | tr -d \")" >> $GITHUB_ENV
-    - uses: cachix/cachix-action@v13
+    - uses: cachix/cachix-action@v14
       if: needs.check_secrets.outputs.cachix == 'true'
       with:
         name: '${{ env.CACHIX_NAME }}'

.gitignore

@@ -141,6 +141,7 @@ compile_commands.json
 nix-rust/target
 
 result
+result-*
 
 # IDE
 .vscode/

.version

@@ -1 +1 @@
-2.20.0
+2.21.0

Makefile

@@ -1,8 +1,12 @@
+# External build directory support
+
 include mk/build-dir.mk
 
 -include $(buildprefix)Makefile.config
 clean-files += $(buildprefix)Makefile.config
 
+# List makefiles
+
 ifeq ($(ENABLE_BUILD), yes)
 makefiles = \
   mk/precompiled-headers.mk \
@@ -24,7 +28,7 @@ makefiles = \
   misc/upstart/local.mk
 endif
 
-ifeq ($(ENABLE_BUILD)_$(ENABLE_TESTS), yes_yes)
+ifeq ($(ENABLE_UNIT_TESTS), yes)
 makefiles += \
   tests/unit/libutil/local.mk \
   tests/unit/libutil-support/local.mk \
@@ -34,7 +38,7 @@ makefiles += \
   tests/unit/libexpr-support/local.mk
 endif
 
-ifeq ($(ENABLE_TESTS), yes)
+ifeq ($(ENABLE_FUNCTIONAL_TESTS), yes)
 makefiles += \
   tests/functional/local.mk \
   tests/functional/ca/local.mk \
@@ -42,11 +46,10 @@ makefiles += \
   tests/functional/local-overlay-store/local.mk \
   tests/functional/test-libstoreconsumer/local.mk \
   tests/functional/plugins/local.mk
-else
-makefiles += \
-  mk/disable-tests.mk
 endif
 
+# Miscellaneous global Flags
+
 OPTIMIZE = 1
 
 ifeq ($(OPTIMIZE), 1)
@@ -56,13 +59,63 @@ else
   GLOBAL_CXXFLAGS += -O0 -U_FORTIFY_SOURCE
 endif
 
-include mk/lib.mk
+include mk/platform.mk
 
-# Must be included after `mk/lib.mk` so rules refer to variables defined
+ifdef HOST_WINDOWS
-# by the library. Rules are not "lazy" like variables, unfortunately.
+  # Windows DLLs are stricter about symbol visibility than Unix shared
-ifeq ($(ENABLE_BUILD), yes)
+  # objects --- see https://gcc.gnu.org/wiki/Visibility for details.
-$(eval $(call include-sub-makefile, doc/manual/local.mk))
+  # This is a temporary sledgehammer to export everything like on Unix,
-$(eval $(call include-sub-makefile, doc/internal-api/local.mk))
+  # and not detail with this yet.
+  #
+  # TODO do not do this, and instead do fine-grained export annotations.
+  GLOBAL_LDFLAGS += -Wl,--export-all-symbols
 endif
 
 GLOBAL_CXXFLAGS += -g -Wall -include $(buildprefix)config.h -std=c++2a -I src
+
+# Include the main lib, causing rules to be defined
+
+include mk/lib.mk
+
+# Fallback stub rules for better UX when things are disabled
+#
+# These must be defined after `mk/lib.mk`. Otherwise the first rule
+# incorrectly becomes the default target.
+
+ifneq ($(ENABLE_UNIT_TESTS), yes)
+.PHONY: check
+check:
+	@echo "Unit tests are disabled. Configure without '--disable-unit-tests', or avoid calling 'make check'."
+	@exit 1
+endif
+
+ifneq ($(ENABLE_FUNCTIONAL_TESTS), yes)
+.PHONY: installcheck
+installcheck:
+	@echo "Functional tests are disabled. Configure without '--disable-functional-tests', or avoid calling 'make installcheck'."
+	@exit 1
+endif
+
+# Documentation or else fallback stub rules.
+#
+# The documentation makefiles be included after `mk/lib.mk` so rules
+# refer to variables defined by `mk/lib.mk`. Rules are not "lazy" like
+# variables, unfortunately.
+
+ifeq ($(ENABLE_DOC_GEN), yes)
+$(eval $(call include-sub-makefile, doc/manual/local.mk))
+else
+.PHONY: manual-html manpages
+manual-html manpages:
+	@echo "Generated docs are disabled. Configure without '--disable-doc-gen', or avoid calling 'make manpages' and 'make manual-html'."
+	@exit 1
+endif
+
+ifeq ($(ENABLE_INTERNAL_API_DOCS), yes)
+$(eval $(call include-sub-makefile, doc/internal-api/local.mk))
+else
+.PHONY: internal-api-html
+internal-api-html:
+	@echo "Internal API docs are disabled. Configure with '--enable-internal-api-docs', or avoid calling 'make internal-api-html'."
+	@exit 1
+endif

Makefile.config.in

@@ -9,8 +9,11 @@ CXXFLAGS = @CXXFLAGS@
 CXXLTO = @CXXLTO@
 EDITLINE_LIBS = @EDITLINE_LIBS@
 ENABLE_BUILD = @ENABLE_BUILD@
+ENABLE_DOC_GEN = @ENABLE_DOC_GEN@
+ENABLE_FUNCTIONAL_TESTS = @ENABLE_FUNCTIONAL_TESTS@
+ENABLE_INTERNAL_API_DOCS = @ENABLE_INTERNAL_API_DOCS@
 ENABLE_S3 = @ENABLE_S3@
-ENABLE_TESTS = @ENABLE_TESTS@
+ENABLE_UNIT_TESTS = @ENABLE_UNIT_TESTS@
 GTEST_LIBS = @GTEST_LIBS@
 HAVE_LIBCPUID = @HAVE_LIBCPUID@
 HAVE_SECCOMP = @HAVE_SECCOMP@
@@ -26,7 +29,6 @@ LOWDOWN_LIBS = @LOWDOWN_LIBS@
 OPENSSL_LIBS = @OPENSSL_LIBS@
 PACKAGE_NAME = @PACKAGE_NAME@
 PACKAGE_VERSION = @PACKAGE_VERSION@
-RAPIDCHECK_HEADERS = @RAPIDCHECK_HEADERS@
 SHELL = @bash@
 SODIUM_LIBS = @SODIUM_LIBS@
 SQLITE3_LIBS = @SQLITE3_LIBS@
@@ -36,12 +38,10 @@ checkbindir = @checkbindir@
 checklibdir = @checklibdir@
 datadir = @datadir@
 datarootdir = @datarootdir@
-doc_generate = @doc_generate@
 docdir = @docdir@
 embedded_sandbox_shell = @embedded_sandbox_shell@
 exec_prefix = @exec_prefix@
 includedir = @includedir@
-internal_api_docs = @internal_api_docs@
 libdir = @libdir@
 libexecdir = @libexecdir@
 localstatedir = @localstatedir@

configure.ac

@@ -122,7 +122,6 @@ AC_PATH_PROG(flex, flex, false)
 AC_PATH_PROG(bison, bison, false)
 AC_PATH_PROG(dot, dot)
 AC_PATH_PROG(lsof, lsof, lsof)
-NEED_PROG(jq, jq)
 
 
 AC_SUBST(coreutils, [$(dirname $(type -p cat))])
@@ -133,6 +132,48 @@ AC_ARG_WITH(store-dir, AS_HELP_STRING([--with-store-dir=PATH],[path of the Nix s
 AC_SUBST(storedir)
 
 
+# Running the functional tests without building Nix is useful for testing
+# different pre-built versions of Nix against each other.
+AC_ARG_ENABLE(build, AS_HELP_STRING([--disable-build],[Do not build nix]),
+  ENABLE_BUILD=$enableval, ENABLE_BUILD=yes)
+AC_SUBST(ENABLE_BUILD)
+
+# Building without unit tests is useful for bootstrapping with a smaller footprint
+# or running the tests in a separate derivation. Otherwise, we do compile and
+# run them.
+
+AC_ARG_ENABLE(unit-tests, AS_HELP_STRING([--disable-unit-tests],[Do not build the tests]),
+  ENABLE_UNIT_TESTS=$enableval, ENABLE_UNIT_TESTS=$ENABLE_BUILD)
+AC_SUBST(ENABLE_UNIT_TESTS)
+
+AS_IF(
+  [test "$ENABLE_BUILD" == "no" && test "$ENABLE_UNIT_TESTS" == "yes"],
+  [AC_MSG_ERROR([Cannot enable unit tests when building overall is disabled. Please do not pass '--enable-unit-tests' or do not pass '--disable-build'.])])
+
+AC_ARG_ENABLE(functional-tests, AS_HELP_STRING([--disable-functional-tests],[Do not build the tests]),
+  ENABLE_FUNCTIONAL_TESTS=$enableval, ENABLE_FUNCTIONAL_TESTS=yes)
+AC_SUBST(ENABLE_FUNCTIONAL_TESTS)
+
+# documentation generation switch
+AC_ARG_ENABLE(doc-gen, AS_HELP_STRING([--disable-doc-gen],[disable documentation generation]),
+  ENABLE_DOC_GEN=$enableval, ENABLE_DOC_GEN=$ENABLE_BUILD)
+AC_SUBST(ENABLE_DOC_GEN)
+
+AS_IF(
+  [test "$ENABLE_BUILD" == "no" && test "$ENABLE_DOC_GEN" == "yes"],
+  [AC_MSG_ERROR([Cannot enable generated docs when building overall is disabled. Please do not pass '--enable-doc-gen' or do not pass '--disable-build'.])])
+
+# Building without API docs is the default as Nix' C++ interfaces are internal and unstable.
+AC_ARG_ENABLE(internal-api-docs, AS_HELP_STRING([--enable-internal-api-docs],[Build API docs for Nix's internal unstable C++ interfaces]),
+  ENABLE_INTERNAL_API_DOCS=$enableval, ENABLE_INTERNAL_API_DOCS=no)
+AC_SUBST(ENABLE_INTERNAL_API_DOCS)
+
+AS_IF(
+  [test "$ENABLE_FUNCTIONAL_TESTS" == "yes" || test "$ENABLE_DOC_GEN" == "yes"],
+  [NEED_PROG(jq, jq)])
+
+AS_IF([test "$ENABLE_BUILD" == "yes"],[
+
 # Look for boost, a required dependency.
 # Note that AX_BOOST_BASE only exports *CPP* BOOST_CPPFLAGS, no CXX flags,
 # and CPPFLAGS are not passed to the C++ compiler automatically.
@@ -155,18 +196,6 @@ if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then
     LDFLAGS="-latomic $LDFLAGS"
 fi
 
-# Running the functional tests without building Nix is useful for testing
-# different pre-built versions of Nix against each other.
-AC_ARG_ENABLE(build, AS_HELP_STRING([--disable-build],[Do not build nix]),
-  ENABLE_BUILD=$enableval, ENABLE_BUILD=yes)
-AC_SUBST(ENABLE_BUILD)
-# Building without tests is useful for bootstrapping with a smaller footprint
-# or running the tests in a separate derivation. Otherwise, we do compile and
-# run them.
-AC_ARG_ENABLE(tests, AS_HELP_STRING([--disable-tests],[Do not build the tests]),
-  ENABLE_TESTS=$enableval, ENABLE_TESTS=yes)
-AC_SUBST(ENABLE_TESTS)
-
 AC_ARG_ENABLE(install-unit-tests, AS_HELP_STRING([--enable-install-unit-tests],[Install the unit tests for running later (default no)]),
   INSTALL_UNIT_TESTS=$enableval, INSTALL_UNIT_TESTS=no)
 AC_SUBST(INSTALL_UNIT_TESTS)
@@ -179,11 +208,6 @@ AC_ARG_WITH(check-lib-dir, AS_HELP_STRING([--with-check-lib-dir=PATH],[path to i
   checklibdir=$withval, checklibdir=$libdir)
 AC_SUBST(checklibdir)
 
-# Building without API docs is the default as Nix' C++ interfaces are internal and unstable.
-AC_ARG_ENABLE(internal_api_docs, AS_HELP_STRING([--enable-internal-api-docs],[Build API docs for Nix's internal unstable C++ interfaces]),
-  internal_api_docs=$enableval, internal_api_docs=no)
-AC_SUBST(internal_api_docs)
-
 # LTO is currently broken with clang for unknown reasons; ld segfaults in the llvm plugin
 AC_ARG_ENABLE(lto, AS_HELP_STRING([--enable-lto],[Enable LTO (only supported with GCC) [default=no]]),
   lto=$enableval, lto=no)
@@ -227,17 +251,25 @@ PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= 3.6.19], [CXXFLAGS="$SQLITE3_CFLAGS $CX
 # Look for libcurl, a required dependency.
 PKG_CHECK_MODULES([LIBCURL], [libcurl], [CXXFLAGS="$LIBCURL_CFLAGS $CXXFLAGS"])
 
-# Look for editline, a required dependency.
+# Look for editline or readline, a required dependency.
 # The the libeditline.pc file was added only in libeditline >= 1.15.2,
 # see https://github.com/troglobit/editline/commit/0a8f2ef4203c3a4a4726b9dd1336869cd0da8607,
-# but e.g. Ubuntu 16.04 has an older version, so we fall back to searching for
+# Older versions are no longer supported.
-# editline.h when the pkg-config approach fails.
+AC_ARG_WITH(
-PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLAGS"], [
+  [readline-flavor],
-  AC_CHECK_HEADERS([editline.h], [true],
+  AS_HELP_STRING([--with-readline-flavor],[Which library to use for nice line editting with the Nix language REPL" [default=editline]]),
-    [AC_MSG_ERROR([Nix requires libeditline; it was found neither via pkg-config nor its normal header.])])
+  [readline_flavor=$withval],
-  AC_SEARCH_LIBS([readline read_history], [editline], [],
+  [readline_flavor=editline])
-    [AC_MSG_ERROR([Nix requires libeditline; it was not found via pkg-config, but via its header, but required functions do not work. Maybe it is too old? >= 1.14 is required.])])
+AS_CASE(["$readline_flavor"],
-])
+  [editline], [
+    readline_flavor_pc=libeditline
+  ],
+  [readline], [
+    readline_flavor_pc=readline
+    AC_DEFINE([USE_READLINE], [1], [Use readline instead of editline])
+  ],
+  [AC_MSG_ERROR([bad value "$readline_flavor" for --with-readline-flavor, must be one of: editline, readline])])
+PKG_CHECK_MODULES([EDITLINE], [$readline_flavor_pc], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLAGS"])
 
 # Look for libsodium.
 PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"])
@@ -283,7 +315,13 @@ esac
 AC_SUBST(HAVE_SECCOMP, [$have_seccomp])
 
 # Optional dependencies for better normalizing file system data
-AC_CHECK_HEADERS[sys/xattr.h]
+AC_CHECK_HEADERS([sys/xattr.h])
+AS_IF([test "$ac_cv_header_sys_xattr_h" = "yes"],[
+  AC_CHECK_FUNCS([llistxattr lremovexattr])
+  AS_IF([test "$ac_cv_func_llistxattr" = "yes" && test "$ac_cv_func_lremovexattr" = "yes"],[
+    AC_DEFINE([HAVE_ACL_SUPPORT], [1], [Define if we can manipulate file system Access Control Lists])
+  ])
+])
 
 # Look for aws-cpp-sdk-s3.
 AC_LANG_PUSH(C++)
@@ -310,48 +348,35 @@ if test "$gc" = yes; then
   AC_DEFINE(HAVE_BOEHMGC, 1, [Whether to use the Boehm garbage collector.])
 fi
 
-
+AS_IF([test "$ENABLE_UNIT_TESTS" == "yes"],[
-if test "$ENABLE_TESTS" = yes; then
 
 # Look for gtest.
-PKG_CHECK_MODULES([GTEST], [gtest_main])
+PKG_CHECK_MODULES([GTEST], [gtest_main gmock_main])
-
 
 # Look for rapidcheck.
-AC_ARG_VAR([RAPIDCHECK_HEADERS], [include path of gtest headers shipped by RAPIDCHECK])
+PKG_CHECK_MODULES([RAPIDCHECK], [rapidcheck rapidcheck_gtest])
-# No pkg-config yet, https://github.com/emil-e/rapidcheck/issues/302
-AC_LANG_PUSH(C++)
-AC_SUBST(RAPIDCHECK_HEADERS)
-[CXXFLAGS="-I $RAPIDCHECK_HEADERS $CXXFLAGS"]
-[LIBS="-lrapidcheck -lgtest $LIBS"]
-AC_CHECK_HEADERS([rapidcheck/gtest.h], [], [], [#include <gtest/gtest.h>])
-dnl AC_CHECK_LIB doesn't work for C++ libs with mangled symbols
-AC_LINK_IFELSE([
-    AC_LANG_PROGRAM([[
-      #include <gtest/gtest.h>
-      #include <rapidcheck/gtest.h>
-    ]], [[
-      return RUN_ALL_TESTS();
-    ]])
-  ],
-  [],
-  [AC_MSG_ERROR([librapidcheck is not found.])])
-AC_LANG_POP(C++)
 
-fi
+])
 
 # Look for nlohmann/json.
 PKG_CHECK_MODULES([NLOHMANN_JSON], [nlohmann_json >= 3.9])
 
 
-# documentation generation switch
-AC_ARG_ENABLE(doc-gen, AS_HELP_STRING([--disable-doc-gen],[disable documentation generation]),
-  doc_generate=$enableval, doc_generate=yes)
-AC_SUBST(doc_generate)
-
-
 # Look for lowdown library.
-PKG_CHECK_MODULES([LOWDOWN], [lowdown >= 0.9.0], [CXXFLAGS="$LOWDOWN_CFLAGS $CXXFLAGS"])
+AC_ARG_ENABLE([markdown], AS_HELP_STRING([--enable-markdown], [Enable Markdown rendering in the Nix binary (requires lowdown) [default=auto]]),
+  enable_markdown=$enableval, enable_markdown=auto)
+AS_CASE(["$enable_markdown"],
+  [yes | auto], [
+    PKG_CHECK_MODULES([LOWDOWN], [lowdown >= 0.9.0], [
+      CXXFLAGS="$LOWDOWN_CFLAGS $CXXFLAGS"
+      have_lowdown=1
+      AC_DEFINE(HAVE_LOWDOWN, 1, [Whether lowdown is available and should be used for Markdown rendering.])
+    ], [
+      AS_IF([test "x$enable_markdown" == "xyes"], [AC_MSG_ERROR([--enable-markdown was specified, but lowdown was not found.])])
+    ])
+  ],
+  [no], [have_lowdown=],
+  [AC_MSG_ERROR([bad value "$enable_markdown" for --enable-markdown, must be one of: yes, no, auto])])
 
 
 # Look for libgit2.
@@ -388,6 +413,8 @@ if test "$embedded_sandbox_shell" = yes; then
   AC_DEFINE(HAVE_EMBEDDED_SANDBOX_SHELL, 1, [Include the sandbox shell in the Nix binary.])
 fi
 
+])
+
 
 # Expand all variables in config.status.
 test "$prefix" = NONE && prefix=$ac_default_prefix

doc/internal-api/doxygen.cfg.in

@@ -81,7 +81,7 @@ EXPAND_ONLY_PREDEF     = YES
 # RECURSIVE has no effect here.
 # This tag requires that the tag SEARCH_INCLUDES is set to YES.
 
-INCLUDE_PATH           = @RAPIDCHECK_HEADERS@
+INCLUDE_PATH           =
 
 # If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this
 # tag can be used to specify a list of macro names that should be expanded. The

doc/internal-api/local.mk

@@ -1,19 +1,7 @@
-.PHONY: internal-api-html
-
-ifeq ($(internal_api_docs), yes)
-
 $(docdir)/internal-api/html/index.html $(docdir)/internal-api/latex: $(d)/doxygen.cfg
 	mkdir -p $(docdir)/internal-api
 	{ cat $< ; echo "OUTPUT_DIRECTORY=$(docdir)/internal-api" ; } | doxygen -
 
 # Generate the HTML API docs for Nix's unstable internal interfaces.
+.PHONY: internal-api-html
 internal-api-html: $(docdir)/internal-api/html/index.html
-
-else
-
-# Make a nicer error message
-internal-api-html:
-	@echo "Internal API docs are disabled. Configure with '--enable-internal-api-docs', or avoid calling 'make internal-api-html'."
-	@exit 1
-
-endif

doc/manual/generate-manpage.nix

@@ -93,9 +93,6 @@ let
 
       maybeProse =
         # FIXME: this is a horrible hack to keep `nix help-stores` working.
-        # the correct answer to this is to remove that command and replace it
-        # by statically generated manpages or the output of something like `nix
-        # store info <store type>`.
         let
           help-stores = ''
             ${index}
@@ -121,7 +118,7 @@ let
           };
         in
         optionalString (details ? doc) (
-          if match "@store-types@" details.doc != [ ]
+          if match ".*@store-types@.*" details.doc != null
           then help-stores
           else details.doc
         );

doc/manual/local.mk

@@ -1,5 +1,3 @@
-ifeq ($(doc_generate),yes)
-
 # The version of Nix used to generate the doc. Can also be
 # `$(nix_INSTALL_PATH)` or just `nix` (to grap ambient from the `PATH`),
 # if one prefers.
@@ -180,6 +178,8 @@ manual-html: $(docdir)/manual/index.html
 install: $(docdir)/manual/index.html
 
 # Generate 'nix' manpages.
+.PHONY: manpages
+manpages: $(mandir)/man1/nix3-manpages
 install: $(mandir)/man1/nix3-manpages
 man: doc/manual/generated/man1/nix3-manpages
 all: doc/manual/generated/man1/nix3-manpages
@@ -225,5 +225,3 @@ $(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/
 	@rm -rf $(DESTDIR)$(docdir)/manual
 	@mv $(DESTDIR)$(docdir)/manual.tmp/html $(DESTDIR)$(docdir)/manual
 	@rm -rf $(DESTDIR)$(docdir)/manual.tmp
-
-endif

doc/manual/redirects.js

@@ -21,6 +21,7 @@ const redirects = {
     "chap-distributed-builds": "advanced-topics/distributed-builds.html",
     "chap-post-build-hook": "advanced-topics/post-build-hook.html",
     "chap-post-build-hook-caveats": "advanced-topics/post-build-hook.html#implementation-caveats",
+    "chap-writing-nix-expressions": "language/index.html",
     "part-command-ref": "command-ref/command-ref.html",
     "conf-allow-import-from-derivation": "command-ref/conf-file.html#conf-allow-import-from-derivation",
     "conf-allow-new-privileges": "command-ref/conf-file.html#conf-allow-new-privileges",

doc/manual/rl-next/hash-format-nix32.md

@@ -1,23 +0,0 @@
----
-synopsis: Rename hash format `base32` to `nix32`
-prs: 9452
----
-
-Hash format `base32` was renamed to `nix32` since it used a special nix-specific character set for 
-[Base32](https://en.wikipedia.org/wiki/Base32).
-
-## Deprecation: Use `nix32` instead of `base32` as `toHashFormat`
-
-For the builtin `convertHash`, the `toHashFormat` parameter now accepts the same hash formats as the `--to`/`--from`
-parameters of the `nix hash conert` command: `"base16"`, `"nix32"`, `"base64"`, and `"sri"`. The former `"base32"` value
-remains as a deprecated alias for `"base32"`. Please convert your code from:
-
-```nix
-builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base32";}
-```
-
-to
-
-```nix
-builtins.convertHash { inherit hash hashAlgo; toHashFormat = "nix32";}
-```

doc/manual/rl-next/leading-period.md

@@ -0,0 +1,10 @@
+---
+synopsis: Store paths are allowed to start with `.`
+issues: 912
+prs: 9867 9091 9095 9120 9121 9122 9130 9219 9224
+---
+
+Leading periods were allowed by accident in Nix 2.4. The Nix team has considered this to be a bug, but this behavior has since been relied on by users, leading to unnecessary difficulties.
+From now on, leading periods are officially, definitively supported. The names `.` and `..` are disallowed, as well as those starting with `.-` or `..-`.
+
+Nix versions that denied leading periods are documented [in the issue](https://github.com/NixOS/nix/issues/912#issuecomment-1919583286).

doc/manual/rl-next/mounted-ssh-store.md

@@ -1,8 +0,0 @@
----
-synopsis: Mounted SSH Store
-issues: 7890
-prs: 7912
----
-
-Introduced the store [`mounted-ssh-ng://`](@docroot@/command-ref/new-cli/nix3-help-stores.md).
-This store allows full access to a Nix store on a remote machine and additionally requires that the store be mounted in the local filesystem.

doc/manual/rl-next/nix-config-show.md

@@ -1,7 +0,0 @@
----
-synopsis: Rename to `nix config show`
-issues: 7672
-prs: 9477
----
-
-`nix show-config` was renamed to `nix config show`, and `nix doctor` was renamed to `nix config check`, to be more consistent with the rest of the command-line interface.

doc/manual/rl-next/nix-env-json-drv-path.md

@@ -1,6 +0,0 @@
----
-synopsis: Fix `nix-env --query --drv-path --json`
-prs: 9257
----
-
-Fixed a bug where `nix-env --query` ignored `--drv-path` when `--json` was set.

doc/manual/rl-next/nix-hash-convert.md

@@ -1,47 +0,0 @@
----
-synopsis: Add `nix hash convert`
-prs: 9452
----
-
-New [`nix hash convert`](https://github.com/NixOS/nix/issues/8876) sub command with a fast track
-to stabilization! Examples:
-
-- Convert the hash to `nix32`.
-
-  ```bash
-  $ nix hash convert --algo "sha1" --to nix32 "800d59cfcd3c05e900cb4e214be48f6b886a08df"
-  vw46m23bizj4n8afrc0fj19wrp7mj3c0
-  ```
-  `nix32` is a base32 encoding with a nix-specific character set.
-  Explicitly specify the hashing algorithm (optional with SRI hashes) but detect hash format by the length of the input
-  hash.
-- Convert the hash to the `sri` format that includes an algorithm specification:
-  ```bash
-  nix hash convert --algo "sha1" "800d59cfcd3c05e900cb4e214be48f6b886a08df"
-  sha1-gA1Zz808BekAy04hS+SPa4hqCN8=
-  ```
-  or with an explicit `-to` format:
-  ```bash
-  nix hash convert --algo "sha1" --to sri "800d59cfcd3c05e900cb4e214be48f6b886a08df"
-  sha1-gA1Zz808BekAy04hS+SPa4hqCN8=
-  ```
-- Assert the input format of the hash:
-  ```bash
-  nix hash convert --algo "sha256" --from nix32 "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0="
-  error: input hash 'ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=' does not have the expected format '--from nix32'
-  nix hash convert --algo "sha256" --from nix32 "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s"
-  sha256-ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=
-  ```
-
-The `--to`/`--from`/`--algo` parameters have context-sensitive auto-completion.
-
-## Related Deprecations
-
-The following commands are still available but will emit a deprecation warning. Please convert your code to
-`nix hash convert`:
-
-- `nix hash to-base16 $hash1 $hash2`: Use `nix hash convert --to base16 $hash1 $hash2` instead.
-- `nix hash to-base32 $hash1 $hash2`: Use `nix hash convert --to nix32 $hash1 $hash2` instead.
-- `nix hash to-base64 $hash1 $hash2`: Use `nix hash convert --to base64 $hash1 $hash2` instead.
-- `nix hash to-sri $hash1 $hash2`: : Use `nix hash convert --to sri $hash1 $hash2`
-  or even just `nix hash convert $hash1 $hash2` instead.

doc/manual/rl-next/source-positions-in-errors.md

@@ -1,42 +0,0 @@
----
-synopsis: Source locations are printed more consistently in errors
-issues: 561
-prs: 9555
----
-
-Source location information is now included in error messages more
-consistently. Given this code:
-
-```nix
-let
-  attr = {foo = "bar";};
-  key = {};
-in
-  attr.${key}
-```
-
-Previously, Nix would show this unhelpful message when attempting to evaluate
-it:
-
-```
-error:
-       … while evaluating an attribute name
-
-       error: value is a set while a string was expected
-```
-
-Now, the error message displays where the problematic value was found:
-
-```
-error:
-       … while evaluating an attribute name
-
-         at bad.nix:4:11:
-
-            3|   key = {};
-            4| in attr.${key}
-             |           ^
-            5|
-
-       error: value is a set while a string was expected
-```

doc/manual/src/SUMMARY.md.in

@@ -104,6 +104,9 @@
     - [Channels](command-ref/files/channels.md)
     - [Default Nix expression](command-ref/files/default-nix-expression.md)
 - [Architecture and Design](architecture/architecture.md)
+- [JSON Formats](json/index.md)
+  - [Store Object Info](json/store-object-info.md)
+  - [Derivation](json/derivation.md)
 - [Protocols](protocols/index.md)
   - [Serving Tarball Flakes](protocols/tarball-fetcher.md)
   - [Derivation "ATerm" file format](protocols/derivation-aterm.md)
@@ -117,6 +120,7 @@
   - [C++ style guide](contributing/cxx.md)
 - [Release Notes](release-notes/index.md)
 {{#include ./SUMMARY-rl-next.md}}
+  - [Release 2.20 (2024-01-29)](release-notes/rl-2.20.md)
   - [Release 2.19 (2023-11-17)](release-notes/rl-2.19.md)
   - [Release 2.18 (2023-09-20)](release-notes/rl-2.18.md)
   - [Release 2.17 (2023-07-24)](release-notes/rl-2.17.md)

doc/manual/src/command-ref/nix-instantiate.md

@@ -35,13 +35,51 @@ standard input.
 
   - `--parse`\
     Just parse the input files, and print their abstract syntax trees on
-    standard output in ATerm format.
+    standard output as a Nix expression.
 
   - `--eval`\
     Just parse and evaluate the input files, and print the resulting
     values on standard output. No instantiation of store derivations
     takes place.
 
+    > **Warning**
+    >
+    > This option produces output which can be parsed as a Nix expression which
+    > will produce a different result than the input expression when evaluated.
+    > For example, these two Nix expressions print the same result despite
+    > having different meaning:
+    >
+    > ```console
+    > $ nix-instantiate --eval --expr '{ a = {}; }'
+    > { a = <CODE>; }
+    > $ nix-instantiate --eval --expr '{ a = <CODE>; }'
+    > { a = <CODE>; }
+    > ```
+    >
+    > For human-readable output, `nix eval` (experimental) is more informative:
+    >
+    > ```console
+    > $ nix-instantiate --eval --expr 'a: a'
+    > <LAMBDA>
+    > $ nix eval --expr 'a: a'
+    > «lambda @ «string»:1:1»
+    > ```
+    >
+    > For machine-readable output, the `--xml` option produces unambiguous
+    > output:
+    >
+    > ```console
+    > $ nix-instantiate --eval --xml --expr '{ foo = <CODE>; }'
+    > <?xml version='1.0' encoding='utf-8'?>
+    > <expr>
+    >   <attrs>
+    >     <attr column="3" line="1" name="foo">
+    >       <unevaluated />
+    >     </attr>
+    >   </attrs>
+    > </expr>
+    > ```
+
   - `--find-file`\
     Look up the given files in Nix’s search path (as specified by the
     `NIX_PATH` environment variable). If found, print the corresponding
@@ -61,11 +99,11 @@ standard input.
 
   - `--json`\
     When used with `--eval`, print the resulting value as an JSON
-    representation of the abstract syntax tree rather than as an ATerm.
+    representation of the abstract syntax tree rather than as a Nix expression.
 
   - `--xml`\
     When used with `--eval`, print the resulting value as an XML
-    representation of the abstract syntax tree rather than as an ATerm.
+    representation of the abstract syntax tree rather than as a Nix expression.
     The schema is the same as that used by the [`toXML`
     built-in](../language/builtins.md).
 
@@ -133,28 +171,24 @@ $ nix-instantiate --eval --xml --expr '1 + 2'
 The difference between non-strict and strict evaluation:
 
 ```console
-$ nix-instantiate --eval --xml --expr 'rec { x = "foo"; y = x; }'
+$ nix-instantiate --eval --xml --expr '{ x = {}; }'
-...
+<?xml version='1.0' encoding='utf-8'?>
-  <attr name="x">
+<expr>
-    <string value="foo" />
+  <attrs>
-  </attr>
+    <attr column="3" line="1" name="x">
-  <attr name="y">
+      <unevaluated />
-    <unevaluated />
+    </attr>
-  </attr>
+  </attrs>
-...
+</expr>
-```
 
-Note that `y` is left unevaluated (the XML representation doesn’t
+$ nix-instantiate --eval --xml --strict --expr '{ x = {}; }'
-attempt to show non-normal forms).
+<?xml version='1.0' encoding='utf-8'?>
-
+<expr>
-```console
+  <attrs>
-$ nix-instantiate --eval --xml --strict --expr 'rec { x = "foo"; y = x; }'
+    <attr column="3" line="1" name="x">
-...
+      <attrs>
-  <attr name="x">
+      </attrs>
-    <string value="foo" />
+    </attr>
-  </attr>
+  </attrs>
-  <attr name="y">
+</expr>
-    <string value="foo" />
-  </attr>
-...
 ```

doc/manual/src/contributing/documentation.md

@@ -172,7 +172,7 @@ Please observe these guidelines to ease reviews:
   > ```
   ````
 
-  Highlight syntax definiions as such, using [EBNF](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form) notation:
+  Highlight syntax definitions as such, using [EBNF](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form) notation:
 
   ````
   > **Syntax**

doc/manual/src/contributing/hacking.md

@@ -31,7 +31,7 @@ This shell also adds `./outputs/bin/nix` to your `$PATH` so you can run `nix` im
 To get a shell with one of the other [supported compilation environments](#compilation-environments):
 
 ```console
-$ nix develop .#native-clang11StdenvPackages
+$ nix develop .#native-clangStdenvPackages
 ```
 
 > **Note**
@@ -51,11 +51,14 @@ To install it in `$(pwd)/outputs` and test it:
 
 ```console
 [nix-shell]$ make install
-[nix-shell]$ make installcheck -j $NIX_BUILD_CORES
+[nix-shell]$ make installcheck check -j $NIX_BUILD_CORES
 [nix-shell]$ nix --version
 nix (Nix) 2.12
 ```
 
+For more information on running and filtering tests, see
+[`testing.md`](./testing.md).
+
 To build a release version of Nix for the current operating system and CPU architecture:
 
 ```console
@@ -75,7 +78,7 @@ $ nix-shell
 To get a shell with one of the other [supported compilation environments](#compilation-environments):
 
 ```console
-$ nix-shell --attr devShells.x86_64-linux.native-clang11StdenvPackages
+$ nix-shell --attr devShells.x86_64-linux.native-clangStdenvPackages
 ```
 
 > **Note**
@@ -108,6 +111,26 @@ $ nix-build
 
 You can also build Nix for one of the [supported platforms](#platforms).
 
+## Makefile variables
+
+You may need `profiledir=$out/etc/profile.d` and `sysconfdir=$out/etc` to run `make install`.
+
+Run `make` with [`-e` / `--environment-overrides`](https://www.gnu.org/software/make/manual/make.html#index-_002de) to allow environment variables to override `Makefile` variables:
+
+- `ENABLE_BUILD=yes` to enable building the C++ code.
+- `ENABLE_DOC_GEN=yes` to enable building the documentation (manual, man pages, etc.).
+
+  The docs can take a while to build, so you may want to disable this for local development.
+- `ENABLE_FUNCTIONAL_TESTS=yes` to enable building the functional tests.
+- `ENABLE_UNIT_TESTS=yes` to enable building the unit tests.
+- `OPTIMIZE=1` to enable optimizations.
+- `libraries=libutil programs=` to only build a specific library.
+
+  This will fail in the linking phase if the other libraries haven't been built, but is useful for checking types.
+- `libraries= programs=nix` to only build a specific program.
+
+  This will not work in general, because the programs need the libraries.
+
 ## Platforms
 
 Nix can be built for various platforms, as specified in [`flake.nix`]:
@@ -281,7 +304,6 @@ See also the [format documentation](https://github.com/haskell/cabal/blob/master
 ### Build process
 
 Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`.
-Set `buildUnreleasedNotes = true;` in `flake.nix` to build the release notes on the fly.
 
 ## Branches
 

doc/manual/src/contributing/testing.md

@@ -77,7 +77,7 @@ there is no risk of any build-system wildcards for the library accidentally pick
 ### Running tests
 
 You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`.
-Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option, or the `GTEST_FILTER` environment variable.
+Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option, or the `GTEST_FILTER` environment variable, e.g. `GTEST_FILTER='ErrorTraceTest.*' make check`.
 
 ### Characterisation testing { #characaterisation-testing-unit }
 

doc/manual/src/glossary.md

@@ -3,10 +3,10 @@
 - [derivation]{#gloss-derivation}
 
   A description of a build task. The result of a derivation is a
-  store object. Derivations are typically specified in Nix expressions
+  store object. Derivations declared in Nix expressions are specified
   using the [`derivation` primitive](./language/derivations.md). These are
   translated into low-level *store derivations* (implicitly by
-  `nix-env` and `nix-build`, or explicitly by `nix-instantiate`).
+  `nix-build`, or explicitly by `nix-instantiate`).
 
   [derivation]: #gloss-derivation
 
@@ -14,6 +14,7 @@
 
   A [derivation] represented as a `.drv` file in the [store].
   It has a [store path], like any [store object].
+  It is the [instantiated][instantiate] form of a derivation.
 
   Example: `/nix/store/g946hcz4c8mdvq2g8vxx42z51qb71rvp-git-2.38.1.drv`
 
@@ -23,9 +24,9 @@
 
 - [instantiate]{#gloss-instantiate}, instantiation
 
-  Translate a [derivation] into a [store derivation].
+  Save an evaluated [derivation] as a [store derivation] in the Nix [store].
 
-  See [`nix-instantiate`](./command-ref/nix-instantiate.md).
+  See [`nix-instantiate`](./command-ref/nix-instantiate.md), which produces a store derivation from a Nix expression that evaluates to a derivation.
 
   [instantiate]: #gloss-instantiate
 
@@ -66,7 +67,7 @@
 
   From the perspective of the location where Nix is invoked, the Nix store can be  referred to _local_ or _remote_.
   Only a [local store]{#gloss-local-store} exposes a location in the file system of the machine where Nix is invoked that allows access to store objects, typically `/nix/store`.
-  Local stores can be used for building [derivations](#derivation).
+  Local stores can be used for building [derivations](#gloss-derivation).
   See [Local Store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-store) for details.
 
   [store]: #gloss-store
@@ -126,7 +127,7 @@
   non-[fixed-output](#gloss-fixed-output-derivation)
   derivation.
 
-- [output-addressed store object]{#gloss-output-addressed-store-object}
+- [content-addressed store object]{#gloss-content-addressed-store-object}
 
   A [store object] whose [store path] is determined by its contents.
   This includes derivations, the outputs of [content-addressed derivations](#gloss-content-addressed-derivation), and the outputs of [fixed-output derivations](#gloss-fixed-output-derivation).
@@ -155,6 +156,11 @@
   builder can rely on external inputs such as the network or the
   system time) but the Nix model assumes it.
 
+- [impure derivation]{#gloss-impure-derivation}
+
+  [An experimental feature](#@docroot@/contributing/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure,
+  so that they are always rebuilt, and their outputs not reused by subsequent calls to realise them.
+
 - [Nix database]{#gloss-nix-database}
 
   An SQlite database to track [reference]s between [store object]s.
@@ -166,11 +172,13 @@
 
 - [Nix expression]{#gloss-nix-expression}
 
-  A high-level description of software packages and compositions
+  1. Commonly, a high-level description of software packages and compositions
-  thereof. Deploying software using Nix entails writing Nix
+    thereof. Deploying software using Nix entails writing Nix
-  expressions for your packages. Nix expressions are translated to
+    expressions for your packages. Nix expressions specify [derivations][derivation],
-  derivations that are stored in the Nix store. These derivations can
+    which are [instantiated][instantiate] into the Nix store as [store derivations][store derivation].
-  then be built.
+    These derivations can then be [realised][realise] to produce [outputs][output].
+
+  2. A syntactically valid use of the [Nix language]. For example, the contents of a `.nix` file form an expression.
 
 - [reference]{#gloss-reference}
 
@@ -222,6 +230,9 @@
 
   The [store derivation] that produced an [output path].
 
+  The deriver for an output path can be queried with the `--deriver` option to
+  [`nix-store --query`](@docroot@/command-ref/nix-store/query.md).
+
 - [validity]{#gloss-validity}
 
   A store path is valid if all [store object]s in its [closure] can be read from the [store].
@@ -266,6 +277,21 @@
 
   The epsilon symbol. In the context of a package, this means the version is empty. More precisely, the derivation does not have a version attribute.
 
+- [package]{#package}
+
+  1. A software package; a collection of files and other data.
+
+  2. A [package attribute set].
+
+- [package attribute set]{#package-attribute-set}
+
+  An [attribute set](@docroot@/language/values.md#attribute-set) containing the attribute `type = "derivation";` (derivation for historical reasons), as well as other attributes, such as
+  - attributes that refer to the files of a [package], typically in the form of [derivation outputs](#output),
+  - attributes that declare something about how the package is supposed to be installed or used,
+  - other metadata or arbitrary attributes.
+
+  [package attribute set]: #package-attribute-set
+
 - [string interpolation]{#gloss-string-interpolation}
 
   Expanding expressions enclosed in `${ }` within a [string], [path], or [attribute name].
@@ -282,3 +308,6 @@
   These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting.
 
   See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/contributing/experimental-features.md).
+
+
+[Nix language]: ./language/index.md

doc/manual/src/installation/installing-binary.md

@@ -1,26 +1,60 @@
 # Installing a Binary Distribution
 
-The easiest way to install Nix is to run the following command:
+To install the latest version Nix, run the following command:
 
 ```console
 $ curl -L https://nixos.org/nix/install | sh
 ```
 
-This will run the installer interactively (causing it to explain what
+This performs the default type of installation for your platform:
-it is doing more explicitly), and perform the default "type" of install
-for your platform:
-- single-user on Linux
-- multi-user on macOS
 
-  > **Notes on read-only filesystem root in macOS 10.15 Catalina +**
+- [Multi-user](#multi-user-installation):
-  >
+  - Linux with systemd and without SELinux
-  > - It took some time to support this cleanly. You may see posts,
+  - macOS
-  >   examples, and tutorials using obsolete workarounds.
+- [Single-user](#single-user-installation):
-  > - Supporting it cleanly made macOS installs too complex to qualify
+  - Linux without systemd
-  >   as single-user, so this type is no longer supported on macOS.
+  - Linux with SELinux
 
-We recommend the multi-user install if it supports your platform and
+We recommend the multi-user installation if it supports your platform and you can authenticate with `sudo`.
-you can authenticate with `sudo`.
+
+The installer can configured with various command line arguments and environment variables.
+To show available command line flags:
+
+```console
+$ curl -L https://nixos.org/nix/install | sh -s -- --help
+```
+
+To check what it does and how it can be customised further, [download and edit the second-stage installation script](#installing-from-a-binary-tarball).
+
+# Installing a pinned Nix version from a URL
+
+Version-specific installation URLs for all Nix versions since 1.11.16 can be found at [releases.nixos.org](https://releases.nixos.org/?prefix=nix/).
+The directory for each version contains the corresponding SHA-256 hash.
+
+All installation scripts are invoked the same way:
+
+```console
+$ export VERSION=2.19.2 
+$ curl -L https://releases.nixos.org/nix/nix-$VERSION/install | sh
+```
+
+# Multi User Installation
+
+The multi-user Nix installation creates system users and a system service for the Nix daemon.
+
+Supported systems:
+
+- Linux running systemd, with SELinux disabled
+- macOS
+
+To explicitly instruct the installer to perform a multi-user installation on your system:
+
+```console
+$ curl -L https://nixos.org/nix/install | sh -s -- --daemon
+```
+
+You can run this under your usual user account or `root`.
+The script will invoke `sudo` as needed.
 
 # Single User Installation
 
@@ -30,60 +64,48 @@ To explicitly select a single-user installation on your system:
 $ curl -L https://nixos.org/nix/install | sh -s -- --no-daemon
 ```
 
-This will perform a single-user installation of Nix, meaning that `/nix`
+In a single-user installation, `/nix` is owned by the invoking user.
-is owned by the invoking user. You can run this under your usual user
+The script will invoke `sudo` to create `/nix` if it doesn’t already exist.
-account or root. The script will invoke `sudo` to create `/nix`
+If you don’t have `sudo`, manually create `/nix` as `root`:
-if it doesn’t already exist. If you don’t have `sudo`, you should
-manually create `/nix` first as root, e.g.:
 
 ```console
-$ mkdir /nix
+$ su root
-$ chown alice /nix
+# mkdir /nix
+# chown alice /nix
 ```
 
-The install script will modify the first writable file from amongst
+# Installing from a binary tarball
-`.bash_profile`, `.bash_login` and `.profile` to source
-`~/.nix-profile/etc/profile.d/nix.sh`. You can set the
-`NIX_INSTALLER_NO_MODIFY_PROFILE` environment variable before executing
-the install script to disable this behaviour.
 
-# Multi User Installation
+You can also download a binary tarball that contains Nix and all its dependencies:
+- Choose a [version](https://releases.nixos.org/?prefix=nix/) and [system type](../contributing/hacking.md#platforms)
+- Download and unpack the tarball
+- Run the installer
 
-The multi-user Nix installation creates system users, and a system
+> **Example**
-service for the Nix daemon.
-
-**Supported Systems**
-- Linux running systemd, with SELinux disabled
-- macOS
-
-You can instruct the installer to perform a multi-user installation on
-your system:
-
-```console
-$ curl -L https://nixos.org/nix/install | sh -s -- --daemon
-```
-
-The multi-user installation of Nix will create build users between the
-user IDs 30001 and 30032, and a group with the group ID 30000. You
-can run this under your usual user account or root. The script
-will invoke `sudo` as needed.
-
-> **Note**
 >
-> If you need Nix to use a different group ID or user ID set, you will
+> ```console
-> have to download the tarball manually and [edit the install
+> $ pushd $(mktemp -d)
-> script](#installing-from-a-binary-tarball).
+> $ export VERSION=2.19.2
+> $ export SYSTEM=x86_64-linux
+> $ curl -LO https://releases.nixos.org/nix/nix-$VERSION/nix-$VERSION-$SYSTEM.tar.xz
+> $ tar xfj nix-$VERSION-$SYSTEM.tar.xz
+> $ cd nix-$VERSION-$SYSTEM
+> $ ./install
+> $ popd
+> ```
 
-The installer will modify `/etc/bashrc`, and `/etc/zshrc` if they exist.
+The installer can be customised with the environment variables declared in the file named `install-multi-user`.
-The installer will first back up these files with a `.backup-before-nix`
+
-extension. The installer will also create `/etc/profile.d/nix.sh`.
+## Native packages for Linux distributions
+
+The Nix community maintains installers for some Linux distributions in their native packaging format(https://nix-community.github.io/nix-installers/).
 
 # macOS Installation
 
+<!-- anchors to catch existing links -->
 []{#sect-macos-installation-change-store-prefix}[]{#sect-macos-installation-encrypted-volume}[]{#sect-macos-installation-symlink}[]{#sect-macos-installation-recommended-notes}
-<!-- Note: anchors above to catch permalinks to old explanations -->
 
-We believe we have ironed out how to cleanly support the read-only root
+We believe we have ironed out how to cleanly support the read-only root file system
 on modern macOS. New installs will do this automatically.
 
 This section previously detailed the situation, options, and trade-offs,
@@ -126,33 +148,3 @@ this to run the installer, but it may help if you run into trouble:
   boot process to avoid problems loading or restoring any programs that
   need access to your Nix store
 
-# Installing a pinned Nix version from a URL
-
-Version-specific installation URLs for all Nix versions
-since 1.11.16 can be found at [releases.nixos.org](https://releases.nixos.org/?prefix=nix/).
-The corresponding SHA-256 hash can be found in the directory for the given version.
-
-These install scripts can be used the same as usual:
-
-```console
-$ curl -L https://releases.nixos.org/nix/nix-<version>/install | sh
-```
-
-# Installing from a binary tarball
-
-You can also download a binary tarball that contains Nix and all its
-dependencies. (This is what the install script at
-<https://nixos.org/nix/install> does automatically.) You should unpack
-it somewhere (e.g. in `/tmp`), and then run the script named `install`
-inside the binary tarball:
-
-```console
-$ cd /tmp
-$ tar xfj nix-1.8-x86_64-darwin.tar.bz2
-$ cd nix-1.8-x86_64-darwin
-$ ./install
-```
-
-If you need to edit the multi-user installation script to use different
-group ID or a different user ID range, modify the variables set in the
-file named `install-multi-user`.

doc/manual/src/installation/prerequisites-source.md

@@ -32,11 +32,15 @@
     your distribution does not provide it, please install it from
     <http://www.sqlite.org/>.
 
-  - The [Boehm garbage collector](http://www.hboehm.info/gc/) to reduce
+  - The [Boehm garbage collector (`bdw-gc`)](http://www.hboehm.info/gc/) to reduce
-    the evaluator’s memory consumption (optional). To enable it, install
+    the evaluator’s memory consumption (optional).
+
+    To enable it, install
     `pkgconfig` and the Boehm garbage collector, and pass the flag
     `--enable-gc` to `configure`.
 
+    For `bdw-gc` <= 8.2.4 Nix needs a [small patch](https://github.com/NixOS/nix/blob/ac4d2e7b857acdfeac35ac8a592bdecee2d29838/boehmgc-traceable_allocator-public.diff) to be applied.
+
   - The `boost` library of version 1.66.0 or higher. It can be obtained
     from the official web site <https://www.boost.org/>.
 
@@ -72,7 +76,7 @@
     This is an optional dependency and can be disabled
     by providing a `--disable-cpuid` to the `configure` script.
 
-  - Unless `./configure --disable-tests` is specified, GoogleTest (GTest) and
+  - Unless `./configure --disable-unit-tests` is specified, GoogleTest (GTest) and
     RapidCheck are required, which are available at
     <https://google.github.io/googletest/> and
     <https://github.com/emil-e/rapidcheck> respectively.

doc/manual/src/installation/upgrading.md

@@ -1,14 +1,40 @@
 # Upgrading Nix
 
-Multi-user Nix users on macOS can upgrade Nix by running: `sudo -i sh -c
+> **Note**
-'nix-channel --update &&
+>
-nix-env --install --attr nixpkgs.nix &&
+> These upgrade instructions apply where Nix was installed following the [installation instructions in this manual](./index.md).
-launchctl remove org.nixos.nix-daemon &&
-launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist'`
 
-Single-user installations of Nix should run this: `nix-channel --update;
+Check which Nix version will be installed, for example from one of the [release channels](http://channels.nixos.org/) such as `nixpkgs-unstable`:
-nix-env --install --attr nixpkgs.nix nixpkgs.cacert`
 
-Multi-user Nix users on Linux should run this with sudo: `nix-channel
+```console
---update; nix-env --install --attr nixpkgs.nix nixpkgs.cacert; systemctl
+$ nix-shell -p nix -I nixpkgs=channel:nixpkgs-unstable --run "nix --version"
-daemon-reload; systemctl restart nix-daemon`
+nix (Nix) 2.18.1
+```
+
+> **Warning**
+>
+> Writing to the [local store](@docroot@/store/types/local-store.md) with a newer version of Nix, for example by building derivations with [`nix-build`](@docroot@/command-ref/nix-build.md) or [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md), may change the database schema!
+> Reverting to an older version of Nix may therefore require purging the store database before it can be used.
+
+### Linux multi-user
+
+```console
+$ sudo su
+# nix-env --install --file '<nixpkgs>' --attr nix cacert -I nixpkgs=channel:nixpkgs-unstable
+# systemctl daemon-reload
+# systemctl restart nix-daemon
+```
+
+## macOS multi-user
+
+```console
+$ sudo nix-env --install --file '<nixpkgs>' --attr nix -I nixpkgs=channel:nixpkgs-unstable
+$ sudo launchctl remove org.nixos.nix-daemon
+$ sudo launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist
+```
+
+## Single-user all platforms
+
+```console
+$ nix-env --install --file '<nixpkgs>' --attr nix cacert -I nixpkgs=channel:nixpkgs-unstable
+```

doc/manual/src/json/derivation.md

@@ -0,0 +1,71 @@
+# Derivation JSON Format
+
+> **Warning**
+>
+> This JSON format is currently
+> [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command)
+> and subject to change.
+
+The JSON serialization of a
+[derivations](@docroot@/glossary.md#gloss-store-derivation)
+is a JSON object with the following fields:
+
+* `name`:
+  The name of the derivation.
+  This is used when calculating the store paths of the derivation's outputs.
+
+* `outputs`:
+  Information about the output paths of the derivation.
+  This is a JSON object with one member per output, where the key is the output name and the value is a JSON object with these fields:
+
+  * `path`: The output path.
+
+  * `hashAlgo`:
+    For fixed-output derivations, the hashing algorithm (e.g. `sha256`), optionally prefixed by `r:` if `hash` denotes a NAR hash rather than a flat file hash.
+
+  * `hash`:
+    For fixed-output derivations, the expected content hash in base-16.
+
+  > **Example**
+  >
+  > ```json
+  > "outputs": {
+  >   "out": {
+  >     "path": "/nix/store/2543j7c6jn75blc3drf4g5vhb1rhdq29-source",
+  >     "hashAlgo": "r:sha256",
+  >     "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62"
+  >   }
+  > }
+  > ```
+
+* `inputSrcs`:
+  A list of store paths on which this derivation depends.
+
+* `inputDrvs`:
+  A JSON object specifying the derivations on which this derivation depends, and what outputs of those derivations.
+
+  > **Example**
+  >
+  > ```json
+  > "inputDrvs": {
+  >   "/nix/store/6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"],
+  >   "/nix/store/fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"]
+  > }
+  > ```
+
+  specifies that this derivation depends on the `dev` output of `curl`, and the `out` output of `unzip`.
+
+* `system`:
+  The system type on which this derivation is to be built
+  (e.g. `x86_64-linux`).
+
+* `builder`:
+  The absolute path of the program to be executed to run the build.
+  Typically this is the `bash` shell
+  (e.g. `/nix/store/r3j288vpmczbl500w6zz89gyfa4nr0b1-bash-4.4-p23/bin/bash`).
+
+* `args`:
+  The command-line arguments passed to the `builder`.
+
+* `env`:
+  The environment passed to the `builder`.

doc/manual/src/json/store-object-info.md

@@ -0,0 +1,97 @@
+# Store object info JSON format
+
+> **Warning**
+>
+> This JSON format is currently
+> [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command)
+> and subject to change.
+
+Info about a [store object].
+
+* `path`:
+
+  [Store path][store path] to the given store object.
+
+* `narHash`:
+
+  Hash of the [file system object] part of the store object when serialized as a [Nix Archive](#gloss-nar).
+
+* `narSize`:
+
+  Size of the [file system object] part of the store object when serialized as a [Nix Archive](#gloss-nar).
+
+* `references`:
+
+  An array of [store paths][store path], possibly including this one.
+
+* `ca` (optional):
+
+  Content address of this store object's file system object, used to compute its store path.
+
+[store path]: @docroot@/glossary.md#gloss-store-path
+[file system object]: @docroot@/store/file-system-object.md
+
+## Impure fields
+
+These are not intrinsic properties of the store object.
+In other words, the same store object residing in different store could have different values for these properties.
+
+* `deriver` (optional):
+
+  The path to the [derivation] from which this store object is produced.
+
+  [derivation]: @docroot@/glossary.md#gloss-store-derivation
+
+* `registrationTime` (optional):
+
+  When this derivation was added to the store.
+
+* `ultimate` (optional):
+
+  Whether this store object is trusted because we built it ourselves, rather than substituted a build product from elsewhere.
+
+* `signatures` (optional):
+
+  Signatures claiming that this store object is what it claims to be.
+  Not relevant for [content-addressed] store objects,
+  but useful for [input-addressed] store objects.
+
+  [content-addressed]: @docroot@/glossary.md#gloss-content-addressed-store-object
+  [input-addressed]: @docroot@/glossary.md#gloss-input-addressed-store-object
+
+### `.narinfo` extra fields
+
+This meta data is specific to the "binary cache" family of Nix store types.
+This information is not intrinsic to the store object, but about how it is stored.
+
+* `url`:
+
+  Where to download a compressed archive of the file system objects of this store object.
+
+* `compression`:
+
+  The compression format that the archive is in.
+
+* `fileHash`:
+
+  A digest for the compressed archive itself, as opposed to the data contained within.
+
+* `fileSize`:
+
+  The size of the compressed archive itself.
+
+## Computed closure fields
+
+These fields are not stored at all, but computed by traverising the other other fields across all the store objects in a [closure].
+
+* `closureSize`:
+
+  The total size of the compressed archive itself for this object, and the compressed archive of every object in this object's [closure].
+
+### `.narinfo` extra fields
+
+* `closureSize`:
+
+  The total size of this store object and every other object in its [closure].
+
+[closure]: @docroot@/glossary.md#gloss-closure

doc/manual/src/language/advanced-attributes.md

@@ -257,29 +257,18 @@ Derivations can declare some infrequently used optional attributes.
     of the environment (typically, a few hundred kilobyte).
 
   - [`preferLocalBuild`]{#adv-attr-preferLocalBuild}\
-    If this attribute is set to `true` and [distributed building is
+    If this attribute is set to `true` and [distributed building is enabled](../advanced-topics/distributed-builds.md), then, if possible, the derivation will be built locally instead of being forwarded to a remote machine.
-    enabled](../advanced-topics/distributed-builds.md), then, if
+    This is useful for derivations that are cheapest to build locally.
-    possible, the derivation will be built locally instead of forwarded
-    to a remote machine. This is appropriate for trivial builders
-    where the cost of doing a download or remote build would exceed
-    the cost of building locally.
 
   - [`allowSubstitutes`]{#adv-attr-allowSubstitutes}\
-    If this attribute is set to `false`, then Nix will always build this
+    If this attribute is set to `false`, then Nix will always build this derivation (locally or remotely); it will not try to substitute its outputs.
-    derivation; it will not try to substitute its outputs. This is
+    This is useful for derivations that are cheaper to build than to substitute.
-    useful for very trivial derivations (such as `writeText` in Nixpkgs)
-    that are cheaper to build than to substitute from a binary cache.
 
-    You may disable the effects of this attibute by enabling the
+    This attribute can be ignored by setting [`always-allow-substitutes`](@docroot@/command-ref/conf-file.md#conf-always-allow-substitutes) to `true`.
-    `always-allow-substitutes` configuration option in Nix.
 
     > **Note**
     >
-    > You need to have a builder configured which satisfies the
+    > If set to `false`, the [`builder`](./derivations.md#attr-builder) should be able to run on the system type specified in the [`system` attribute](./derivations.md#attr-system), since the derivation cannot be substituted.
-    > derivation’s `system` attribute, since the derivation cannot be
-    > substituted. Thus it is usually a good idea to align `system` with
-    > `builtins.currentSystem` when setting `allowSubstitutes` to
-    > `false`. For most trivial derivations this should be the case.
 
   - [`__structuredAttrs`]{#adv-attr-structuredAttrs}\
     If the special attribute `__structuredAttrs` is set to `true`, the other derivation

doc/manual/src/language/derivations.md

@@ -274,7 +274,7 @@ The [`builder`](#attr-builder) is executed as follows:
     directory (typically, `/nix/store`).
 
   - `NIX_ATTRS_JSON_FILE` & `NIX_ATTRS_SH_FILE` if `__structuredAttrs`
-    is set to `true` for the dervation. A detailed explanation of this
+    is set to `true` for the derivation. A detailed explanation of this
     behavior can be found in the
     [section about structured attrs](./advanced-attributes.md#adv-attr-structuredAttrs).
 

doc/manual/src/language/string-interpolation.md

@@ -189,7 +189,7 @@ If neither is present, an error is thrown.
 > "${a}"
 > ```
 >
->     error: cannot coerce a set to a string
+>     error: cannot coerce a set to a string: { }
 >
 >            at «string»:4:2:
 >

doc/manual/src/quick-start.md

@@ -10,7 +10,6 @@ For more in-depth information you are kindly referred to subsequent chapters.
    ```
 
    The install script will use `sudo`, so make sure you have sufficient rights.
-   On Linux, `--daemon` can be omitted for a single-user install.
 
    For other installation methods, see the detailed [installation instructions](installation/index.md).
 

doc/manual/src/release-notes/rl-2.20.md

@@ -0,0 +1,169 @@
+# Release 2.20.0 (2024-01-29)
+
+- Option `allowed-uris` can now match whole schemes in URIs without slashes [#9547](https://github.com/NixOS/nix/pull/9547)
+
+  If a scheme, such as `github:` is specified in the `allowed-uris` option, all URIs starting with `github:` are allowed.
+  Previously this only worked for schemes whose URIs used the `://` syntax.
+
+- Include cgroup stats when building through the daemon [#9598](https://github.com/NixOS/nix/pull/9598)
+
+  Nix now also reports cgroup statistics when building through the Nix daemon and when doing remote builds using `ssh-ng`,
+  if both sides of the connection are using Nix 2.20 or newer.
+
+- Disallow empty search regex in `nix search` [#9481](https://github.com/NixOS/nix/pull/9481)
+
+  [`nix search`](@docroot@/command-ref/new-cli/nix3-search.md) now requires a search regex to be passed. To show all packages, use `^`.
+
+- Add new `eval-system` setting [#4093](https://github.com/NixOS/nix/pull/4093)
+
+  Add a new `eval-system` option.
+  Unlike `system`, it just overrides the value of `builtins.currentSystem`.
+  This is more useful than overriding `system`, because you can build these derivations on remote builders which can work on the given system.
+  In contrast, `system` also affects scheduling which will cause Nix to build those derivations locally even if that doesn't make sense.
+
+  `eval-system` only takes effect if it is non-empty.
+  If empty (the default) `system` is used as before, so there is no breakage.
+
+- Import-from-derivation builds the derivation in the build store [#9661](https://github.com/NixOS/nix/pull/9661)
+
+  When using `--eval-store`, `import`ing from a derivation will now result in the derivation being built on the build store, i.e. the store specified in the `store` Nix option.
+
+  Because the resulting Nix expression must be copied back to the evaluation store in order to be imported, this requires the evaluation store to trust the build store's signatures.
+
+- Mounted SSH Store [#7890](https://github.com/NixOS/nix/issues/7890) [#7912](https://github.com/NixOS/nix/pull/7912)
+
+  Introduced the store [`mounted-ssh-ng://`](@docroot@/command-ref/new-cli/nix3-help-stores.md).
+  This store allows full access to a Nix store on a remote machine and additionally requires that the store be mounted in the local filesystem.
+
+- Rename `nix show-config` to `nix config show` [#7672](https://github.com/NixOS/nix/issues/7672) [#9477](https://github.com/NixOS/nix/pull/9477)
+
+  `nix show-config` was renamed to `nix config show`, and `nix doctor` was renamed to `nix config check`, to be more consistent with the rest of the command line interface.
+
+- Add command `nix hash convert` [#9452](https://github.com/NixOS/nix/pull/9452)
+
+  This replaces the old `nix hash to-*` commands, which are still available but will emit a deprecation warning. Please convert as follows:
+
+  - `nix hash to-base16 $hash1 $hash2`: Use `nix hash convert --to base16 $hash1 $hash2` instead.
+  - `nix hash to-base32 $hash1 $hash2`: Use `nix hash convert --to nix32 $hash1 $hash2` instead.
+  - `nix hash to-base64 $hash1 $hash2`: Use `nix hash convert --to base64 $hash1 $hash2` instead.
+  - `nix hash to-sri $hash1 $hash2`: : Use `nix hash convert --to sri $hash1 $hash2` or even just `nix hash convert $hash1 $hash2` instead.
+
+- Rename hash format `base32` to `nix32` [#9452](https://github.com/NixOS/nix/pull/9452)
+
+  Hash format `base32` was renamed to `nix32` since it used a special Nix-specific character set for
+  [Base32](https://en.wikipedia.org/wiki/Base32).
+
+- `nix profile` now allows referring to elements by human-readable names [#8678](https://github.com/NixOS/nix/pull/8678)
+
+  [`nix profile`](@docroot@/command-ref/new-cli/nix3-profile.md) now uses names to refer to installed packages when running [`list`](@docroot@/command-ref/new-cli/nix3-profile-list.md), [`remove`](@docroot@/command-ref/new-cli/nix3-profile-remove.md) or [`upgrade`](@docroot@/command-ref/new-cli/nix3-profile-upgrade.md) as opposed to indices. Profile element names are generated when a package is installed and remain the same until the package is removed.
+
+  **Warning**: The `manifest.nix` file used to record the contents of profiles has changed. Nix will automatically upgrade profiles to the new version when you modify the profile. After that, the profile can no longer be used by older versions of Nix.
+
+- Give `nix store add` a `--hash-algo` flag [#9809](https://github.com/NixOS/nix/pull/9809)
+
+  Adds a missing feature that was present in the old CLI, and matches our
+  plans to have similar flags for `nix hash convert` and `nix hash path`.
+
+- Coercion errors include the failing value
+
+  The `error: cannot coerce a <TYPE> to a string` message now includes the value
+  which caused the error.
+
+  Before:
+
+  ```
+  error: cannot coerce a set to a string
+  ```
+
+  After:
+
+  ```
+  error: cannot coerce a set to a string: { aesSupport = «thunk»;
+    avx2Support = «thunk»; avx512Support = «thunk»; avxSupport = «thunk»;
+    canExecute = «thunk»; config = «thunk»; darwinArch = «thunk»; darwinMinVersion
+    = «thunk»; darwinMinVersionVariable = «thunk»; darwinPlatform = «thunk»; «84
+    attributes elided»}
+  ```
+
+- Type errors include the failing value
+
+  In errors like `value is an integer while a list was expected`, the message now
+  includes the failing value.
+
+  Before:
+
+  ```
+  error: value is a set while a string was expected
+  ```
+
+  After:
+
+  ```
+  error: expected a string but found a set: { ghc810 = «thunk»;
+    ghc8102Binary = «thunk»; ghc8107 = «thunk»; ghc8107Binary = «thunk»;
+    ghc865Binary = «thunk»; ghc90 = «thunk»; ghc902 = «thunk»; ghc92 = «thunk»;
+    ghc924Binary = «thunk»; ghc925 = «thunk»;  «17 attributes elided»}
+  ```
+
+- Source locations are printed more consistently in errors [#561](https://github.com/NixOS/nix/issues/561) [#9555](https://github.com/NixOS/nix/pull/9555)
+
+  Source location information is now included in error messages more
+  consistently. Given this code:
+
+  ```nix
+  let
+    attr = {foo = "bar";};
+    key = {};
+  in
+    attr.${key}
+  ```
+
+  Previously, Nix would show this unhelpful message when attempting to evaluate
+  it:
+
+  ```
+  error:
+         … while evaluating an attribute name
+
+         error: value is a set while a string was expected
+  ```
+
+  Now, the error message displays where the problematic value was found:
+
+  ```
+  error:
+         … while evaluating an attribute name
+
+           at bad.nix:4:11:
+
+              3|   key = {};
+              4| in attr.${key}
+               |           ^
+              5|
+
+         error: expected a string but found a set
+  ```
+
+- Some stack overflow segfaults are fixed [#9616](https://github.com/NixOS/nix/issues/9616) [#9617](https://github.com/NixOS/nix/pull/9617)
+
+  The number of nested function calls has been restricted, to detect and report
+  infinite function call recursions. The default maximum call depth is 10,000 and
+  can be set with [the `max-call-depth`
+  option](@docroot@/command-ref/conf-file.md#conf-max-call-depth).
+
+  This replaces the `stack overflow (possible infinite recursion)` message.
+
+- Better error reporting for `with` expressions [#9658](https://github.com/NixOS/nix/pull/9658)
+
+  `with` expressions using non-attrset values to resolve variables are now reported with proper positions, e.g.
+
+  ```
+  nix-repl> with 1; a
+  error:
+         … while evaluating the first subexpression of a with expression
+           at «string»:1:1:
+              1| with 1; a
+               | ^
+
+         error: expected a set but found an integer
+  ```

flake.lock

@@ -32,34 +32,18 @@
         "type": "github"
       }
     },
-    "lowdown-src": {
-      "flake": false,
-      "locked": {
-        "lastModified": 1633514407,
-        "narHash": "sha256-Dw32tiMjdK9t3ETl5fzGrutQTzh2rufgZV4A/BbxuD4=",
-        "owner": "kristapsdz",
-        "repo": "lowdown",
-        "rev": "d2c2b44ff6c27b936ec27358a2653caaef8f73b8",
-        "type": "github"
-      },
-      "original": {
-        "owner": "kristapsdz",
-        "repo": "lowdown",
-        "type": "github"
-      }
-    },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1701355166,
+        "lastModified": 1705033721,
-        "narHash": "sha256-4V7XMI0Gd+y0zsi++cEHd99u3GNL0xSTGRmiWKzGnUQ=",
+        "narHash": "sha256-K5eJHmL1/kev6WuqyqqbS1cdNnSidIZ3jeqJ7GbrYnQ=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "36c4ac09e9bebcec1fa7b7539cddb0c9e837409c",
+        "rev": "a1982c92d8980a0114372973cbdfe0a307f1bdea",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "staging-23.05",
+        "ref": "nixos-23.05-small",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -84,7 +68,6 @@
       "inputs": {
         "flake-compat": "flake-compat",
         "libgit2": "libgit2",
-        "lowdown-src": "lowdown-src",
         "nixpkgs": "nixpkgs",
         "nixpkgs-regression": "nixpkgs-regression"
       }

flake.nix

@@ -1,23 +1,24 @@
 {
   description = "The purely functional package manager";
 
-  # TODO Go back to nixos-23.05-small once
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small";
-  # https://github.com/NixOS/nixpkgs/pull/271202 is merged.
-  #
-  # Also, do not grab arbitrary further staging commits. This PR was
-  # carefully made to be based on release-23.05 and just contain
-  # rebuild-causing changes to packages that Nix actually uses.
-  inputs.nixpkgs.url = "github:NixOS/nixpkgs/staging-23.05";
   inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2";
-  inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; };
   inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; };
   inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; };
 
-  outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src, flake-compat, libgit2 }:
+  outputs = { self, nixpkgs, nixpkgs-regression, libgit2, ... }:
 
     let
       inherit (nixpkgs) lib;
 
+      # Experimental fileset library: https://github.com/NixOS/nixpkgs/pull/222981
+      # Not an "idiomatic" flake input because:
+      #  - Propagation to dependent locks: https://github.com/NixOS/nix/issues/7730
+      #  - Subflake would download redundant and huge parent flake
+      #  - No git tree hash support: https://github.com/NixOS/nix/issues/6044
+      inherit (import (builtins.fetchTarball { url = "https://github.com/NixOS/nix/archive/1bdcd7fc8a6a40b2e805bad759b36e64e911036b.tar.gz"; sha256 = "sha256:14ljlpdsp4x7h1fkhbmc4bd3vsqnx8zdql4h3037wh09ad6a0893"; }))
+        fileset;
+
       officialRelease = false;
 
       # Set to true to build the release notes for the next release.
@@ -36,11 +37,26 @@
       systems = linuxSystems ++ darwinSystems;
 
       crossSystems = [
-        "armv6l-linux" "armv7l-linux"
+        "armv6l-unknown-linux-gnueabihf"
-        "x86_64-freebsd13" "x86_64-netbsd"
+        "armv7l-unknown-linux-gnueabihf"
+        "x86_64-unknown-freebsd13"
+        "x86_64-unknown-netbsd"
       ];
 
-      stdenvs = [ "gccStdenv" "clangStdenv" "clang11Stdenv" "stdenv" "libcxxStdenv" "ccacheStdenv" ];
+      # Nix doesn't yet build on this platform, so we put it in a
+      # separate list. We just use this for `devShells` and
+      # `nixpkgsFor`, which this depends on.
+      shellCrossSystems = crossSystems ++ [
+        "x86_64-w64-mingw32"
+      ];
+
+      stdenvs = [
+        "ccacheStdenv"
+        "clangStdenv"
+        "gccStdenv"
+        "libcxxStdenv"
+        "stdenv"
+      ];
 
       forAllSystems = lib.genAttrs systems;
 
@@ -55,57 +71,6 @@
             })
             stdenvs);
 
-      # Experimental fileset library: https://github.com/NixOS/nixpkgs/pull/222981
-      # Not an "idiomatic" flake input because:
-      #  - Propagation to dependent locks: https://github.com/NixOS/nix/issues/7730
-      #  - Subflake would download redundant and huge parent flake
-      #  - No git tree hash support: https://github.com/NixOS/nix/issues/6044
-      inherit (import (builtins.fetchTarball { url = "https://github.com/NixOS/nix/archive/1bdcd7fc8a6a40b2e805bad759b36e64e911036b.tar.gz"; sha256 = "sha256:14ljlpdsp4x7h1fkhbmc4bd3vsqnx8zdql4h3037wh09ad6a0893"; }))
-        fileset;
-
-      baseFiles =
-        # .gitignore has already been processed, so any changes in it are irrelevant
-        # at this point. It is not represented verbatim for test purposes because
-        # that would interfere with repo semantics.
-        fileset.fileFilter (f: f.name != ".gitignore") ./.;
-
-      configureFiles = fileset.unions [
-        ./.version
-        ./configure.ac
-        ./m4
-        # TODO: do we really need README.md? It doesn't seem used in the build.
-        ./README.md
-      ];
-
-      topLevelBuildFiles = fileset.unions [
-        ./local.mk
-        ./Makefile
-        ./Makefile.config.in
-        ./mk
-      ];
-
-      functionalTestFiles = fileset.unions [
-        ./tests/functional
-        (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts)
-      ];
-
-      nixSrc = fileset.toSource {
-        root = ./.;
-        fileset = fileset.intersect baseFiles (fileset.unions [
-          configureFiles
-          topLevelBuildFiles
-          ./boehmgc-coroutine-sp-fallback.diff
-          ./doc
-          ./misc
-          ./precompiled-headers.h
-          ./src
-          ./tests/unit
-          ./COPYING
-          ./scripts/local.mk
-          functionalTestFiles
-        ]);
-      };
-
       # Memoize nixpkgs for different platforms for efficiency.
       nixpkgsFor = forAllSystems
         (system: let
@@ -114,8 +79,8 @@
               inherit system;
             };
             crossSystem = if crossSystem == null then null else {
-              system = crossSystem;
+              config = crossSystem;
-            } // lib.optionalAttrs (crossSystem == "x86_64-freebsd13") {
+            } // lib.optionalAttrs (crossSystem == "x86_64-unknown-freebsd13") {
               useLLVM = true;
             };
             overlays = [
@@ -127,407 +92,114 @@
         in {
           inherit stdenvs native;
           static = native.pkgsStatic;
-          cross = forAllCrossSystems (crossSystem: make-pkgs crossSystem "stdenv");
+          cross = lib.genAttrs shellCrossSystems (crossSystem: make-pkgs crossSystem "stdenv");
         });
 
-      commonDeps =
+      installScriptFor = tarballs:
-        { pkgs
+        nixpkgsFor.x86_64-linux.native.callPackage ./scripts/installer.nix {
-        , isStatic ? pkgs.stdenv.hostPlatform.isStatic
+          inherit tarballs;
-        }:
-        with pkgs; rec {
-        # Use "busybox-sandbox-shell" if present,
-        # if not (legacy) fallback and hope it's sufficient.
-        sh = pkgs.busybox-sandbox-shell or (busybox.override {
-          useMusl = true;
-          enableStatic = true;
-          enableMinimal = true;
-          extraConfig = ''
-            CONFIG_FEATURE_FANCY_ECHO y
-            CONFIG_FEATURE_SH_MATH y
-            CONFIG_FEATURE_SH_MATH_64 y
-
-            CONFIG_ASH y
-            CONFIG_ASH_OPTIMIZE_FOR_SIZE y
-
-            CONFIG_ASH_ALIAS y
-            CONFIG_ASH_BASH_COMPAT y
-            CONFIG_ASH_CMDCMD y
-            CONFIG_ASH_ECHO y
-            CONFIG_ASH_GETOPTS y
-            CONFIG_ASH_INTERNAL_GLOB y
-            CONFIG_ASH_JOB_CONTROL y
-            CONFIG_ASH_PRINTF y
-            CONFIG_ASH_TEST y
-          '';
-        });
-
-        configureFlags =
-          lib.optionals stdenv.isLinux [
-            "--with-boost=${boost}/lib"
-            "--with-sandbox-shell=${sh}/bin/busybox"
-          ]
-          ++ lib.optionals (stdenv.isLinux && !(isStatic && stdenv.system == "aarch64-linux")) [
-            "LDFLAGS=-fuse-ld=gold"
-          ];
-
-        testConfigureFlags = [
-          "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include"
-        ] ++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [
-          "--enable-install-unit-tests"
-          "--with-check-bin-dir=${builtins.placeholder "check"}/bin"
-          "--with-check-lib-dir=${builtins.placeholder "check"}/lib"
-        ];
-
-        internalApiDocsConfigureFlags = [
-          "--enable-internal-api-docs"
-        ];
-
-        changelog-d = pkgs.buildPackages.callPackage ./misc/changelog-d.nix { };
-
-        nativeBuildDeps =
-          [
-            buildPackages.bison
-            buildPackages.flex
-            (lib.getBin buildPackages.lowdown-nix)
-            buildPackages.mdbook
-            buildPackages.mdbook-linkcheck
-            buildPackages.autoconf-archive
-            buildPackages.autoreconfHook
-            buildPackages.pkg-config
-
-            # Tests
-            buildPackages.git
-            buildPackages.mercurial # FIXME: remove? only needed for tests
-            buildPackages.jq # Also for custom mdBook preprocessor.
-            buildPackages.openssh # only needed for tests (ssh-keygen)
-          ]
-          ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)]
-          # Official releases don't have rl-next, so we don't need to compile a changelog
-          ++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d
-          ;
-
-        buildDeps =
-          [ curl
-            bzip2 xz brotli editline
-            openssl sqlite
-            libarchive
-            (pkgs.libgit2.overrideAttrs (attrs: {
-              src = libgit2;
-              version = libgit2.lastModifiedDate;
-              cmakeFlags = (attrs.cmakeFlags or []) ++ ["-DUSE_SSH=exec"];
-            }))
-            boost
-            lowdown-nix
-            libsodium
-          ]
-          ++ lib.optionals stdenv.isLinux [libseccomp]
-          ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid;
-
-        checkDeps = [
-          gtest
-          rapidcheck
-        ];
-
-        internalApiDocsDeps = [
-          buildPackages.doxygen
-        ];
-
-        awsDeps = lib.optional (stdenv.isLinux || stdenv.isDarwin)
-          (aws-sdk-cpp.override {
-            apis = ["s3" "transfer"];
-            customMemoryManagement = false;
-          });
-
-        propagatedDeps =
-          [ ((boehmgc.override {
-              enableLargeConfig = true;
-            }).overrideAttrs(o: {
-              patches = (o.patches or []) ++ [
-                ./boehmgc-coroutine-sp-fallback.diff
-
-                # https://github.com/ivmai/bdwgc/pull/586
-                ./boehmgc-traceable_allocator-public.diff
-              ];
-            })
-            )
-            nlohmann_json
-          ];
-      };
-
-      installScriptFor = systems:
-        with nixpkgsFor.x86_64-linux.native;
-        runCommand "installer-script"
-          { buildInputs = [ nix ];
-          }
-          ''
-            mkdir -p $out/nix-support
-
-            # Converts /nix/store/50p3qk8k...-nix-2.4pre20201102_550e11f/bin/nix to 50p3qk8k.../bin/nix.
-            tarballPath() {
-              # Remove the store prefix
-              local path=''${1#${builtins.storeDir}/}
-              # Get the path relative to the derivation root
-              local rest=''${path#*/}
-              # Get the derivation hash
-              local drvHash=''${path%%-*}
-              echo "$drvHash/$rest"
-            }
-
-            substitute ${./scripts/install.in} $out/install \
-              ${pkgs.lib.concatMapStrings
-                (system: let
-                    tarball = if builtins.elem system crossSystems then self.hydraJobs.binaryTarballCross.x86_64-linux.${system} else self.hydraJobs.binaryTarball.${system};
-                  in '' \
-                  --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${tarball}/*.tar.xz) \
-                  --replace '@tarballPath_${system}@' $(tarballPath ${tarball}/*.tar.xz) \
-                  ''
-                )
-                systems
-              } --replace '@nixVersion@' ${version}
-
-            echo "file installer $out/install" >> $out/nix-support/hydra-build-products
-          '';
-
-      testNixVersions = pkgs: client: daemon: with commonDeps { inherit pkgs; }; with pkgs.lib; pkgs.stdenv.mkDerivation {
-        NIX_DAEMON_PACKAGE = daemon;
-        NIX_CLIENT_PACKAGE = client;
-        name =
-          "nix-tests"
-          + optionalString
-            (versionAtLeast daemon.version "2.4pre20211005" &&
-             versionAtLeast client.version "2.4pre20211005")
-            "-${client.version}-against-${daemon.version}";
-        inherit version;
-
-        src = fileset.toSource {
-          root = ./.;
-          fileset = fileset.intersect baseFiles (fileset.unions [
-            configureFiles
-            topLevelBuildFiles
-            functionalTestFiles
-          ]);
         };
 
-        VERSION_SUFFIX = versionSuffix;
+      testNixVersions = pkgs: client: daemon:
+        pkgs.callPackage ./package.nix {
+          pname =
+            "nix-tests"
+            + lib.optionalString
+              (lib.versionAtLeast daemon.version "2.4pre20211005" &&
+               lib.versionAtLeast client.version "2.4pre20211005")
+              "-${client.version}-against-${daemon.version}";
 
-        nativeBuildInputs = nativeBuildDeps;
+          inherit fileset;
-        buildInputs = buildDeps ++ awsDeps ++ checkDeps;
-        propagatedBuildInputs = propagatedDeps;
 
-        enableParallelBuilding = true;
+          test-client = client;
+          test-daemon = daemon;
 
-        configureFlags =
+          doBuild = false;
-          testConfigureFlags # otherwise configure fails
+        };
-          ++ [ "--disable-build" ];
-        dontBuild = true;
-        doInstallCheck = true;
 
-        installPhase = ''
+      binaryTarball = nix: pkgs: pkgs.callPackage ./scripts/binary-tarball.nix {
-          mkdir -p $out
+        inherit nix;
-        '';
-
-        installCheckPhase = ''
-          mkdir -p src/nix-channel
-          make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES
-        '';
       };
 
-      binaryTarball = nix: pkgs:
-        let
-          inherit (pkgs) buildPackages;
-          inherit (pkgs) cacert;
-          installerClosureInfo = buildPackages.closureInfo { rootPaths = [ nix cacert ]; };
-        in
-
-        buildPackages.runCommand "nix-binary-tarball-${version}"
-          { #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck;
-            meta.description = "Distribution-independent Nix bootstrap binaries for ${pkgs.system}";
-          }
-          ''
-            cp ${installerClosureInfo}/registration $TMPDIR/reginfo
-            cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh
-            substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \
-              --subst-var-by nix ${nix} \
-              --subst-var-by cacert ${cacert}
-
-            substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \
-              --subst-var-by nix ${nix} \
-              --subst-var-by cacert ${cacert}
-            substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \
-              --subst-var-by nix ${nix} \
-              --subst-var-by cacert ${cacert}
-            substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \
-              --subst-var-by nix ${nix} \
-              --subst-var-by cacert ${cacert}
-
-            if type -p shellcheck; then
-              # SC1090: Don't worry about not being able to find
-              #         $nix/etc/profile.d/nix.sh
-              shellcheck --exclude SC1090 $TMPDIR/install
-              shellcheck $TMPDIR/create-darwin-volume.sh
-              shellcheck $TMPDIR/install-darwin-multi-user.sh
-              shellcheck $TMPDIR/install-systemd-multi-user.sh
-
-              # SC1091: Don't panic about not being able to source
-              #         /etc/profile
-              # SC2002: Ignore "useless cat" "error", when loading
-              #         .reginfo, as the cat is a much cleaner
-              #         implementation, even though it is "useless"
-              # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving
-              #         root's home directory
-              shellcheck --external-sources \
-                --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user
-            fi
-
-            chmod +x $TMPDIR/install
-            chmod +x $TMPDIR/create-darwin-volume.sh
-            chmod +x $TMPDIR/install-darwin-multi-user.sh
-            chmod +x $TMPDIR/install-systemd-multi-user.sh
-            chmod +x $TMPDIR/install-multi-user
-            dir=nix-${version}-${pkgs.system}
-            fn=$out/$dir.tar.xz
-            mkdir -p $out/nix-support
-            echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products
-            tar cvfJ $fn \
-              --owner=0 --group=0 --mode=u+rw,uga+r \
-              --mtime='1970-01-01' \
-              --absolute-names \
-              --hard-dereference \
-              --transform "s,$TMPDIR/install,$dir/install," \
-              --transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \
-              --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \
-              --transform "s,$NIX_STORE,$dir/store,S" \
-              $TMPDIR/install \
-              $TMPDIR/create-darwin-volume.sh \
-              $TMPDIR/install-darwin-multi-user.sh \
-              $TMPDIR/install-systemd-multi-user.sh \
-              $TMPDIR/install-multi-user \
-              $TMPDIR/reginfo \
-              $(cat ${installerClosureInfo}/store-paths)
-          '';
-
       overlayFor = getStdenv: final: prev:
-        let currentStdenv = getStdenv final; in
+        let
+          stdenv = getStdenv final;
+        in
         {
           nixStable = prev.nix;
 
-          # Forward from the previous stage as we don’t want it to pick the lowdown override
+          default-busybox-sandbox-shell = final.busybox.override {
-          nixUnstable = prev.nixUnstable;
+            useMusl = true;
+            enableStatic = true;
+            enableMinimal = true;
+            extraConfig = ''
+              CONFIG_FEATURE_FANCY_ECHO y
+              CONFIG_FEATURE_SH_MATH y
+              CONFIG_FEATURE_SH_MATH_64 y
 
-          nix =
+              CONFIG_ASH y
-          with final;
+              CONFIG_ASH_OPTIMIZE_FOR_SIZE y
-          with commonDeps {
-            inherit pkgs;
-            inherit (currentStdenv.hostPlatform) isStatic;
-          };
-          let
-            canRunInstalled = currentStdenv.buildPlatform.canExecute currentStdenv.hostPlatform;
-          in currentStdenv.mkDerivation (finalAttrs: {
-            name = "nix-${version}";
-            inherit version;
 
-            src = nixSrc;
+              CONFIG_ASH_ALIAS y
-            VERSION_SUFFIX = versionSuffix;
+              CONFIG_ASH_BASH_COMPAT y
-
+              CONFIG_ASH_CMDCMD y
-            outputs = [ "out" "dev" "doc" ]
+              CONFIG_ASH_ECHO y
-              ++ lib.optional (currentStdenv.hostPlatform != currentStdenv.buildPlatform) "check";
+              CONFIG_ASH_GETOPTS y
-
+              CONFIG_ASH_INTERNAL_GLOB y
-            nativeBuildInputs = nativeBuildDeps;
+              CONFIG_ASH_JOB_CONTROL y
-            buildInputs = buildDeps
+              CONFIG_ASH_PRINTF y
-              # There have been issues building these dependencies
+              CONFIG_ASH_TEST y
-              ++ lib.optionals (currentStdenv.hostPlatform == currentStdenv.buildPlatform) awsDeps
-              ++ lib.optionals finalAttrs.doCheck checkDeps;
-
-            propagatedBuildInputs = propagatedDeps;
-
-            disallowedReferences = [ boost ];
-
-            preConfigure = lib.optionalString (! currentStdenv.hostPlatform.isStatic)
-              ''
-                # Copy libboost_context so we don't get all of Boost in our closure.
-                # https://github.com/NixOS/nixpkgs/issues/45462
-                mkdir -p $out/lib
-                cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
-                rm -f $out/lib/*.a
-                ${lib.optionalString currentStdenv.hostPlatform.isLinux ''
-                  chmod u+w $out/lib/*.so.*
-                  patchelf --set-rpath $out/lib:${currentStdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
-                ''}
-                ${lib.optionalString currentStdenv.hostPlatform.isDarwin ''
-                  for LIB in $out/lib/*.dylib; do
-                    chmod u+w $LIB
-                    install_name_tool -id $LIB $LIB
-                    install_name_tool -delete_rpath ${boost}/lib/ $LIB || true
-                  done
-                  install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib
-                ''}
-              '';
-
-            configureFlags = configureFlags ++
-              [ "--sysconfdir=/etc" ] ++
-              lib.optional stdenv.hostPlatform.isStatic "--enable-embedded-sandbox-shell" ++
-              [ (lib.enableFeature finalAttrs.doCheck "tests") ] ++
-              lib.optionals finalAttrs.doCheck testConfigureFlags ++
-              lib.optional (!canRunInstalled) "--disable-doc-gen";
-
-            enableParallelBuilding = true;
-
-            makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1";
-
-            doCheck = true;
-
-            installFlags = "sysconfdir=$(out)/etc";
-
-            postInstall = ''
-              mkdir -p $doc/nix-support
-              echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
-              ${lib.optionalString currentStdenv.hostPlatform.isStatic ''
-              mkdir -p $out/nix-support
-              echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products
-              ''}
-              ${lib.optionalString currentStdenv.isDarwin ''
-              install_name_tool \
-                -change ${boost}/lib/libboost_context.dylib \
-                $out/lib/libboost_context.dylib \
-                $out/lib/libnixutil.dylib
-              ''}
             '';
+          };
 
-            doInstallCheck = finalAttrs.doCheck;
+          libgit2-nix = final.libgit2.overrideAttrs (attrs: {
-            installCheckFlags = "sysconfdir=$(out)/etc";
+            src = libgit2;
-            installCheckTarget = "installcheck"; # work around buggy detection in stdenv
+            version = libgit2.lastModifiedDate;
-
+            cmakeFlags = attrs.cmakeFlags or []
-            separateDebugInfo = !currentStdenv.hostPlatform.isStatic;
+              ++ [ "-DUSE_SSH=exec" ];
-
-            strictDeps = true;
-
-            hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-
-            passthru.perl-bindings = final.callPackage ./perl {
-              inherit fileset;
-              stdenv = currentStdenv;
-            };
-
-            meta.platforms = lib.platforms.unix;
-            meta.mainProgram = "nix";
           });
 
-          lowdown-nix = with final; currentStdenv.mkDerivation rec {
+          boehmgc-nix = (final.boehmgc.override {
-            name = "lowdown-0.9.0";
+            enableLargeConfig = true;
+          }).overrideAttrs(o: {
+            patches = (o.patches or []) ++ [
+              ./dep-patches/boehmgc-coroutine-sp-fallback.diff
 
-            src = lowdown-src;
+              # https://github.com/ivmai/bdwgc/pull/586
+              ./dep-patches/boehmgc-traceable_allocator-public.diff
+            ];
+          });
 
-            outputs = [ "out" "bin" "dev" ];
+          changelog-d-nix = final.buildPackages.callPackage ./misc/changelog-d.nix { };
 
-            nativeBuildInputs = [ buildPackages.which ];
+          nix =
+            let
+              officialRelease = false;
+              versionSuffix =
+                if officialRelease
+                then ""
+                else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}";
 
-            configurePhase = ''
+            in final.callPackage ./package.nix {
-                ${if (currentStdenv.isDarwin && currentStdenv.isAarch64) then "echo \"HAVE_SANDBOX_INIT=false\" > configure.local" else ""}
+              inherit
-                ./configure \
+                fileset
-                  PREFIX=${placeholder "dev"} \
+                stdenv
-                  BINDIR=${placeholder "bin"}/bin
+                versionSuffix
-            '';
+                ;
+              officialRelease = false;
+              boehmgc = final.boehmgc-nix;
+              libgit2 = final.libgit2-nix;
+              busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell;
+            } // {
+              # this is a proper separate downstream package, but put
+              # here also for back compat reasons.
+              perl-bindings = final.nix-perl-bindings;
+            };
+
+          nix-perl-bindings = final.callPackage ./perl {
+            inherit fileset stdenv;
           };
+
         };
 
     in {
@@ -547,14 +219,25 @@
         buildCross = forAllCrossSystems (crossSystem:
           lib.genAttrs ["x86_64-linux"] (system: self.packages.${system}."nix-${crossSystem}"));
 
-        buildNoGc = forAllSystems (system: self.packages.${system}.nix.overrideAttrs (a: { configureFlags = (a.configureFlags or []) ++ ["--enable-gc=no"];}));
+        buildNoGc = forAllSystems (system:
+          self.packages.${system}.nix.override { enableGC = false; }
+        );
 
         buildNoTests = forAllSystems (system:
-          self.packages.${system}.nix.overrideAttrs (a: {
+          self.packages.${system}.nix.override {
-            doCheck =
+            doCheck = false;
-              assert ! a?dontCheck;
+            doInstallCheck = false;
-              false;
+            installUnitTests = false;
-          })
+          }
+        );
+
+        # Toggles some settings for better coverage. Windows needs these
+        # library combinations, and Debian build Nix with GNU readline too.
+        buildReadlineNoMarkdown = forAllSystems (system:
+          self.packages.${system}.nix.override {
+            enableMarkdown = false;
+            readlineFlavor = "readline";
+          }
         );
 
         # Perl bindings for various platforms.
@@ -575,67 +258,41 @@
         # to https://nixos.org/nix/install. It downloads the binary
         # tarball for the user's system and calls the second half of the
         # installation script.
-        installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" "armv6l-linux" "armv7l-linux" ];
+        installerScript = installScriptFor [
-        installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" "armv6l-linux" "armv7l-linux"];
+          # Native
+          self.hydraJobs.binaryTarball."x86_64-linux"
+          self.hydraJobs.binaryTarball."i686-linux"
+          self.hydraJobs.binaryTarball."aarch64-linux"
+          self.hydraJobs.binaryTarball."x86_64-darwin"
+          self.hydraJobs.binaryTarball."aarch64-darwin"
+          # Cross
+          self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf"
+          self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf"
+        ];
+        installerScriptForGHA = installScriptFor [
+          # Native
+          self.hydraJobs.binaryTarball."x86_64-linux"
+          self.hydraJobs.binaryTarball."x86_64-darwin"
+          # Cross
+          self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf"
+          self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf"
+        ];
 
         # docker image with Nix inside
         dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage);
 
         # Line coverage analysis.
-        coverage =
+        coverage = nixpkgsFor.x86_64-linux.native.nix.override {
-          with nixpkgsFor.x86_64-linux.native;
+          pname = "nix-coverage";
-          with commonDeps { inherit pkgs; };
+          withCoverageChecks = true;
-
+        };
-          releaseTools.coverageAnalysis {
-            name = "nix-coverage-${version}";
-
-            src = nixSrc;
-
-            configureFlags = testConfigureFlags;
-
-            enableParallelBuilding = true;
-
-            nativeBuildInputs = nativeBuildDeps;
-            buildInputs = buildDeps ++ propagatedDeps ++ awsDeps ++ checkDeps;
-
-            dontInstall = false;
-
-            doInstallCheck = true;
-            installCheckTarget = "installcheck"; # work around buggy detection in stdenv
-
-            lcovFilter = [ "*/boost/*" "*-tab.*" ];
-
-            hardeningDisable = ["fortify"];
-
-            NIX_CFLAGS_COMPILE = "-DCOVERAGE=1";
-          };
 
         # API docs for Nix's unstable internal C++ interfaces.
-        internal-api-docs =
+        internal-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ./package.nix {
-          with nixpkgsFor.x86_64-linux.native;
+          inherit fileset;
-          with commonDeps { inherit pkgs; };
+          doBuild = false;
-
+          enableInternalAPIDocs = true;
-          stdenv.mkDerivation {
+        };
-            pname = "nix-internal-api-docs";
-            inherit version;
-
-            src = nixSrc;
-
-            configureFlags = testConfigureFlags ++ internalApiDocsConfigureFlags;
-
-            nativeBuildInputs = nativeBuildDeps;
-            buildInputs = buildDeps ++ propagatedDeps
-              ++ awsDeps ++ checkDeps ++ internalApiDocsDeps;
-
-            dontBuild = true;
-
-            installTargets = [ "internal-api-html" ];
-
-            postInstall = ''
-              mkdir -p $out/nix-support
-              echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> $out/nix-support/hydra-build-products
-            '';
-          };
 
         # System tests.
         tests = import ./tests/nixos { inherit lib nixpkgs nixpkgsFor; } // {
@@ -643,7 +300,9 @@
           # Make sure that nix-env still produces the exact same result
           # on a particular version of Nixpkgs.
           evalNixpkgs =
-            with nixpkgsFor.x86_64-linux.native;
+            let
+              inherit (nixpkgsFor.x86_64-linux.native) runCommand nix;
+            in
             runCommand "eval-nixos" { buildInputs = [ nix ]; }
               ''
                 type -p nix-env
@@ -696,14 +355,14 @@
         rl-next =
           let pkgs = nixpkgsFor.${system}.native;
           in pkgs.buildPackages.runCommand "test-rl-next-release-notes" { } ''
-          LANG=C.UTF-8 ${(commonDeps { inherit pkgs; }).changelog-d}/bin/changelog-d ${./doc/manual/rl-next} >$out
+          LANG=C.UTF-8 ${pkgs.changelog-d-nix}/bin/changelog-d ${./doc/manual/rl-next} >$out
         '';
       } // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) {
         dockerImage = self.hydraJobs.dockerImage.${system};
       });
 
       packages = forAllSystems (system: rec {
-        inherit (nixpkgsFor.${system}.native) nix;
+        inherit (nixpkgsFor.${system}.native) nix changelog-d-nix;
         default = nix;
       } // (lib.optionalAttrs (builtins.elem system linux64BitSystems) {
         nix-static = nixpkgsFor.${system}.static.nix;
@@ -735,47 +394,20 @@
           stdenvs)));
 
       devShells = let
-        makeShell = pkgs: stdenv:
+        makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; forDevShell = true; }).overrideAttrs (attrs: {
-          let
+          installFlags = "sysconfdir=$(out)/etc";
-            canRunInstalled = stdenv.buildPlatform.canExecute stdenv.hostPlatform;
+          shellHook = ''
-          in
+            PATH=$prefix/bin:$PATH
-          with commonDeps { inherit pkgs; };
+            unset PYTHONPATH
-          stdenv.mkDerivation {
+            export MANPATH=$out/share/man:$MANPATH
-            name = "nix";
 
-            outputs = [ "out" "dev" "doc" ]
+            # Make bash completion work.
-              ++ lib.optional (stdenv.hostPlatform != stdenv.buildPlatform) "check";
+            XDG_DATA_DIRS+=:$out/share
-
+          '';
-            nativeBuildInputs = nativeBuildDeps
+          nativeBuildInputs = attrs.nativeBuildInputs or []
-              ++ lib.optional stdenv.cc.isClang pkgs.buildPackages.bear
+            ++ lib.optional stdenv.cc.isClang pkgs.buildPackages.bear
-              ++ lib.optional
+            ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) pkgs.buildPackages.clang-tools;
-                (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform)
+        });
-                pkgs.buildPackages.clang-tools
-              # We want changelog-d in the shell even if the current build doesn't need it
-              ++ lib.optional (officialRelease || ! buildUnreleasedNotes) changelog-d
-              ;
-
-            buildInputs = buildDeps ++ propagatedDeps
-              ++ awsDeps ++ checkDeps ++ internalApiDocsDeps;
-
-            configureFlags = configureFlags
-              ++ testConfigureFlags ++ internalApiDocsConfigureFlags
-              ++ lib.optional (!canRunInstalled) "--disable-doc-gen";
-
-            enableParallelBuilding = true;
-
-            installFlags = "sysconfdir=$(out)/etc";
-
-            shellHook =
-              ''
-                PATH=$prefix/bin:$PATH
-                unset PYTHONPATH
-                export MANPATH=$out/share/man:$MANPATH
-
-                # Make bash completion work.
-                XDG_DATA_DIRS+=:$out/share
-              '';
-          };
         in
         forAllSystems (system:
           let
@@ -786,7 +418,7 @@
           in
             (makeShells "native" nixpkgsFor.${system}.native) //
             (makeShells "static" nixpkgsFor.${system}.static) //
-            (forAllCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv)) //
+            (lib.genAttrs shellCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv)) //
             {
               default = self.devShells.${system}.native-stdenvPackages;
             }

maintainers/README.md

@@ -43,7 +43,11 @@ The team meets twice a week:
 - Discussion meeting: [Fridays 13:00-14:00 CET](https://calendar.google.com/calendar/event?eid=MHNtOGVuNWtrZXNpZHR2bW1sM3QyN2ZjaGNfMjAyMjExMjVUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn)
 
   1. Triage issues and pull requests from the [No Status](#no-status) column (30 min)
-  2. Discuss issues and pull requests from the [To discuss](#to-discuss) column (30 min)
+  2. Discuss issues and pull requests from the [To discuss](#to-discuss) column (30 min).
+     Once a month, each team member checks the [Assigned](#assigned) column for prs/issues assigned to them, to either
+       - unblock it by providing input
+       - mark it as draft if it is blocked on the contributor
+       - escalate it back to the team by moving it to To discuss, and leaving a comment as to why the issue needs to be discussed again.
 
 - Work meeting: [Mondays 13:00-15:00 CET](https://calendar.google.com/calendar/event?eid=NTM1MG1wNGJnOGpmOTZhYms3bTB1bnY5cWxfMjAyMjExMjFUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn)
 

maintainers/release-notes

@@ -1,7 +1,5 @@
-#!/usr/bin/env nix-shell
+#!/usr/bin/env nix
-#!nix-shell -i bash ../shell.nix -I nixpkgs=channel:nixos-unstable-small
+#!nix shell .#changelog-d-nix --command bash
-#                                   ^^^^^^^
-#                Only used for bash. shell.nix goes to the flake.
 
 # --- CONFIGURATION ---
 

maintainers/release-process.md

@@ -27,8 +27,9 @@ release:
 * Compile the release notes by running
 
   ```console
+  $ export VERSION=X.YY
   $ git checkout -b release-notes
-  $ VERSION=X.YY ./maintainers/release-notes
+  $ ./maintainers/release-notes
   ```
 
   where `X.YY` is *without* the patch level, e.g. `2.12` rather than ~~`2.12.0`~~.

maintainers/upload-release.pl

@@ -154,8 +154,8 @@ downloadFile("binaryTarball.x86_64-linux", "1");
 downloadFile("binaryTarball.aarch64-linux", "1");
 downloadFile("binaryTarball.x86_64-darwin", "1");
 downloadFile("binaryTarball.aarch64-darwin", "1");
-downloadFile("binaryTarballCross.x86_64-linux.armv6l-linux", "1");
+downloadFile("binaryTarballCross.x86_64-linux.armv6l-unknown-linux-gnueabihf", "1");
-downloadFile("binaryTarballCross.x86_64-linux.armv7l-linux", "1");
+downloadFile("binaryTarballCross.x86_64-linux.armv7l-unknown-linux-gnueabihf", "1");
 downloadFile("installerScript", "1");
 
 # Upload docker images to dockerhub.

mk/disable-tests.mk

@@ -1,12 +0,0 @@
-# This file is only active for `./configure --disable-tests`.
-# Running `make check` or `make installcheck` would indicate a mistake in the
-# caller.
-
-installcheck:
-	@echo "Tests are disabled. Configure without '--disable-tests', or avoid calling 'make installcheck'."
-	@exit 1
-
-# This currently has little effect.
-check:
-	@echo "Tests are disabled. Configure without '--disable-tests', or avoid calling 'make check'."
-	@exit 1

mk/lib.mk

@@ -12,24 +12,7 @@ man-pages :=
 install-tests :=
 install-tests-groups :=
 
-ifdef HOST_OS
+include mk/platform.mk
-  HOST_KERNEL = $(firstword $(subst -, ,$(HOST_OS)))
-  ifeq ($(HOST_KERNEL), cygwin)
-    HOST_CYGWIN = 1
-  endif
-  ifeq ($(patsubst darwin%,,$(HOST_KERNEL)),)
-    HOST_DARWIN = 1
-  endif
-  ifeq ($(patsubst freebsd%,,$(HOST_KERNEL)),)
-    HOST_FREEBSD = 1
-  endif
-  ifeq ($(HOST_KERNEL), linux)
-    HOST_LINUX = 1
-  endif
-  ifeq ($(patsubst solaris%,,$(HOST_KERNEL)),)
-    HOST_SOLARIS = 1
-  endif
-endif
 
 # Hack to define a literal space.
 space :=

mk/libraries.mk

@@ -3,13 +3,19 @@ libs-list :=
 ifdef HOST_DARWIN
   SO_EXT = dylib
 else
-  ifdef HOST_CYGWIN
+  ifdef HOST_WINDOWS
     SO_EXT = dll
   else
     SO_EXT = so
   endif
 endif
 
+ifdef HOST_UNIX
+  THREAD_LDFLAGS = -pthread
+else
+  THREAD_LDFLAGS =
+endif
+
 # Build a library with symbolic name $(1).  The library is defined by
 # various variables prefixed by ‘$(1)_’:
 #
@@ -59,7 +65,7 @@ define build-library
   $(1)_OBJS := $$(addprefix $(buildprefix), $$(addsuffix .o, $$(basename $$(_srcs))))
   _libs := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_PATH))
 
-  ifdef HOST_CYGWIN
+  ifdef HOST_WINDOWS
     $(1)_INSTALL_DIR ?= $$(bindir)
   else
     $(1)_INSTALL_DIR ?= $$(libdir)
@@ -79,7 +85,7 @@ define build-library
       endif
     else
       ifndef HOST_DARWIN
-        ifndef HOST_CYGWIN
+        ifndef HOST_WINDOWS
           $(1)_LDFLAGS += -Wl,-z,defs
         endif
       endif

mk/platform.mk

@@ -0,0 +1,32 @@
+ifdef HOST_OS
+  HOST_KERNEL = $(firstword $(subst -, ,$(HOST_OS)))
+  ifeq ($(patsubst mingw%,,$(HOST_KERNEL)),)
+    HOST_MINGW = 1
+    HOST_WINDOWS = 1
+  endif
+  ifeq ($(HOST_KERNEL), cygwin)
+    HOST_CYGWIN = 1
+    HOST_WINDOWS = 1
+    HOST_UNIX = 1
+  endif
+  ifeq ($(patsubst darwin%,,$(HOST_KERNEL)),)
+    HOST_DARWIN = 1
+    HOST_UNIX = 1
+  endif
+  ifeq ($(patsubst freebsd%,,$(HOST_KERNEL)),)
+    HOST_FREEBSD = 1
+    HOST_UNIX = 1
+  endif
+  ifeq ($(patsubst netbsd%,,$(HOST_KERNEL)),)
+    HOST_NETBSD = 1
+    HOST_UNIX = 1
+  endif
+  ifeq ($(HOST_KERNEL), linux)
+    HOST_LINUX = 1
+    HOST_UNIX = 1
+  endif
+  ifeq ($(patsubst solaris%,,$(HOST_KERNEL)),)
+    HOST_SOLARIS = 1
+    HOST_UNIX = 1
+  endif
+endif

mk/programs.mk

@@ -1,5 +1,11 @@
 programs-list :=
 
+ifdef HOST_WINDOWS
+  EXE_EXT = .exe
+else
+  EXE_EXT =
+endif
+
 # Build a program with symbolic name $(1).  The program is defined by
 # various variables prefixed by ‘$(1)_’:
 #
@@ -31,7 +37,7 @@ define build-program
   _srcs := $$(sort $$(foreach src, $$($(1)_SOURCES), $$(src)))
   $(1)_OBJS := $$(addprefix $(buildprefix), $$(addsuffix .o, $$(basename $$(_srcs))))
   _libs := $$(foreach lib, $$($(1)_LIBS), $$(foreach lib2, $$($$(lib)_LIB_CLOSURE), $$($$(lib2)_PATH)))
-  $(1)_PATH := $$(_d)/$$($(1)_NAME)
+  $(1)_PATH := $$(_d)/$$($(1)_NAME)$(EXE_EXT)
 
   $$(eval $$(call create-dir, $$(_d)))
 
@@ -42,7 +48,7 @@ define build-program
 
   ifdef $(1)_INSTALL_DIR
 
-    $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$$($(1)_NAME)
+    $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$$($(1)_NAME)$(EXE_EXT)
 
     $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR)))
 

package.nix

@@ -0,0 +1,390 @@
+{ lib
+, stdenv
+, releaseTools
+, autoconf-archive
+, autoreconfHook
+, aws-sdk-cpp
+, boehmgc
+, nlohmann_json
+, bison
+, boost
+, brotli
+, bzip2
+, curl
+, editline
+, readline
+, fileset
+, flex
+, git
+, gtest
+, jq
+, doxygen
+, libarchive
+, libcpuid
+, libgit2
+, libseccomp
+, libsodium
+, lowdown
+, mdbook
+, mdbook-linkcheck
+, mercurial
+, openssh
+, openssl
+, pkg-config
+, rapidcheck
+, sqlite
+, util-linux
+, xz
+
+, busybox-sandbox-shell ? null
+
+# Configuration Options
+#:
+# This probably seems like too many degrees of freedom, but it
+# faithfully reflects how the underlying configure + make build system
+# work. The top-level flake.nix will choose useful combinations of these
+# options to CI.
+
+, pname ? "nix"
+
+, versionSuffix ? ""
+, officialRelease ? false
+
+# Whether to build Nix. Useful to skip for tasks like (a) just
+# generating API docs or (b) testing existing pre-built versions of Nix
+, doBuild ? true
+
+# Run the unit tests as part of the build. See `installUnitTests` for an
+# alternative to this.
+, doCheck ? __forDefaults.canRunInstalled
+
+# Run the functional tests as part of the build.
+, doInstallCheck ? test-client != null || __forDefaults.canRunInstalled
+
+# Check test coverage of Nix. Probably want to use with with at least
+# one of `doCHeck` or `doInstallCheck` enabled.
+, withCoverageChecks ? false
+
+# Whether to build the regular manual
+, enableManual ? __forDefaults.canRunInstalled
+
+# Whether to use garbage collection for the Nix language evaluator.
+#
+# If it is disabled, we just leak memory, but this is not as bad as it
+# sounds so long as evaluation just takes places within short-lived
+# processes. (When the process exits, the memory is reclaimed; it is
+# only leaked *within* the process.)
+, enableGC ? true
+
+# Whether to enable Markdown rendering in the Nix binary.
+, enableMarkdown ? !stdenv.hostPlatform.isWindows
+
+# Which interactive line editor library to use for Nix's repl.
+#
+# Currently supported choices are:
+#
+# - editline (default)
+# - readline
+, readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline"
+
+# Whether to build the internal API docs, can be done separately from
+# everything else.
+, enableInternalAPIDocs ? false
+
+# Whether to install unit tests. This is useful when cross compiling
+# since we cannot run them natively during the build, but can do so
+# later.
+, installUnitTests ? doBuild && !__forDefaults.canExecuteHost
+
+# For running the functional tests against a pre-built Nix. Probably
+# want to use in conjunction with `doBuild = false;`.
+, test-daemon ? null
+, test-client ? null
+
+# Avoid setting things that would interfere with a functioning devShell
+, forDevShell ? false
+
+# Not a real argument, just the only way to approximate let-binding some
+# stuff for argument defaults.
+, __forDefaults ? {
+    canExecuteHost = stdenv.buildPlatform.canExecute stdenv.hostPlatform;
+    canRunInstalled = doBuild && __forDefaults.canExecuteHost;
+  }
+}:
+
+let
+  version = lib.fileContents ./.version + versionSuffix;
+
+  # selected attributes with defaults, will be used to define some
+  # things which should instead be gotten via `finalAttrs` in order to
+  # work with overriding.
+  attrs = {
+    inherit doBuild doCheck doInstallCheck;
+  };
+
+  mkDerivation =
+    if withCoverageChecks
+    then
+      # TODO support `finalAttrs` args function in
+      # `releaseTools.coverageAnalysis`.
+      argsFun:
+         releaseTools.coverageAnalysis (let args = argsFun args; in args)
+    else stdenv.mkDerivation;
+in
+
+mkDerivation (finalAttrs: let
+
+  inherit (finalAttrs)
+    doCheck
+    doInstallCheck
+    ;
+
+  doBuild = !finalAttrs.dontBuild;
+
+  # Either running the unit tests during the build, or installing them
+  # to be run later, requiresthe unit tests to be built.
+  buildUnitTests = doCheck || installUnitTests;
+
+in {
+  inherit pname version;
+
+  src =
+    let
+      baseFiles = fileset.fileFilter (f: f.name != ".gitignore") ./.;
+    in
+      fileset.toSource {
+        root = ./.;
+        fileset = fileset.intersect baseFiles (fileset.unions ([
+          # For configure
+          ./.version
+          ./configure.ac
+          ./m4
+          # TODO: do we really need README.md? It doesn't seem used in the build.
+          ./README.md
+          # For make, regardless of what we are building
+          ./local.mk
+          ./Makefile
+          ./Makefile.config.in
+          ./mk
+          (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts)
+        ] ++ lib.optionals doBuild [
+          ./doc
+          ./misc
+          ./precompiled-headers.h
+          ./src
+          ./COPYING
+          ./scripts/local.mk
+        ] ++ lib.optionals buildUnitTests [
+          ./doc/manual
+        ] ++ lib.optionals enableInternalAPIDocs [
+          ./doc/internal-api
+          # Source might not be compiled, but still must be available
+          # for Doxygen to gather comments.
+          ./src
+          ./tests/unit
+        ] ++ lib.optionals buildUnitTests [
+          ./tests/unit
+        ] ++ lib.optionals doInstallCheck [
+          ./tests/functional
+        ]));
+      };
+
+  VERSION_SUFFIX = versionSuffix;
+
+  outputs = [ "out" ]
+    ++ lib.optional doBuild "dev"
+    # If we are doing just build or just docs, the one thing will use
+    # "out". We only need additional outputs if we are doing both.
+    ++ lib.optional (doBuild && (enableManual || enableInternalAPIDocs)) "doc"
+    ++ lib.optional installUnitTests "check";
+
+  nativeBuildInputs = [
+    autoconf-archive
+    autoreconfHook
+    pkg-config
+  ] ++ lib.optionals doBuild [
+    bison
+    flex
+  ] ++ lib.optionals enableManual [
+    (lib.getBin lowdown)
+    mdbook
+    mdbook-linkcheck
+  ] ++ lib.optionals (doInstallCheck || enableManual) [
+    jq # Also for custom mdBook preprocessor.
+  ] ++ lib.optional stdenv.hostPlatform.isLinux util-linux
+    ++ lib.optional enableInternalAPIDocs doxygen
+  ;
+
+  buildInputs = lib.optionals doBuild [
+    boost
+    brotli
+    bzip2
+    curl
+    libarchive
+    libgit2
+    libsodium
+    openssl
+    sqlite
+    xz
+    ({ inherit readline editline; }.${readlineFlavor})
+  ] ++ lib.optionals enableMarkdown [
+    lowdown
+  ] ++ lib.optionals buildUnitTests [
+    gtest
+    rapidcheck
+  ] ++ lib.optional stdenv.isLinux libseccomp
+    ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid
+    # There have been issues building these dependencies
+    ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin))
+      (aws-sdk-cpp.override {
+        apis = ["s3" "transfer"];
+        customMemoryManagement = false;
+      })
+  ;
+
+  propagatedBuildInputs = [
+    nlohmann_json
+  ] ++ lib.optional enableGC boehmgc;
+
+  dontBuild = !attrs.doBuild;
+  doCheck = attrs.doCheck;
+
+  nativeCheckInputs = [
+    git
+    mercurial
+    openssh
+  ];
+
+  disallowedReferences = [ boost ];
+
+  preConfigure = lib.optionalString (doBuild && ! stdenv.hostPlatform.isStatic) (
+    ''
+      # Copy libboost_context so we don't get all of Boost in our closure.
+      # https://github.com/NixOS/nixpkgs/issues/45462
+      mkdir -p $out/lib
+      cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
+      rm -f $out/lib/*.a
+    '' + lib.optionalString stdenv.hostPlatform.isLinux ''
+      chmod u+w $out/lib/*.so.*
+      patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
+    '' + lib.optionalString stdenv.hostPlatform.isDarwin ''
+      for LIB in $out/lib/*.dylib; do
+        chmod u+w $LIB
+        install_name_tool -id $LIB $LIB
+        install_name_tool -delete_rpath ${boost}/lib/ $LIB || true
+      done
+      install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib
+    ''
+  );
+
+  configureFlags = [
+    (lib.enableFeature doBuild "build")
+    (lib.enableFeature buildUnitTests "unit-tests")
+    (lib.enableFeature doInstallCheck "functional-tests")
+    (lib.enableFeature enableInternalAPIDocs "internal-api-docs")
+    (lib.enableFeature enableManual "doc-gen")
+    (lib.enableFeature enableGC "gc")
+    (lib.enableFeature enableMarkdown "markdown")
+    (lib.enableFeature installUnitTests "install-unit-tests")
+    (lib.withFeatureAs true "readline-flavor" readlineFlavor)
+  ] ++ lib.optionals (!forDevShell) [
+    "--sysconfdir=/etc"
+  ] ++ lib.optionals installUnitTests [
+    "--with-check-bin-dir=${builtins.placeholder "check"}/bin"
+    "--with-check-lib-dir=${builtins.placeholder "check"}/lib"
+  ] ++ lib.optionals (doBuild) [
+    "--with-boost=${boost}/lib"
+  ] ++ lib.optionals (doBuild && stdenv.isLinux) [
+    "--with-sandbox-shell=${busybox-sandbox-shell}/bin/busybox"
+  ] ++ lib.optional (doBuild && stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux"))
+       "LDFLAGS=-fuse-ld=gold"
+    ++ lib.optional (doBuild && stdenv.hostPlatform.isStatic) "--enable-embedded-sandbox-shell"
+    ;
+
+  enableParallelBuilding = true;
+
+  makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1";
+
+  installTargets = lib.optional doBuild "install"
+    ++ lib.optional enableInternalAPIDocs "internal-api-html";
+
+  installFlags = "sysconfdir=$(out)/etc";
+
+  # In this case we are probably just running tests, and so there isn't
+  # anything to install, we just make an empty directory to signify tests
+  # succeeded.
+  installPhase = if finalAttrs.installTargets != [] then null else ''
+    mkdir -p $out
+  '';
+
+  postInstall = lib.optionalString doBuild (
+    lib.optionalString stdenv.hostPlatform.isStatic ''
+      mkdir -p $out/nix-support
+      echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products
+    '' + lib.optionalString stdenv.isDarwin ''
+      install_name_tool \
+      -change ${boost}/lib/libboost_context.dylib \
+      $out/lib/libboost_context.dylib \
+      $out/lib/libnixutil.dylib
+    ''
+  ) + lib.optionalString enableManual ''
+    mkdir -p ''${!outputDoc}/nix-support
+    echo "doc manual ''${!outputDoc}/share/doc/nix/manual" >> ''${!outputDoc}/nix-support/hydra-build-products
+  '' + lib.optionalString enableInternalAPIDocs ''
+    mkdir -p ''${!outputDoc}/nix-support
+    echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products
+  '';
+
+  doInstallCheck = attrs.doInstallCheck;
+
+  installCheckFlags = "sysconfdir=$(out)/etc";
+  # Work around buggy detection in stdenv.
+  installCheckTarget = "installcheck";
+
+  # Work around weird bug where it doesn't think there is a Makefile.
+  installCheckPhase = if (!doBuild && doInstallCheck) then ''
+    mkdir -p src/nix-channel
+    make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES
+  '' else null;
+
+  # Needed for tests if we are not doing a build, but testing existing
+  # built Nix.
+  preInstallCheck = lib.optionalString (! doBuild) ''
+    mkdir -p src/nix-channel
+  '';
+
+  separateDebugInfo = !stdenv.hostPlatform.isStatic;
+
+  # TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated
+  # to work with `strictDeps`.
+  strictDeps = !withCoverageChecks;
+
+  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
+
+  meta = {
+    platforms = lib.platforms.unix ++ lib.platforms.windows;
+    mainProgram = "nix";
+    broken = !(lib.all (a: a) [
+      # We cannot run or install unit tests if we don't build them or
+      # Nix proper (which they depend on).
+      (installUnitTests -> doBuild)
+      (doCheck -> doBuild)
+      # The build process for the manual currently requires extracting
+      # data from the Nix executable we are trying to document.
+      (enableManual -> doBuild)
+    ]);
+  };
+
+} // lib.optionalAttrs withCoverageChecks {
+  lcovFilter = [ "*/boost/*" "*-tab.*" ];
+
+  hardeningDisable = ["fortify"];
+
+  NIX_CFLAGS_COMPILE = "-DCOVERAGE=1";
+
+  dontInstall = false;
+} // lib.optionalAttrs (test-daemon != null) {
+  NIX_DAEMON_PACKAGE = test-daemon;
+} // lib.optionalAttrs (test-client != null) {
+  NIX_CLIENT_PACKAGE = test-client;
+})

perl/lib/Nix/Store.xs

@@ -12,7 +12,7 @@
 #include "realisation.hh"
 #include "globals.hh"
 #include "store-api.hh"
-#include "crypto.hh"
+#include "posix-source-accessor.hh"
 
 #include <sodium.h>
 #include <nlohmann/json.hpp>
@@ -205,7 +205,10 @@ void importPaths(int fd, int dontCheckSigs)
 SV * hashPath(char * algo, int base32, char * path)
     PPCODE:
         try {
-            Hash h = hashPath(parseHashAlgo(algo), path).first;
+            PosixSourceAccessor accessor;
+            Hash h = hashPath(
+                accessor, CanonPath::fromCwd(path),
+                FileIngestionMethod::Recursive, parseHashAlgo(algo)).first;
             auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
@@ -281,7 +284,11 @@ SV * addToStore(char * srcPath, int recursive, char * algo)
     PPCODE:
         try {
             auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
-            auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, method, parseHashAlgo(algo));
+            PosixSourceAccessor accessor;
+            auto path = store()->addToStore(
+                std::string(baseNameOf(srcPath)),
+                accessor, CanonPath::fromCwd(srcPath),
+                method, parseHashAlgo(algo));
             XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());

scripts/binary-tarball.nix

@@ -0,0 +1,84 @@
+{ runCommand
+, system
+, buildPackages
+, cacert
+, nix
+}:
+
+let
+
+  installerClosureInfo = buildPackages.closureInfo {
+    rootPaths = [ nix cacert ];
+  };
+
+  inherit (nix) version;
+
+  env = {
+    #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck;
+    meta.description = "Distribution-independent Nix bootstrap binaries for ${system}";
+  };
+
+in
+
+runCommand "nix-binary-tarball-${version}" env ''
+  cp ${installerClosureInfo}/registration $TMPDIR/reginfo
+  cp ${./create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh
+  substitute ${./install-nix-from-closure.sh} $TMPDIR/install \
+    --subst-var-by nix ${nix} \
+    --subst-var-by cacert ${cacert}
+
+  substitute ${./install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \
+    --subst-var-by nix ${nix} \
+    --subst-var-by cacert ${cacert}
+  substitute ${./install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \
+    --subst-var-by nix ${nix} \
+    --subst-var-by cacert ${cacert}
+  substitute ${./install-multi-user.sh} $TMPDIR/install-multi-user \
+    --subst-var-by nix ${nix} \
+    --subst-var-by cacert ${cacert}
+
+  if type -p shellcheck; then
+    # SC1090: Don't worry about not being able to find
+    #         $nix/etc/profile.d/nix.sh
+    shellcheck --exclude SC1090 $TMPDIR/install
+    shellcheck $TMPDIR/create-darwin-volume.sh
+    shellcheck $TMPDIR/install-darwin-multi-user.sh
+    shellcheck $TMPDIR/install-systemd-multi-user.sh
+
+    # SC1091: Don't panic about not being able to source
+    #         /etc/profile
+    # SC2002: Ignore "useless cat" "error", when loading
+    #         .reginfo, as the cat is a much cleaner
+    #         implementation, even though it is "useless"
+    # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving
+    #         root's home directory
+    shellcheck --external-sources \
+      --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user
+  fi
+
+  chmod +x $TMPDIR/install
+  chmod +x $TMPDIR/create-darwin-volume.sh
+  chmod +x $TMPDIR/install-darwin-multi-user.sh
+  chmod +x $TMPDIR/install-systemd-multi-user.sh
+  chmod +x $TMPDIR/install-multi-user
+  dir=nix-${version}-${system}
+  fn=$out/$dir.tar.xz
+  mkdir -p $out/nix-support
+  echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products
+  tar cvfJ $fn \
+    --owner=0 --group=0 --mode=u+rw,uga+r \
+    --mtime='1970-01-01' \
+    --absolute-names \
+    --hard-dereference \
+    --transform "s,$TMPDIR/install,$dir/install," \
+    --transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \
+    --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \
+    --transform "s,$NIX_STORE,$dir/store,S" \
+    $TMPDIR/install \
+    $TMPDIR/create-darwin-volume.sh \
+    $TMPDIR/install-darwin-multi-user.sh \
+    $TMPDIR/install-systemd-multi-user.sh \
+    $TMPDIR/install-multi-user \
+    $TMPDIR/reginfo \
+    $(cat ${installerClosureInfo}/store-paths)
+''

scripts/install-darwin-multi-user.sh

@@ -3,11 +3,13 @@
 set -eu
 set -o pipefail
 
+# System specific settings
+export NIX_FIRST_BUILD_UID="${NIX_FIRST_BUILD_UID:-301}"
+export NIX_BUILD_USER_NAME_TEMPLATE="_nixbld%d"
+
 readonly NIX_DAEMON_DEST=/Library/LaunchDaemons/org.nixos.nix-daemon.plist
 # create by default; set 0 to DIY, use a symlink, etc.
 readonly NIX_VOLUME_CREATE=${NIX_VOLUME_CREATE:-1} # now default
-NIX_FIRST_BUILD_UID="301"
-NIX_BUILD_USER_NAME_TEMPLATE="_nixbld%d"
 
 # caution: may update times on / if not run as normal non-root user
 read_only_root() {

scripts/install-multi-user.sh

@@ -25,9 +25,9 @@ readonly RED='\033[31m'
 readonly NIX_USER_COUNT=${NIX_USER_COUNT:-32}
 readonly NIX_BUILD_GROUP_ID="${NIX_BUILD_GROUP_ID:-30000}"
 readonly NIX_BUILD_GROUP_NAME="nixbld"
-# darwin installer needs to override these
+# each system specific installer must set these:
-NIX_FIRST_BUILD_UID="${NIX_FIRST_BUILD_UID:-30001}"
+#   NIX_FIRST_BUILD_UID
-NIX_BUILD_USER_NAME_TEMPLATE="nixbld%d"
+#   NIX_BUILD_USER_NAME_TEMPLATE
 # Please don't change this. We don't support it, because the
 # default shell profile that comes with Nix doesn't support it.
 readonly NIX_ROOT="/nix"
@@ -707,6 +707,12 @@ EOF
     fi
 }
 
+check_required_system_specific_settings() {
+    if [ -z "${NIX_FIRST_BUILD_UID+x}" ] || [ -z "${NIX_BUILD_USER_NAME_TEMPLATE+x}" ]; then
+        failure "Internal error: System specific installer for $(uname) ($1) does not export required settings."
+    fi
+}
+
 welcome_to_nix() {
     local -r NIX_UID_RANGES="${NIX_FIRST_BUILD_UID}..$((NIX_FIRST_BUILD_UID + NIX_USER_COUNT - 1))"
     local -r RANGE_TEXT=$(echo -ne "${BLUE}(uids [${NIX_UID_RANGES}])${ESC}")
@@ -726,7 +732,9 @@ manager. This will happen in a few stages:
    if you are ready to continue.
 
 3. Create the system users ${RANGE_TEXT} and groups ${GROUP_TEXT}
-   that the Nix daemon uses to run builds.
+   that the Nix daemon uses to run builds. To create system users
+   in a different range, exit and run this tool again with
+   NIX_FIRST_BUILD_UID set.
 
 4. Perform the basic installation of the Nix files daemon.
 
@@ -968,13 +976,16 @@ main() {
     if is_os_darwin; then
         # shellcheck source=./install-darwin-multi-user.sh
         . "$EXTRACTED_NIX_PATH/install-darwin-multi-user.sh"
+        check_required_system_specific_settings "install-darwin-multi-user.sh"
     elif is_os_linux; then
         # shellcheck source=./install-systemd-multi-user.sh
         . "$EXTRACTED_NIX_PATH/install-systemd-multi-user.sh" # most of this works on non-systemd distros also
+        check_required_system_specific_settings "install-systemd-multi-user.sh"
     else
         failure "Sorry, I don't know what to do on $(uname)"
     fi
 
+
     welcome_to_nix
 
     if ! is_root; then

scripts/install-systemd-multi-user.sh

@@ -3,6 +3,10 @@
 set -eu
 set -o pipefail
 
+# System specific settings
+export NIX_FIRST_BUILD_UID="${NIX_FIRST_BUILD_UID:-30001}"
+export NIX_BUILD_USER_NAME_TEMPLATE="nixbld%d"
+
 readonly SERVICE_SRC=/lib/systemd/system/nix-daemon.service
 readonly SERVICE_DEST=/etc/systemd/system/nix-daemon.service
 

scripts/installer.nix

@@ -0,0 +1,36 @@
+{ lib
+, runCommand
+, nix
+, tarballs
+}:
+
+runCommand "installer-script" {
+  buildInputs = [ nix ];
+} ''
+  mkdir -p $out/nix-support
+
+  # Converts /nix/store/50p3qk8k...-nix-2.4pre20201102_550e11f/bin/nix to 50p3qk8k.../bin/nix.
+  tarballPath() {
+    # Remove the store prefix
+    local path=''${1#${builtins.storeDir}/}
+    # Get the path relative to the derivation root
+    local rest=''${path#*/}
+    # Get the derivation hash
+    local drvHash=''${path%%-*}
+    echo "$drvHash/$rest"
+  }
+
+  substitute ${./install.in} $out/install \
+    ${lib.concatMapStrings
+      (tarball: let
+          inherit (tarball.stdenv.hostPlatform) system;
+        in '' \
+        --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${tarball}/*.tar.xz) \
+        --replace '@tarballPath_${system}@' $(tarballPath ${tarball}/*.tar.xz) \
+        ''
+      )
+      tarballs
+    } --replace '@nixVersion@' ${nix.version}
+
+  echo "file installer $out/install" >> $out/nix-support/hydra-build-products
+''

src/build-remote/build-remote.cc

@@ -137,11 +137,8 @@ static int main_build_remote(int argc, char * * argv)
                 for (auto & m : machines) {
                     debug("considering building on remote machine '%s'", m.storeUri);
 
-                    if (m.enabled
+                    if (m.enabled &&
-                        && (neededSystem == "builtin"
+                        m.systemSupported(neededSystem) &&
-                            || std::find(m.systemTypes.begin(),
-                                m.systemTypes.end(),
-                                neededSystem) != m.systemTypes.end()) &&
                         m.allSupported(requiredFeatures) &&
                         m.mandatoryMet(requiredFeatures))
                     {
@@ -214,7 +211,7 @@ static int main_build_remote(int argc, char * * argv)
 
                         for (auto & m : machines)
                             error
-                                % concatStringsSep<std::vector<std::string>>(", ", m.systemTypes)
+                                % concatStringsSep<StringSet>(", ", m.systemTypes)
                                 % m.maxJobs
                                 % concatStringsSep<StringSet>(", ", m.supportedFeatures)
                                 % concatStringsSep<StringSet>(", ", m.mandatoryFeatures);

src/libcmd/built-path.cc

@@ -12,9 +12,9 @@ namespace nix {
     bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \
     { \
         const MY_TYPE* me = this; \
-        auto fields1 = std::make_tuple<const CHILD_TYPE &, const FIELD_TYPE &>(*me->drvPath, me->FIELD); \
+        auto fields1 = std::tie(*me->drvPath, me->FIELD); \
         me = &other; \
-        auto fields2 = std::make_tuple<const CHILD_TYPE &, const FIELD_TYPE &>(*me->drvPath, me->FIELD); \
+        auto fields2 = std::tie(*me->drvPath, me->FIELD); \
         return fields1 COMPARATOR fields2; \
     }
 #define CMP(CHILD_TYPE, MY_TYPE, FIELD) \

src/libcmd/editor-for.cc

@@ -1,5 +1,6 @@
 #include "editor-for.hh"
 #include "environment-variables.hh"
+#include "source-path.hh"
 
 namespace nix {
 

src/libcmd/editor-for.hh

@@ -2,7 +2,7 @@
 ///@file
 
 #include "types.hh"
-#include "input-accessor.hh"
+#include "source-path.hh"
 
 namespace nix {
 

src/libcmd/installable-attr-path.cc

@@ -58,22 +58,22 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths()
 
     Bindings & autoArgs = *cmd.getAutoArgs(*state);
 
-    DrvInfos drvInfos;
+    PackageInfos packageInfos;
-    getDerivations(*state, *v, "", autoArgs, drvInfos, false);
+    getDerivations(*state, *v, "", autoArgs, packageInfos, false);
 
     // Backward compatibility hack: group results by drvPath. This
     // helps keep .all output together.
     std::map<StorePath, OutputsSpec> byDrvPath;
 
-    for (auto & drvInfo : drvInfos) {
+    for (auto & packageInfo : packageInfos) {
-        auto drvPath = drvInfo.queryDrvPath();
+        auto drvPath = packageInfo.queryDrvPath();
         if (!drvPath)
             throw Error("'%s' is not a derivation", what());
 
         auto newOutputs = std::visit(overloaded {
             [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec {
                 std::set<std::string> outputsToInstall;
-                for (auto & output : drvInfo.queryOutputs(false, true))
+                for (auto & output : packageInfo.queryOutputs(false, true))
                     outputsToInstall.insert(output.first);
                 return OutputsSpec::Names { std::move(outputsToInstall) };
             },

src/libcmd/installable-flake.cc

@@ -52,7 +52,7 @@ Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::Locked
     auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs"));
     assert(aOutputs);
 
-    state.forceValue(*aOutputs->value, [&]() { return aOutputs->value->determinePos(noPos); });
+    state.forceValue(*aOutputs->value, aOutputs->value->determinePos(noPos));
 
     return aOutputs->value;
 }

src/libcmd/installable-value.cc

@@ -1,5 +1,6 @@
 #include "installable-value.hh"
 #include "eval-cache.hh"
+#include "fetch-to-store.hh"
 
 namespace nix {
 
@@ -44,7 +45,7 @@ ref<InstallableValue> InstallableValue::require(ref<Installable> installable)
 std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx)
 {
     if (v.type() == nPath) {
-        auto storePath = v.path().fetchToStore(state->store);
+        auto storePath = fetchToStore(*state->store, v.path());
         return {{
             .path = DerivedPath::Opaque {
                 .path = std::move(storePath),

src/libcmd/installable-value.hh

@@ -6,7 +6,7 @@
 
 namespace nix {
 
-struct DrvInfo;
+struct PackageInfo;
 struct SourceExprCommand;
 
 namespace eval_cache { class EvalCache; class AttrCursor; }

src/libcmd/installables.cc

@@ -715,7 +715,7 @@ BuiltPaths Installable::toBuiltPaths(
     }
 }
 
-StorePathSet Installable::toStorePaths(
+StorePathSet Installable::toStorePathSet(
     ref<Store> evalStore,
     ref<Store> store,
     Realise mode, OperateOn operateOn,
@@ -729,13 +729,27 @@ StorePathSet Installable::toStorePaths(
     return outPaths;
 }
 
+StorePaths Installable::toStorePaths(
+    ref<Store> evalStore,
+    ref<Store> store,
+    Realise mode, OperateOn operateOn,
+    const Installables & installables)
+{
+    StorePaths outPaths;
+    for (auto & path : toBuiltPaths(evalStore, store, mode, operateOn, installables)) {
+        auto thisOutPaths = path.outPaths();
+        outPaths.insert(outPaths.end(), thisOutPaths.begin(), thisOutPaths.end());
+    }
+    return outPaths;
+}
+
 StorePath Installable::toStorePath(
     ref<Store> evalStore,
     ref<Store> store,
     Realise mode, OperateOn operateOn,
     ref<Installable> installable)
 {
-    auto paths = toStorePaths(evalStore, store, mode, operateOn, {installable});
+    auto paths = toStorePathSet(evalStore, store, mode, operateOn, {installable});
 
     if (paths.size() != 1)
         throw Error("argument '%s' should evaluate to one store path", installable->what());

src/libcmd/installables.hh

@@ -12,7 +12,7 @@
 
 namespace nix {
 
-struct DrvInfo;
+struct PackageInfo;
 
 enum class Realise {
     /**
@@ -165,7 +165,14 @@ struct Installable
         const Installables & installables,
         BuildMode bMode = bmNormal);
 
-    static std::set<StorePath> toStorePaths(
+    static std::set<StorePath> toStorePathSet(
+        ref<Store> evalStore,
+        ref<Store> store,
+        Realise mode,
+        OperateOn operateOn,
+        const Installables & installables);
+
+    static std::vector<StorePath> toStorePaths(
         ref<Store> evalStore,
         ref<Store> store,
         Realise mode,

src/libcmd/local.mk

@@ -8,7 +8,7 @@ libcmd_SOURCES := $(wildcard $(d)/*.cc)
 
 libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers
 
-libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) -pthread
+libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) $(THREAD_LDFLAGS)
 
 libcmd_LIBS = libstore libutil libexpr libmain libfetchers
 

src/libcmd/markdown.cc

@@ -4,12 +4,15 @@
 #include "terminal.hh"
 
 #include <sys/queue.h>
+#if HAVE_LOWDOWN
 #include <lowdown.h>
+#endif
 
 namespace nix {
 
 std::string renderMarkdownToTerminal(std::string_view markdown)
 {
+#if HAVE_LOWDOWN
     int windowWidth = getWindowSize().second;
 
     struct lowdown_opts opts {
@@ -48,6 +51,9 @@ std::string renderMarkdownToTerminal(std::string_view markdown)
         throw Error("allocation error while rendering Markdown");
 
     return filterANSIEscapes(std::string(buf->data, buf->size), !shouldANSI());
+#else
+    return std::string(markdown);
+#endif
 }
 
 }

src/libcmd/repl.cc

@@ -5,7 +5,7 @@
 
 #include <setjmp.h>
 
-#ifdef READLINE
+#ifdef USE_READLINE
 #include <readline/history.h>
 #include <readline/readline.h>
 #else
@@ -93,9 +93,17 @@ struct NixRepl
     void evalString(std::string s, Value & v);
     void loadDebugTraceEnv(DebugTrace & dt);
 
-    typedef std::set<Value *> ValuesSeen;
+    void printValue(std::ostream & str,
-    std::ostream & printValue(std::ostream & str, Value & v, unsigned int maxDepth);
+                              Value & v,
-    std::ostream & printValue(std::ostream & str, Value & v, unsigned int maxDepth, ValuesSeen & seen);
+                              unsigned int maxDepth = std::numeric_limits<unsigned int>::max())
+    {
+        ::nix::printValue(*state, str, v, PrintOptions {
+            .ansiColors = true,
+            .force = true,
+            .derivationPaths = true,
+            .maxDepth = maxDepth
+        });
+    }
 };
 
 std::string removeWhitespace(std::string s)
@@ -112,7 +120,7 @@ NixRepl::NixRepl(const SearchPath & searchPath, nix::ref<Store> store, ref<EvalS
     : AbstractNixRepl(state)
     , debugTraceIndex(0)
     , getValues(getValues)
-    , staticEnv(new StaticEnv(false, state->staticBaseEnv.get()))
+    , staticEnv(new StaticEnv(nullptr, state->staticBaseEnv.get()))
     , historyFile(getDataDir() + "/nix/repl-history")
 {
 }
@@ -221,7 +229,7 @@ static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positi
     // prefer direct pos, but if noPos then try the expr.
     auto pos = dt.pos
         ? dt.pos
-        : static_cast<std::shared_ptr<AbstractPos>>(positions[dt.expr.getPos() ? dt.expr.getPos() : noPos]);
+        : positions[dt.expr.getPos() ? dt.expr.getPos() : noPos];
 
     if (pos) {
         out << pos;
@@ -246,17 +254,17 @@ void NixRepl::mainLoop()
     rl_readline_name = "nix-repl";
     try {
         createDirs(dirOf(historyFile));
-    } catch (SysError & e) {
+    } catch (SystemError & e) {
         logWarning(e.info());
     }
-#ifndef READLINE
+#ifndef USE_READLINE
     el_hist_size = 1000;
 #endif
     read_history(historyFile.c_str());
     auto oldRepl = curRepl;
     curRepl = this;
     Finally restoreRepl([&] { curRepl = oldRepl; });
-#ifndef READLINE
+#ifndef USE_READLINE
     rl_set_complete_func(completionCallback);
     rl_set_list_possib_func(listPossibleCallback);
 #endif
@@ -442,10 +450,10 @@ static bool isVarName(std::string_view s)
 
 
 StorePath NixRepl::getDerivationPath(Value & v) {
-    auto drvInfo = getDerivation(*state, v, false);
+    auto packageInfo = getDerivation(*state, v, false);
-    if (!drvInfo)
+    if (!packageInfo)
         throw Error("expression does not evaluate to a derivation, so I can't build it");
-    auto drvPath = drvInfo->queryDrvPath();
+    auto drvPath = packageInfo->queryDrvPath();
     if (!drvPath)
         throw Error("expression did not evaluate to a valid derivation (no 'drvPath' attribute)");
     if (!state->store->isValidPath(*drvPath))
@@ -708,7 +716,8 @@ bool NixRepl::processLine(std::string line)
     else if (command == ":p" || command == ":print") {
         Value v;
         evalString(arg, v);
-        printValue(std::cout, v, 1000000000) << std::endl;
+        printValue(std::cout, v);
+        std::cout << std::endl;
     }
 
     else if (command == ":q" || command == ":quit") {
@@ -770,7 +779,8 @@ bool NixRepl::processLine(std::string line)
         } else {
             Value v;
             evalString(line, v);
-            printValue(std::cout, v, 1) << std::endl;
+            printValue(std::cout, v, 1);
+            std::cout << std::endl;
         }
     }
 
@@ -888,145 +898,7 @@ void NixRepl::evalString(std::string s, Value & v)
 {
     Expr * e = parseString(s);
     e->eval(*state, *env, v);
-    state->forceValue(v, [&]() { return v.determinePos(noPos); });
+    state->forceValue(v, v.determinePos(noPos));
-}
-
-
-std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int maxDepth)
-{
-    ValuesSeen seen;
-    return printValue(str, v, maxDepth, seen);
-}
-
-
-
-
-// FIXME: lot of cut&paste from Nix's eval.cc.
-std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int maxDepth, ValuesSeen & seen)
-{
-    str.flush();
-    checkInterrupt();
-
-    state->forceValue(v, [&]() { return v.determinePos(noPos); });
-
-    switch (v.type()) {
-
-    case nInt:
-        str << ANSI_CYAN << v.integer << ANSI_NORMAL;
-        break;
-
-    case nBool:
-        str << ANSI_CYAN;
-        printLiteralBool(str, v.boolean);
-        str << ANSI_NORMAL;
-        break;
-
-    case nString:
-        str << ANSI_WARNING;
-        printLiteralString(str, v.string_view());
-        str << ANSI_NORMAL;
-        break;
-
-    case nPath:
-        str << ANSI_GREEN << v.path().to_string() << ANSI_NORMAL; // !!! escaping?
-        break;
-
-    case nNull:
-        str << ANSI_CYAN "null" ANSI_NORMAL;
-        break;
-
-    case nAttrs: {
-        seen.insert(&v);
-
-        bool isDrv = state->isDerivation(v);
-
-        if (isDrv) {
-            str << "«derivation ";
-            Bindings::iterator i = v.attrs->find(state->sDrvPath);
-            NixStringContext context;
-            if (i != v.attrs->end())
-                str << state->store->printStorePath(state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the drvPath of a derivation"));
-            else
-                str << "???";
-            str << "»";
-        }
-
-        else if (maxDepth > 0) {
-            str << "{ ";
-
-            typedef std::map<std::string, Value *> Sorted;
-            Sorted sorted;
-            for (auto & i : *v.attrs)
-                sorted.emplace(state->symbols[i.name], i.value);
-
-            for (auto & i : sorted) {
-                printAttributeName(str, i.first);
-                str << " = ";
-                if (seen.count(i.second))
-                    str << "«repeated»";
-                else
-                    try {
-                        printValue(str, *i.second, maxDepth - 1, seen);
-                    } catch (AssertionError & e) {
-                        str << ANSI_RED "«error: " << e.msg() << "»" ANSI_NORMAL;
-                    }
-                str << "; ";
-            }
-
-            str << "}";
-        } else
-            str << "{ ... }";
-
-        break;
-    }
-
-    case nList:
-        seen.insert(&v);
-
-        str << "[ ";
-        if (maxDepth > 0)
-            for (auto elem : v.listItems()) {
-                if (seen.count(elem))
-                    str << "«repeated»";
-                else
-                    try {
-                        printValue(str, *elem, maxDepth - 1, seen);
-                    } catch (AssertionError & e) {
-                        str << ANSI_RED "«error: " << e.msg() << "»" ANSI_NORMAL;
-                    }
-                str << " ";
-            }
-        else
-            str << "... ";
-        str << "]";
-        break;
-
-    case nFunction:
-        if (v.isLambda()) {
-            std::ostringstream s;
-            s << state->positions[v.lambda.fun->pos];
-            str << ANSI_BLUE "«lambda @ " << filterANSIEscapes(s.str()) << "»" ANSI_NORMAL;
-        } else if (v.isPrimOp()) {
-            str << ANSI_MAGENTA "«primop»" ANSI_NORMAL;
-        } else if (v.isPrimOpApp()) {
-            str << ANSI_BLUE "«primop-app»" ANSI_NORMAL;
-        } else {
-            abort();
-        }
-        break;
-
-    case nFloat:
-        str << v.fpoint;
-        break;
-
-    case nThunk:
-    case nExternal:
-    default:
-        str << ANSI_RED "«unknown»" ANSI_NORMAL;
-        break;
-    }
-
-    return str;
 }
 
 

src/libexpr/eval-inline.hh

@@ -1,6 +1,7 @@
 #pragma once
 ///@file
 
+#include "print.hh"
 #include "eval.hh"
 
 namespace nix {
@@ -73,8 +74,6 @@ Env & EvalState::allocEnv(size_t size)
 #endif
         env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *));
 
-    env->type = Env::Plain;
-
     /* We assume that env->values has been cleared by the allocator; maybeThunk() and lookupVar fromWith expect this. */
 
     return *env;
@@ -83,13 +82,6 @@ Env & EvalState::allocEnv(size_t size)
 
 [[gnu::always_inline]]
 void EvalState::forceValue(Value & v, const PosIdx pos)
-{
-    forceValue(v, [&]() { return pos; });
-}
-
-
-template<typename Callable>
-void EvalState::forceValue(Value & v, Callable getPos)
 {
     if (v.isThunk()) {
         Env * env = v.thunk.env;
@@ -100,15 +92,12 @@ void EvalState::forceValue(Value & v, Callable getPos)
             expr->eval(*this, *env, v);
         } catch (...) {
             v.mkThunk(env, expr);
+            tryFixupBlackHolePos(v, pos);
             throw;
         }
     }
-    else if (v.isApp()) {
+    else if (v.isApp())
-        PosIdx pos = getPos();
         callFunction(*v.app.left, *v.app.right, v, pos);
-    }
-    else if (v.isBlackhole())
-        error("infinite recursion encountered").atPos(getPos()).template debugThrow<EvalError>();
 }
 
 
@@ -126,7 +115,10 @@ inline void EvalState::forceAttrs(Value & v, Callable getPos, std::string_view e
     PosIdx pos = getPos();
     forceValue(v, pos);
     if (v.type() != nAttrs) {
-        error("value is %1% while a set was expected", showType(v)).withTrace(pos, errorCtx).debugThrow<TypeError>();
+        error("expected a set but found %1%: %2%",
+              showType(v),
+              ValuePrinter(*this, v, errorPrintOptions))
+            .withTrace(pos, errorCtx).debugThrow<TypeError>();
     }
 }
 
@@ -136,7 +128,10 @@ inline void EvalState::forceList(Value & v, const PosIdx pos, std::string_view e
 {
     forceValue(v, pos);
     if (!v.isList()) {
-        error("value is %1% while a list was expected", showType(v)).withTrace(pos, errorCtx).debugThrow<TypeError>();
+        error("expected a list but found %1%: %2%",
+              showType(v),
+              ValuePrinter(*this, v, errorPrintOptions))
+            .withTrace(pos, errorCtx).debugThrow<TypeError>();
     }
 }
 

src/libexpr/eval-settings.cc

@@ -89,6 +89,12 @@ std::string EvalSettings::resolvePseudoUrl(std::string_view url)
         return std::string(url);
 }
 
+const std::string & EvalSettings::getCurrentSystem()
+{
+    const auto & evalSystem = currentSystem.get();
+    return evalSystem != "" ? evalSystem : settings.thisSystem.get();
+}
+
 EvalSettings evalSettings;
 
 static GlobalConfig::Register rEvalSettings(&evalSettings);

src/libexpr/eval-settings.hh

@@ -27,6 +27,26 @@ struct EvalSettings : Config
           [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath).
         )"};
 
+    Setting<std::string> currentSystem{
+        this, "", "eval-system",
+        R"(
+          This option defines
+          [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem)
+          in the Nix language if it is set as a non-empty string.
+          Otherwise, if it is defined as the empty string (the default), the value of the
+          [`system` ](#conf-system)
+          configuration setting is used instead.
+
+          Unlike `system`, this setting does not change what kind of derivations can be built locally.
+          This is useful for evaluating Nix code on one system to produce derivations to be built on another type of system.
+        )"};
+
+    /**
+     * Implements the `eval-system` vs `system` defaulting logic
+     * described for `eval-system`.
+     */
+    const std::string & getCurrentSystem();
+
     Setting<bool> restrictEval{
         this, false, "restrict-eval",
         R"(
@@ -68,6 +88,11 @@ struct EvalSettings : Config
           evaluation mode. For example, when set to
           `https://github.com/NixOS`, builtin functions such as `fetchGit` are
           allowed to access `https://github.com/NixOS/patchelf.git`.
+
+          Access is granted when
+          - the URI is equal to the prefix,
+          - or the URI is a subpath of the prefix,
+          - or the prefix is a URI scheme ended by a colon `:` and the URI has the same scheme.
         )"};
 
     Setting<bool> traceFunctionCalls{this, false, "trace-function-calls",
@@ -99,6 +124,9 @@ struct EvalSettings : Config
 
     Setting<bool> traceVerbose{this, false, "trace-verbose",
         "Whether `builtins.traceVerbose` should trace its first argument when evaluated."};
+
+    Setting<unsigned int> maxCallDepth{this, 10000, "max-call-depth",
+        "The maximum function call depth to allow before erroring."};
 };
 
 extern EvalSettings evalSettings;

src/libexpr/eval.cc

@@ -2,6 +2,7 @@
 #include "eval-settings.hh"
 #include "hash.hh"
 #include "primops.hh"
+#include "print-options.hh"
 #include "types.hh"
 #include "util.hh"
 #include "store-api.hh"
@@ -18,6 +19,11 @@
 #include "memory-input-accessor.hh"
 #include "signals.hh"
 #include "gc-small-vector.hh"
+#include "url.hh"
+#include "fetch-to-store.hh"
+#include "tarball.hh"
+#include "flake/flakeref.hh"
+#include "parser-tab.hh"
 
 #include <algorithm>
 #include <chrono>
@@ -27,9 +33,9 @@
 #include <unistd.h>
 #include <sys/time.h>
 #include <sys/resource.h>
-#include <iostream>
 #include <fstream>
 #include <functional>
+#include <iostream>
 
 #include <sys/resource.h>
 #include <nlohmann/json.hpp>
@@ -103,116 +109,23 @@ RootValue allocRootValue(Value * v)
 #endif
 }
 
-void Value::print(const SymbolTable &symbols, std::ostream &str,
-                  std::set<const void *> *seen, int depth) const
-
-{
-    checkInterrupt();
-
-    if (depth <= 0) {
-        str << "«too deep»";
-        return;
-    }
-    switch (internalType) {
-    case tInt:
-        str << integer;
-        break;
-    case tBool:
-        printLiteralBool(str, boolean);
-        break;
-    case tString:
-        printLiteralString(str, string_view());
-        break;
-    case tPath:
-        str << path().to_string(); // !!! escaping?
-        break;
-    case tNull:
-        str << "null";
-        break;
-    case tAttrs: {
-        if (seen && !attrs->empty() && !seen->insert(attrs).second)
-            str << "«repeated»";
-        else {
-            str << "{ ";
-            for (auto & i : attrs->lexicographicOrder(symbols)) {
-                str << symbols[i->name] << " = ";
-                i->value->print(symbols, str, seen, depth - 1);
-                str << "; ";
-            }
-            str << "}";
-        }
-        break;
-    }
-    case tList1:
-    case tList2:
-    case tListN:
-        if (seen && listSize() && !seen->insert(listElems()).second)
-            str << "«repeated»";
-        else {
-            str << "[ ";
-            for (auto v2 : listItems()) {
-                if (v2)
-                    v2->print(symbols, str, seen, depth - 1);
-                else
-                    str << "(nullptr)";
-                str << " ";
-            }
-            str << "]";
-        }
-        break;
-    case tThunk:
-    case tApp:
-        str << "<CODE>";
-        break;
-    case tLambda:
-        str << "<LAMBDA>";
-        break;
-    case tPrimOp:
-        str << "<PRIMOP>";
-        break;
-    case tPrimOpApp:
-        str << "<PRIMOP-APP>";
-        break;
-    case tExternal:
-        str << *external;
-        break;
-    case tFloat:
-        str << fpoint;
-        break;
-    case tBlackhole:
-        // Although we know for sure that it's going to be an infinite recursion
-        // when this value is accessed _in the current context_, it's likely
-        // that the user will misinterpret a simpler «infinite recursion» output
-        // as a definitive statement about the value, while in fact it may be
-        // a valid value after `builtins.trace` and perhaps some other steps
-        // have completed.
-        str << "«potential infinite recursion»";
-        break;
-    default:
-        printError("Nix evaluator internal error: Value::print(): invalid value type %1%", internalType);
-        abort();
-    }
-}
-
-void Value::print(const SymbolTable &symbols, std::ostream &str,
-                  bool showRepeated, int depth) const {
-    std::set<const void *> seen;
-    print(symbols, str, showRepeated ? nullptr : &seen, depth);
-}
-
 // Pretty print types for assertion errors
 std::ostream & operator << (std::ostream & os, const ValueType t) {
     os << showType(t);
     return os;
 }
 
-std::string printValue(const EvalState & state, const Value & v)
+std::string printValue(EvalState & state, Value & v)
 {
     std::ostringstream out;
-    v.print(state.symbols, out);
+    v.print(state, out);
     return out.str();
 }
 
+void Value::print(EvalState & state, std::ostream & str, PrintOptions options)
+{
+    printValue(state, str, *this, options);
+}
 
 const Value * getPrimOp(const Value &v) {
     const Value * primOp = &v;
@@ -255,9 +168,8 @@ std::string showType(const Value & v)
         case tPrimOpApp:
             return fmt("the partially applied built-in function '%s'", std::string(getPrimOp(v)->primOp->name));
         case tExternal: return v.external->showType();
-        case tThunk: return "a thunk";
+        case tThunk: return v.isBlackhole() ? "a black hole" : "a thunk";
         case tApp: return "a function application";
-        case tBlackhole: return "a black hole";
     default:
         return std::string(showType(v.type()));
     }
@@ -508,6 +420,16 @@ EvalState::EvalState(
     , sPath(symbols.create("path"))
     , sPrefix(symbols.create("prefix"))
     , sOutputSpecified(symbols.create("outputSpecified"))
+    , exprSymbols{
+        .sub = symbols.create("__sub"),
+        .lessThan = symbols.create("__lessThan"),
+        .mul = symbols.create("__mul"),
+        .div = symbols.create("__div"),
+        .or_ = symbols.create("or"),
+        .findFile = symbols.create("__findFile"),
+        .nixPath = symbols.create("__nixPath"),
+        .body = symbols.create("body"),
+    }
     , repair(NoRepair)
     , emptyBindings(0)
     , rootFS(
@@ -542,7 +464,7 @@ EvalState::EvalState(
     , env1AllocCache(std::allocate_shared<void *>(traceable_allocator<void *>(), nullptr))
 #endif
     , baseEnv(allocEnv(128))
-    , staticBaseEnv{std::make_shared<StaticEnv>(false, nullptr)}
+    , staticBaseEnv{std::make_shared<StaticEnv>(nullptr, nullptr)}
 {
     corepkgsFS->setPathDisplay("<nix", ">");
     internalFS->setPathDisplay("«nix-internal»", "");
@@ -553,6 +475,8 @@ EvalState::EvalState(
 
     static_assert(sizeof(Env) <= 16, "environment must be <= 16 bytes");
 
+    vEmptyList.mkList(0);
+
     /* Initialise the Nix expression search path. */
     if (!evalSettings.pureEval) {
         for (auto & i : _searchPath.elements)
@@ -599,21 +523,45 @@ void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value &
     mkStorePathString(storePath, v);
 }
 
+inline static bool isJustSchemePrefix(std::string_view prefix)
+{
+    return
+        !prefix.empty()
+        && prefix[prefix.size() - 1] == ':'
+        && isValidSchemeName(prefix.substr(0, prefix.size() - 1));
+}
+
+bool isAllowedURI(std::string_view uri, const Strings & allowedUris)
+{
+    /* 'uri' should be equal to a prefix, or in a subdirectory of a
+       prefix. Thus, the prefix https://github.co does not permit
+       access to https://github.com. */
+    for (auto & prefix : allowedUris) {
+        if (uri == prefix
+            // Allow access to subdirectories of the prefix.
+            || (uri.size() > prefix.size()
+                && prefix.size() > 0
+                && hasPrefix(uri, prefix)
+                && (
+                    // Allow access to subdirectories of the prefix.
+                    prefix[prefix.size() - 1] == '/'
+                    || uri[prefix.size()] == '/'
+
+                    // Allow access to whole schemes
+                    || isJustSchemePrefix(prefix)
+                    )
+                ))
+            return true;
+    }
+
+    return false;
+}
+
 void EvalState::checkURI(const std::string & uri)
 {
     if (!evalSettings.restrictEval) return;
 
-    /* 'uri' should be equal to a prefix, or in a subdirectory of a
+    if (isAllowedURI(uri, evalSettings.allowedUris.get())) return;
-       prefix. Thus, the prefix https://github.co does not permit
-       access to https://github.com. Note: this allows 'http://' and
-       'https://' as prefixes for any http/https URI. */
-    for (auto & prefix : evalSettings.allowedUris.get())
-        if (uri == prefix ||
-            (uri.size() > prefix.size()
-            && prefix.size() > 0
-            && hasPrefix(uri, prefix)
-            && (prefix[prefix.size() - 1] == '/' || uri[prefix.size()] == '/')))
-            return;
 
     /* If the URI is a path, then check it against allowedPaths as
        well. */
@@ -682,6 +630,26 @@ void PrimOp::check()
 }
 
 
+std::ostream & operator<<(std::ostream & output, PrimOp & primOp)
+{
+    output << "primop " << primOp.name;
+    return output;
+}
+
+
+PrimOp * Value::primOpAppPrimOp() const
+{
+    Value * left = primOpApp.left;
+    while (left && !left->isPrimOp()) {
+        left = left->primOpApp.left;
+    }
+
+    if (!left)
+        return nullptr;
+    return left->primOp;
+}
+
+
 void Value::mkPrimOp(PrimOp * p)
 {
     p->check();
@@ -756,7 +724,7 @@ void printStaticEnvBindings(const SymbolTable & st, const StaticEnv & se)
 // just for the current level of Env, not the whole chain.
 void printWithBindings(const SymbolTable & st, const Env & env)
 {
-    if (env.type == Env::HasWithAttrs) {
+    if (!env.values[0]->isThunk()) {
         std::cout << "with: ";
         std::cout << ANSI_MAGENTA;
         Bindings::iterator j = env.values[0]->attrs->begin();
@@ -810,7 +778,7 @@ void mapStaticEnvBindings(const SymbolTable & st, const StaticEnv & se, const En
     if (env.up && se.up) {
         mapStaticEnvBindings(st, *se.up, *env.up, vm);
 
-        if (env.type == Env::HasWithAttrs) {
+        if (!env.values[0]->isThunk()) {
             // add 'with' bindings.
             Bindings::iterator j = env.values[0]->attrs->begin();
             while (j != env.values[0]->attrs->end()) {
@@ -843,7 +811,7 @@ void EvalState::runDebugRepl(const Error * error, const Env & env, const Expr &
         ? std::make_unique<DebugTraceStacker>(
             *this,
             DebugTrace {
-                .pos = error->info().errPos ? error->info().errPos : static_cast<std::shared_ptr<AbstractPos>>(positions[expr.getPos()]),
+                .pos = error->info().errPos ? error->info().errPos : positions[expr.getPos()],
                 .expr = expr,
                 .env = env,
                 .hint = error->info().msg,
@@ -882,7 +850,7 @@ static std::unique_ptr<DebugTraceStacker> makeDebugTraceStacker(
     EvalState & state,
     Expr & expr,
     Env & env,
-    std::shared_ptr<AbstractPos> && pos,
+    std::shared_ptr<Pos> && pos,
     const char * s,
     const std::string & s2)
 {
@@ -948,22 +916,23 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
 
     if (!var.fromWith) return env->values[var.displ];
 
+    // This early exit defeats the `maybeThunk` optimization for variables from `with`,
+    // The added complexity of handling this appears to be similarly in cost, or
+    // the cases where applicable were insignificant in the first place.
+    if (noEval) return nullptr;
+
+    auto * fromWith = var.fromWith;
     while (1) {
-        if (env->type == Env::HasWithExpr) {
+        forceAttrs(*env->values[0], fromWith->pos, "while evaluating the first subexpression of a with expression");
-            if (noEval) return 0;
-            Value * v = allocValue();
-            evalAttrs(*env->up, (Expr *) env->values[0], *v, noPos, "<borked>");
-            env->values[0] = v;
-            env->type = Env::HasWithAttrs;
-        }
         Bindings::iterator j = env->values[0]->attrs->find(var.name);
         if (j != env->values[0]->attrs->end()) {
             if (countCalls) attrSelects[j->pos]++;
             return j->value;
         }
-        if (!env->prevWith)
+        if (!fromWith->parentWith)
             error("undefined variable '%1%'", symbols[var.name]).atPos(var.pos).withFrame(*env, var).debugThrow<UndefinedVarError>();
-        for (size_t l = env->prevWith; l; --l, env = env->up) ;
+        for (size_t l = fromWith->prevWith; l; --l, env = env->up) ;
+        fromWith = fromWith->parentWith;
     }
 }
 
@@ -1159,7 +1128,7 @@ void EvalState::evalFile(const SourcePath & path, Value & v, bool mustBeTrivial)
                 *this,
                 *e,
                 this->baseEnv,
-                e->getPos() ? static_cast<std::shared_ptr<AbstractPos>>(positions[e->getPos()]) : nullptr,
+                e->getPos() ? std::make_shared<Pos>(positions[e->getPos()]) : nullptr,
                 "while evaluating the file '%1%':", resolvedPath.to_string())
             : nullptr;
 
@@ -1198,7 +1167,10 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const PosIdx pos, std::stri
         Value v;
         e->eval(*this, env, v);
         if (v.type() != nBool)
-            error("value is %1% while a Boolean was expected", showType(v)).withFrame(env, *e).debugThrow<TypeError>();
+            error("expected a Boolean but found %1%: %2%",
+                  showType(v),
+                  ValuePrinter(*this, v, errorPrintOptions))
+                .withFrame(env, *e).debugThrow<TypeError>();
         return v.boolean;
     } catch (Error & e) {
         e.addTrace(positions[pos], errorCtx);
@@ -1212,7 +1184,10 @@ inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v, const PosIdx po
     try {
         e->eval(*this, env, v);
         if (v.type() != nAttrs)
-            error("value is %1% while a set was expected", showType(v)).withFrame(env, *e).debugThrow<TypeError>();
+            error("expected a set but found %1%: %2%",
+                  showType(v),
+                  ValuePrinter(*this, v, errorPrintOptions))
+                .withFrame(env, *e).debugThrow<TypeError>();
     } catch (Error & e) {
         e.addTrace(positions[pos], errorCtx);
         throw;
@@ -1359,6 +1334,15 @@ void ExprList::eval(EvalState & state, Env & env, Value & v)
 }
 
 
+Value * ExprList::maybeThunk(EvalState & state, Env & env)
+{
+    if (elems.empty()) {
+        return &state.vEmptyList;
+    }
+    return Expr::maybeThunk(state, env);
+}
+
+
 void ExprVar::eval(EvalState & state, Env & env, Value & v)
 {
     Value * v2 = state.lookupVar(&env, *this, false);
@@ -1480,9 +1464,27 @@ void ExprLambda::eval(EvalState & state, Env & env, Value & v)
     v.mkLambda(&env, this);
 }
 
+namespace {
+/** Increments a count on construction and decrements on destruction.
+ */
+class CallDepth {
+  size_t & count;
+public:
+  CallDepth(size_t & count) : count(count) {
+    ++count;
+  }
+  ~CallDepth() {
+    --count;
+  }
+};
+};
 
 void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const PosIdx pos)
 {
+    if (callDepth > evalSettings.maxCallDepth)
+        error("stack overflow; max-call-depth exceeded").atPos(pos).template debugThrow<EvalError>();
+    CallDepth _level(callDepth);
+
     auto trace = evalSettings.traceFunctionCalls
         ? std::make_unique<FunctionCallTrace>(positions[pos])
         : nullptr;
@@ -1621,15 +1623,15 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                 return;
             } else {
                 /* We have all the arguments, so call the primop. */
-                auto name = vCur.primOp->name;
+                auto * fn = vCur.primOp;
 
                 nrPrimOpCalls++;
-                if (countCalls) primOpCalls[name]++;
+                if (countCalls) primOpCalls[fn->name]++;
 
                 try {
-                    vCur.primOp->fun(*this, vCur.determinePos(noPos), args, vCur);
+                    fn->fun(*this, vCur.determinePos(noPos), args, vCur);
                 } catch (Error & e) {
-                    addErrorTrace(e, pos, "while calling the '%1%' builtin", name);
+                    addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name);
                     throw;
                 }
 
@@ -1666,18 +1668,18 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                 for (size_t i = 0; i < argsLeft; ++i)
                     vArgs[argsDone + i] = args[i];
 
-                auto name = primOp->primOp->name;
+                auto fn = primOp->primOp;
                 nrPrimOpCalls++;
-                if (countCalls) primOpCalls[name]++;
+                if (countCalls) primOpCalls[fn->name]++;
 
                 try {
                     // TODO:
                     // 1. Unify this and above code. Heavily redundant.
                     // 2. Create a fake env (arg1, arg2, etc.) and a fake expr (arg1: arg2: etc: builtins.name arg1 arg2 etc)
                     //    so the debugger allows to inspect the wrong parameters passed to the builtin.
-                    primOp->primOp->fun(*this, vCur.determinePos(noPos), vArgs, vCur);
+                    fn->fun(*this, vCur.determinePos(noPos), vArgs, vCur);
                 } catch (Error & e) {
-                    addErrorTrace(e, pos, "while calling the '%1%' builtin", name);
+                    addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name);
                     throw;
                 }
 
@@ -1703,7 +1705,11 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
         }
 
         else
-            error("attempt to call something which is not a function but %1%", showType(vCur)).atPos(pos).debugThrow<TypeError>();
+            error("attempt to call something which is not a function but %1%: %2%",
+                    showType(vCur),
+                    ValuePrinter(*this, vCur, errorPrintOptions))
+                .atPos(pos)
+                .debugThrow<TypeError>();
     }
 
     vRes = vCur;
@@ -1791,9 +1797,7 @@ void ExprWith::eval(EvalState & state, Env & env, Value & v)
 {
     Env & env2(state.allocEnv(1));
     env2.up = &env;
-    env2.prevWith = prevWith;
+    env2.values[0] = attrs->maybeThunk(state, env);
-    env2.type = Env::HasWithExpr;
-    env2.values[0] = (Value *) attrs;
 
     body->eval(state, env2, v);
 }
@@ -2031,6 +2035,29 @@ void ExprPos::eval(EvalState & state, Env & env, Value & v)
 }
 
 
+void ExprBlackHole::eval(EvalState & state, Env & env, Value & v)
+{
+    state.error("infinite recursion encountered")
+        .debugThrow<InfiniteRecursionError>();
+}
+
+// always force this to be separate, otherwise forceValue may inline it and take
+// a massive perf hit
+[[gnu::noinline]]
+void EvalState::tryFixupBlackHolePos(Value & v, PosIdx pos)
+{
+    if (!v.isBlackhole())
+        return;
+    auto e = std::current_exception();
+    try {
+        std::rethrow_exception(e);
+    } catch (InfiniteRecursionError & e) {
+        e.err.errPos = positions[pos];
+    } catch (...) {
+    }
+}
+
+
 void EvalState::forceValueDeep(Value & v)
 {
     std::set<const Value *> seen;
@@ -2040,7 +2067,7 @@ void EvalState::forceValueDeep(Value & v)
     recurse = [&](Value & v) {
         if (!seen.insert(&v).second) return;
 
-        forceValue(v, [&]() { return v.determinePos(noPos); });
+        forceValue(v, v.determinePos(noPos));
 
         if (v.type() == nAttrs) {
             for (auto & i : *v.attrs)
@@ -2073,7 +2100,10 @@ NixInt EvalState::forceInt(Value & v, const PosIdx pos, std::string_view errorCt
     try {
         forceValue(v, pos);
         if (v.type() != nInt)
-            error("value is %1% while an integer was expected", showType(v)).debugThrow<TypeError>();
+            error("expected an integer but found %1%: %2%",
+                  showType(v),
+                  ValuePrinter(*this, v, errorPrintOptions))
+                .debugThrow<TypeError>();
         return v.integer;
     } catch (Error & e) {
         e.addTrace(positions[pos], errorCtx);
@@ -2089,7 +2119,10 @@ NixFloat EvalState::forceFloat(Value & v, const PosIdx pos, std::string_view err
         if (v.type() == nInt)
             return v.integer;
         else if (v.type() != nFloat)
-            error("value is %1% while a float was expected", showType(v)).debugThrow<TypeError>();
+            error("expected a float but found %1%: %2%",
+                  showType(v),
+                  ValuePrinter(*this, v, errorPrintOptions))
+                .debugThrow<TypeError>();
         return v.fpoint;
     } catch (Error & e) {
         e.addTrace(positions[pos], errorCtx);
@@ -2103,7 +2136,10 @@ bool EvalState::forceBool(Value & v, const PosIdx pos, std::string_view errorCtx
     try {
         forceValue(v, pos);
         if (v.type() != nBool)
-            error("value is %1% while a Boolean was expected", showType(v)).debugThrow<TypeError>();
+            error("expected a Boolean but found %1%: %2%",
+                  showType(v),
+                  ValuePrinter(*this, v, errorPrintOptions))
+                .debugThrow<TypeError>();
         return v.boolean;
     } catch (Error & e) {
         e.addTrace(positions[pos], errorCtx);
@@ -2123,7 +2159,10 @@ void EvalState::forceFunction(Value & v, const PosIdx pos, std::string_view erro
     try {
         forceValue(v, pos);
         if (v.type() != nFunction && !isFunctor(v))
-            error("value is %1% while a function was expected", showType(v)).debugThrow<TypeError>();
+            error("expected a function but found %1%: %2%",
+                  showType(v),
+                  ValuePrinter(*this, v, errorPrintOptions))
+                .debugThrow<TypeError>();
     } catch (Error & e) {
         e.addTrace(positions[pos], errorCtx);
         throw;
@@ -2136,7 +2175,10 @@ std::string_view EvalState::forceString(Value & v, const PosIdx pos, std::string
     try {
         forceValue(v, pos);
         if (v.type() != nString)
-            error("value is %1% while a string was expected", showType(v)).debugThrow<TypeError>();
+            error("expected a string but found %1%: %2%",
+                  showType(v),
+                  ValuePrinter(*this, v, errorPrintOptions))
+                .debugThrow<TypeError>();
         return v.string_view();
     } catch (Error & e) {
         e.addTrace(positions[pos], errorCtx);
@@ -2230,7 +2272,9 @@ BackedStringView EvalState::coerceToString(
             return std::move(*maybeString);
         auto i = v.attrs->find(sOutPath);
         if (i == v.attrs->end()) {
-            error("cannot coerce %1% to a string", showType(v))
+            error("cannot coerce %1% to a string: %2%",
+                  showType(v),
+                  ValuePrinter(*this, v, errorPrintOptions))
                 .withTrace(pos, errorCtx)
                 .debugThrow<TypeError>();
         }
@@ -2276,7 +2320,9 @@ BackedStringView EvalState::coerceToString(
         }
     }
 
-    error("cannot coerce %1% to a string", showType(v))
+    error("cannot coerce %1% to a string: %2%",
+          showType(v),
+          ValuePrinter(*this, v, errorPrintOptions))
         .withTrace(pos, errorCtx)
         .debugThrow<TypeError>();
 }
@@ -2292,7 +2338,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
     auto dstPath = i != srcToStore.end()
         ? i->second
         : [&]() {
-            auto dstPath = path.fetchToStore(store, path.baseName(), FileIngestionMethod::Recursive, nullptr, repair);
+            auto dstPath = fetchToStore(*store, path.resolveSymlinks(), path.baseName(), FileIngestionMethod::Recursive, nullptr, repair);
             allowPath(dstPath);
             srcToStore.insert_or_assign(path, dstPath);
             printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath));
@@ -2432,7 +2478,7 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v
             return v1.boolean == v2.boolean;
 
         case nString:
-            return v1.string_view().compare(v2.string_view()) == 0;
+            return strcmp(v1.c_str(), v2.c_str()) == 0;
 
         case nPath:
             return
@@ -2633,10 +2679,187 @@ void EvalState::printStatistics()
 }
 
 
+SourcePath resolveExprPath(SourcePath path)
+{
+    unsigned int followCount = 0, maxFollow = 1024;
+
+    /* If `path' is a symlink, follow it.  This is so that relative
+       path references work. */
+    while (!path.path.isRoot()) {
+        // Basic cycle/depth limit to avoid infinite loops.
+        if (++followCount >= maxFollow)
+            throw Error("too many symbolic links encountered while traversing the path '%s'", path);
+        auto p = path.parent().resolveSymlinks() + path.baseName();
+        if (p.lstat().type != InputAccessor::tSymlink) break;
+        path = {path.accessor, CanonPath(p.readLink(), path.path.parent().value_or(CanonPath::root))};
+    }
+
+    /* If `path' refers to a directory, append `/default.nix'. */
+    if (path.resolveSymlinks().lstat().type == InputAccessor::tDirectory)
+        return path + "default.nix";
+
+    return path;
+}
+
+
+Expr * EvalState::parseExprFromFile(const SourcePath & path)
+{
+    return parseExprFromFile(path, staticBaseEnv);
+}
+
+
+Expr * EvalState::parseExprFromFile(const SourcePath & path, std::shared_ptr<StaticEnv> & staticEnv)
+{
+    auto buffer = path.resolveSymlinks().readFile();
+    // readFile hopefully have left some extra space for terminators
+    buffer.append("\0\0", 2);
+    return parse(buffer.data(), buffer.size(), Pos::Origin(path), path.parent(), staticEnv);
+}
+
+
+Expr * EvalState::parseExprFromString(std::string s_, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv)
+{
+    auto s = make_ref<std::string>(std::move(s_));
+    s->append("\0\0", 2);
+    return parse(s->data(), s->size(), Pos::String{.source = s}, basePath, staticEnv);
+}
+
+
+Expr * EvalState::parseExprFromString(std::string s, const SourcePath & basePath)
+{
+    return parseExprFromString(std::move(s), basePath, staticBaseEnv);
+}
+
+
+Expr * EvalState::parseStdin()
+{
+    //Activity act(*logger, lvlTalkative, "parsing standard input");
+    auto buffer = drainFD(0);
+    // drainFD should have left some extra space for terminators
+    buffer.append("\0\0", 2);
+    auto s = make_ref<std::string>(std::move(buffer));
+    return parse(s->data(), s->size(), Pos::Stdin{.source = s}, rootPath(CanonPath::fromCwd()), staticBaseEnv);
+}
+
+
+SourcePath EvalState::findFile(const std::string_view path)
+{
+    return findFile(searchPath, path);
+}
+
+
+SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos)
+{
+    for (auto & i : searchPath.elements) {
+        auto suffixOpt = i.prefix.suffixIfPotentialMatch(path);
+
+        if (!suffixOpt) continue;
+        auto suffix = *suffixOpt;
+
+        auto rOpt = resolveSearchPathPath(i.path);
+        if (!rOpt) continue;
+        auto r = *rOpt;
+
+        Path res = suffix == "" ? r : concatStrings(r, "/", suffix);
+        if (pathExists(res)) return rootPath(CanonPath(canonPath(res)));
+    }
+
+    if (hasPrefix(path, "nix/"))
+        return {corepkgsFS, CanonPath(path.substr(3))};
+
+    debugThrow(ThrownError({
+        .msg = hintfmt(evalSettings.pureEval
+            ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)"
+            : "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)",
+            path),
+        .errPos = positions[pos]
+    }), 0, 0);
+}
+
+
+std::optional<std::string> EvalState::resolveSearchPathPath(const SearchPath::Path & value0, bool initAccessControl)
+{
+    auto & value = value0.s;
+    auto i = searchPathResolved.find(value);
+    if (i != searchPathResolved.end()) return i->second;
+
+    std::optional<std::string> res;
+
+    if (EvalSettings::isPseudoUrl(value)) {
+        try {
+            auto storePath = fetchers::downloadTarball(
+                store, EvalSettings::resolvePseudoUrl(value), "source", false).storePath;
+            res = { store->toRealPath(storePath) };
+        } catch (FileTransferError & e) {
+            logWarning({
+                .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value)
+            });
+        }
+    }
+
+    else if (hasPrefix(value, "flake:")) {
+        experimentalFeatureSettings.require(Xp::Flakes);
+        auto flakeRef = parseFlakeRef(value.substr(6), {}, true, false);
+        debug("fetching flake search path element '%s''", value);
+        auto storePath = flakeRef.resolve(store).fetchTree(store).first;
+        res = { store->toRealPath(storePath) };
+    }
+
+    else {
+        auto path = absPath(value);
+
+        /* Allow access to paths in the search path. */
+        if (initAccessControl) {
+            allowPath(path);
+            if (store->isInStore(path)) {
+                try {
+                    StorePathSet closure;
+                    store->computeFSClosure(store->toStorePath(path).first, closure);
+                    for (auto & p : closure)
+                        allowPath(p);
+                } catch (InvalidPath &) { }
+            }
+        }
+
+        if (pathExists(path))
+            res = { path };
+        else {
+            logWarning({
+                .msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", value)
+            });
+            res = std::nullopt;
+        }
+    }
+
+    if (res)
+        debug("resolved search path element '%s' to '%s'", value, *res);
+    else
+        debug("failed to resolve search path element '%s'", value);
+
+    searchPathResolved.emplace(value, res);
+    return res;
+}
+
+
+Expr * EvalState::parse(
+    char * text,
+    size_t length,
+    Pos::Origin origin,
+    const SourcePath & basePath,
+    std::shared_ptr<StaticEnv> & staticEnv)
+{
+    auto result = parseExprFromBuf(text, length, origin, basePath, symbols, positions, rootFS, exprSymbols);
+
+    result->bindVars(*this, staticEnv);
+
+    return result;
+}
+
+
 std::string ExternalValueBase::coerceToString(const Pos & pos, NixStringContext & context, bool copyMore, bool copyToStore) const
 {
     throw TypeError({
-        .msg = hintfmt("cannot coerce %1% to a string", showType())
+        .msg = hintfmt("cannot coerce %1% to a string: %2%", showType(), *this)
     });
 }
 

src/libexpr/eval.hh

@@ -84,6 +84,8 @@ struct PrimOp
     void check();
 };
 
+std::ostream & operator<<(std::ostream & output, PrimOp & primOp);
+
 /**
  * Info about a constant
  */
@@ -116,11 +118,6 @@ struct Constant
 struct Env
 {
     Env * up;
-    /**
-     * Number of of levels up to next `with` environment
-     */
-    unsigned short prevWith:14;
-    enum { Plain = 0, HasWithExpr, HasWithAttrs } type:2;
     Value * values[0];
 };
 
@@ -132,7 +129,7 @@ std::unique_ptr<ValMap> mapStaticEnvBindings(const SymbolTable & st, const Stati
 void copyContext(const Value & v, NixStringContext & context);
 
 
-std::string printValue(const EvalState & state, const Value & v);
+std::string printValue(EvalState & state, Value & v);
 std::ostream & operator << (std::ostream & os, const ValueType t);
 
 
@@ -147,7 +144,7 @@ struct RegexCache;
 std::shared_ptr<RegexCache> makeRegexCache();
 
 struct DebugTrace {
-    std::shared_ptr<AbstractPos> pos;
+    std::shared_ptr<Pos> pos;
     const Expr & expr;
     const Env & env;
     hintformat hint;
@@ -210,6 +207,8 @@ public:
         sPrefix,
         sOutputSpecified;
 
+    const Expr::AstSymbols exprSymbols;
+
     /**
      * If set, force copying files to the Nix store even if they
      * already exist there.
@@ -218,6 +217,11 @@ public:
 
     Bindings emptyBindings;
 
+    /**
+     * Empty list constant.
+     */
+    Value vEmptyList;
+
     /**
      * The accessor for the root filesystem.
      */
@@ -335,11 +339,6 @@ private:
 
     std::map<std::string, std::optional<std::string>> searchPathResolved;
 
-    /**
-     * Cache used by checkSourcePath().
-     */
-    std::unordered_map<Path, SourcePath> resolvedPaths;
-
     /**
      * Cache used by prim_match().
      */
@@ -465,8 +464,7 @@ public:
      */
     inline void forceValue(Value & v, const PosIdx pos);
 
-    template <typename Callable>
+    void tryFixupBlackHolePos(Value & v, PosIdx pos);
-    inline void forceValue(Value & v, Callable getPos);
 
     /**
      * Force a value, then recursively force list elements and
@@ -628,6 +626,11 @@ private:
         const SourcePath & basePath,
         std::shared_ptr<StaticEnv> & staticEnv);
 
+    /**
+     * Current Nix call stack depth, used with `max-call-depth` setting to throw stack overflow hopefully before we run out of system stack.
+     */
+    size_t callDepth = 0;
+
 public:
 
     /**
@@ -837,6 +840,11 @@ std::string showType(const Value & v);
  */
 SourcePath resolveExprPath(SourcePath path);
 
+/**
+ * Whether a URI is allowed, assuming restrictEval is enabled
+ */
+bool isAllowedURI(std::string_view uri, const Strings & allowedPaths);
+
 struct InvalidPathError : EvalError
 {
     Path path;

src/libexpr/flake/flakeref.cc

@@ -90,7 +90,7 @@ std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
         fragment = percentDecode(url.substr(fragmentStart+1));
     }
     if (pathEnd != std::string::npos && fragmentStart != std::string::npos) {
-        query = decodeQuery(url.substr(pathEnd+1, fragmentStart));
+        query = decodeQuery(url.substr(pathEnd+1, fragmentStart-pathEnd-1));
     }
 
     if (baseDir) {
@@ -190,7 +190,7 @@ std::optional<std::pair<FlakeRef, std::string>> parseFlakeIdRef(
 
     static std::regex flakeRegex(
         "((" + flakeIdRegexS + ")(?:/(?:" + refAndOrRevRegex + "))?)"
-        + "(?:#(" + queryRegex + "))?",
+        + "(?:#(" + fragmentRegex + "))?",
         std::regex::ECMAScript);
 
     if (std::regex_match(url, match, flakeRegex)) {

src/libexpr/flake/url-name.cc

@@ -0,0 +1,48 @@
+#include "url-name.hh"
+#include <regex>
+#include <iostream>
+
+namespace nix {
+
+static const std::string attributeNamePattern("[a-zA-Z0-9_-]+");
+static const std::regex lastAttributeRegex("(?:" + attributeNamePattern + "\\.)*(?!default)(" + attributeNamePattern +")(\\^.*)?");
+static const std::string pathSegmentPattern("[a-zA-Z0-9_-]+");
+static const std::regex lastPathSegmentRegex(".*/(" + pathSegmentPattern +")");
+static const std::regex secondPathSegmentRegex("(?:" + pathSegmentPattern + ")/(" + pathSegmentPattern +")(?:/.*)?");
+static const std::regex gitProviderRegex("github|gitlab|sourcehut");
+static const std::regex gitSchemeRegex("git($|\\+.*)");
+static const std::regex defaultOutputRegex(".*\\.default($|\\^.*)");
+
+std::optional<std::string> getNameFromURL(const ParsedURL & url)
+{
+    std::smatch match;
+
+    /* If there is a dir= argument, use its value */
+    if (url.query.count("dir") > 0)
+        return url.query.at("dir");
+
+    /* If the fragment isn't a "default" and contains two attribute elements, use the last one */
+    if (std::regex_match(url.fragment, match, lastAttributeRegex))
+        return match.str(1);
+
+    /* If this is a github/gitlab/sourcehut flake, use the repo name */
+    if (std::regex_match(url.scheme, gitProviderRegex) && std::regex_match(url.path, match, secondPathSegmentRegex))
+        return match.str(1);
+
+    /* If it is a regular git flake, use the directory name */
+    if (std::regex_match(url.scheme, gitSchemeRegex) && std::regex_match(url.path, match, lastPathSegmentRegex))
+        return match.str(1);
+
+    /* If everything failed but there is a non-default fragment, use it in full */
+    if (!url.fragment.empty() && !std::regex_match(url.fragment, defaultOutputRegex))
+        return url.fragment;
+
+    /* If there is no fragment, take the last element of the path */
+    if (std::regex_match(url.path, match, lastPathSegmentRegex))
+        return match.str(1);
+
+    /* If even that didn't work, the URL does not contain enough info to determine a useful name */
+    return {};
+}
+
+}

src/libexpr/flake/url-name.hh

@@ -0,0 +1,20 @@
+#include "url.hh"
+#include "url-parts.hh"
+#include "util.hh"
+#include "split.hh"
+
+namespace nix {
+
+/**
+ * Try to extract a reasonably unique and meaningful, human-readable
+ * name of a flake output from a parsed URL.
+ * When nullopt is returned, the callsite should use information available
+ * to it outside of the URL to determine a useful name.
+ * This is a heuristic approach intended for user interfaces.
+ * @return nullopt if the extracted name is not useful to identify a
+ * flake output, for example because it is empty or "default".
+ * Otherwise returns the extracted name.
+ */
+std::optional<std::string> getNameFromURL(const ParsedURL & url);
+
+}

src/libexpr/get-drvs.cc

@@ -11,13 +11,13 @@
 namespace nix {
 
 
-DrvInfo::DrvInfo(EvalState & state, std::string attrPath, Bindings * attrs)
+PackageInfo::PackageInfo(EvalState & state, std::string attrPath, Bindings * attrs)
     : state(&state), attrs(attrs), attrPath(std::move(attrPath))
 {
 }
 
 
-DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs)
+PackageInfo::PackageInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs)
     : state(&state), attrs(nullptr), attrPath("")
 {
     auto [drvPath, selectedOutputs] = parsePathWithOutputs(*store, drvPathWithOutputs);
@@ -45,7 +45,7 @@ DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPat
 }
 
 
-std::string DrvInfo::queryName() const
+std::string PackageInfo::queryName() const
 {
     if (name == "" && attrs) {
         auto i = attrs->find(state->sName);
@@ -56,7 +56,7 @@ std::string DrvInfo::queryName() const
 }
 
 
-std::string DrvInfo::querySystem() const
+std::string PackageInfo::querySystem() const
 {
     if (system == "" && attrs) {
         auto i = attrs->find(state->sSystem);
@@ -66,7 +66,7 @@ std::string DrvInfo::querySystem() const
 }
 
 
-std::optional<StorePath> DrvInfo::queryDrvPath() const
+std::optional<StorePath> PackageInfo::queryDrvPath() const
 {
     if (!drvPath && attrs) {
         Bindings::iterator i = attrs->find(state->sDrvPath);
@@ -80,7 +80,7 @@ std::optional<StorePath> DrvInfo::queryDrvPath() const
 }
 
 
-StorePath DrvInfo::requireDrvPath() const
+StorePath PackageInfo::requireDrvPath() const
 {
     if (auto drvPath = queryDrvPath())
         return *drvPath;
@@ -88,7 +88,7 @@ StorePath DrvInfo::requireDrvPath() const
 }
 
 
-StorePath DrvInfo::queryOutPath() const
+StorePath PackageInfo::queryOutPath() const
 {
     if (!outPath && attrs) {
         Bindings::iterator i = attrs->find(state->sOutPath);
@@ -102,7 +102,7 @@ StorePath DrvInfo::queryOutPath() const
 }
 
 
-DrvInfo::Outputs DrvInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall)
+PackageInfo::Outputs PackageInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall)
 {
     if (outputs.empty()) {
         /* Get the ‘outputs’ list. */
@@ -164,7 +164,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall
 }
 
 
-std::string DrvInfo::queryOutputName() const
+std::string PackageInfo::queryOutputName() const
 {
     if (outputName == "" && attrs) {
         Bindings::iterator i = attrs->find(state->sOutputName);
@@ -174,7 +174,7 @@ std::string DrvInfo::queryOutputName() const
 }
 
 
-Bindings * DrvInfo::getMeta()
+Bindings * PackageInfo::getMeta()
 {
     if (meta) return meta;
     if (!attrs) return 0;
@@ -186,7 +186,7 @@ Bindings * DrvInfo::getMeta()
 }
 
 
-StringSet DrvInfo::queryMetaNames()
+StringSet PackageInfo::queryMetaNames()
 {
     StringSet res;
     if (!getMeta()) return res;
@@ -196,9 +196,9 @@ StringSet DrvInfo::queryMetaNames()
 }
 
 
-bool DrvInfo::checkMeta(Value & v)
+bool PackageInfo::checkMeta(Value & v)
 {
-    state->forceValue(v, [&]() { return v.determinePos(noPos); });
+    state->forceValue(v, v.determinePos(noPos));
     if (v.type() == nList) {
         for (auto elem : v.listItems())
             if (!checkMeta(*elem)) return false;
@@ -216,7 +216,7 @@ bool DrvInfo::checkMeta(Value & v)
 }
 
 
-Value * DrvInfo::queryMeta(const std::string & name)
+Value * PackageInfo::queryMeta(const std::string & name)
 {
     if (!getMeta()) return 0;
     Bindings::iterator a = meta->find(state->symbols.create(name));
@@ -225,7 +225,7 @@ Value * DrvInfo::queryMeta(const std::string & name)
 }
 
 
-std::string DrvInfo::queryMetaString(const std::string & name)
+std::string PackageInfo::queryMetaString(const std::string & name)
 {
     Value * v = queryMeta(name);
     if (!v || v->type() != nString) return "";
@@ -233,7 +233,7 @@ std::string DrvInfo::queryMetaString(const std::string & name)
 }
 
 
-NixInt DrvInfo::queryMetaInt(const std::string & name, NixInt def)
+NixInt PackageInfo::queryMetaInt(const std::string & name, NixInt def)
 {
     Value * v = queryMeta(name);
     if (!v) return def;
@@ -247,7 +247,7 @@ NixInt DrvInfo::queryMetaInt(const std::string & name, NixInt def)
     return def;
 }
 
-NixFloat DrvInfo::queryMetaFloat(const std::string & name, NixFloat def)
+NixFloat PackageInfo::queryMetaFloat(const std::string & name, NixFloat def)
 {
     Value * v = queryMeta(name);
     if (!v) return def;
@@ -262,7 +262,7 @@ NixFloat DrvInfo::queryMetaFloat(const std::string & name, NixFloat def)
 }
 
 
-bool DrvInfo::queryMetaBool(const std::string & name, bool def)
+bool PackageInfo::queryMetaBool(const std::string & name, bool def)
 {
     Value * v = queryMeta(name);
     if (!v) return def;
@@ -277,7 +277,7 @@ bool DrvInfo::queryMetaBool(const std::string & name, bool def)
 }
 
 
-void DrvInfo::setMeta(const std::string & name, Value * v)
+void PackageInfo::setMeta(const std::string & name, Value * v)
 {
     getMeta();
     auto attrs = state->buildBindings(1 + (meta ? meta->size() : 0));
@@ -300,18 +300,18 @@ typedef std::set<Bindings *> Done;
    The result boolean indicates whether it makes sense
    for the caller to recursively search for derivations in `v'. */
 static bool getDerivation(EvalState & state, Value & v,
-    const std::string & attrPath, DrvInfos & drvs, Done & done,
+    const std::string & attrPath, PackageInfos & drvs, Done & done,
     bool ignoreAssertionFailures)
 {
     try {
-        state.forceValue(v, [&]() { return v.determinePos(noPos); });
+        state.forceValue(v, v.determinePos(noPos));
         if (!state.isDerivation(v)) return true;
 
         /* Remove spurious duplicates (e.g., a set like `rec { x =
            derivation {...}; y = x;}'. */
         if (!done.insert(v.attrs).second) return false;
 
-        DrvInfo drv(state, attrPath, v.attrs);
+        PackageInfo drv(state, attrPath, v.attrs);
 
         drv.queryName();
 
@@ -326,11 +326,11 @@ static bool getDerivation(EvalState & state, Value & v,
 }
 
 
-std::optional<DrvInfo> getDerivation(EvalState & state, Value & v,
+std::optional<PackageInfo> getDerivation(EvalState & state, Value & v,
     bool ignoreAssertionFailures)
 {
     Done done;
-    DrvInfos drvs;
+    PackageInfos drvs;
     getDerivation(state, v, "", drvs, done, ignoreAssertionFailures);
     if (drvs.size() != 1) return {};
     return std::move(drvs.front());
@@ -348,7 +348,7 @@ static std::regex attrRegex("[A-Za-z_][A-Za-z0-9-_+]*");
 
 static void getDerivations(EvalState & state, Value & vIn,
     const std::string & pathPrefix, Bindings & autoArgs,
-    DrvInfos & drvs, Done & done,
+    PackageInfos & drvs, Done & done,
     bool ignoreAssertionFailures)
 {
     Value v;
@@ -401,7 +401,7 @@ static void getDerivations(EvalState & state, Value & vIn,
 
 
 void getDerivations(EvalState & state, Value & v, const std::string & pathPrefix,
-    Bindings & autoArgs, DrvInfos & drvs, bool ignoreAssertionFailures)
+    Bindings & autoArgs, PackageInfos & drvs, bool ignoreAssertionFailures)
 {
     Done done;
     getDerivations(state, v, pathPrefix, autoArgs, drvs, done, ignoreAssertionFailures);

src/libexpr/get-drvs.hh

@@ -10,8 +10,10 @@
 
 namespace nix {
 
-
+/**
-struct DrvInfo
+ * A "parsed" package attribute set.
+ */
+struct PackageInfo
 {
 public:
     typedef std::map<std::string, std::optional<StorePath>> Outputs;
@@ -43,9 +45,9 @@ public:
      */
     std::string attrPath;
 
-    DrvInfo(EvalState & state) : state(&state) { };
+    PackageInfo(EvalState & state) : state(&state) { };
-    DrvInfo(EvalState & state, std::string attrPath, Bindings * attrs);
+    PackageInfo(EvalState & state, std::string attrPath, Bindings * attrs);
-    DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs);
+    PackageInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs);
 
     std::string queryName() const;
     std::string querySystem() const;
@@ -82,21 +84,21 @@ public:
 
 
 #if HAVE_BOEHMGC
-typedef std::list<DrvInfo, traceable_allocator<DrvInfo>> DrvInfos;
+typedef std::list<PackageInfo, traceable_allocator<PackageInfo>> PackageInfos;
 #else
-typedef std::list<DrvInfo> DrvInfos;
+typedef std::list<PackageInfo> PackageInfos;
 #endif
 
 
 /**
- * If value `v` denotes a derivation, return a DrvInfo object
+ * If value `v` denotes a derivation, return a PackageInfo object
  * describing it. Otherwise return nothing.
  */
-std::optional<DrvInfo> getDerivation(EvalState & state,
+std::optional<PackageInfo> getDerivation(EvalState & state,
     Value & v, bool ignoreAssertionFailures);
 
 void getDerivations(EvalState & state, Value & v, const std::string & pathPrefix,
-    Bindings & autoArgs, DrvInfos & drvs,
+    Bindings & autoArgs, PackageInfos & drvs,
     bool ignoreAssertionFailures);
 
 

src/libexpr/lexer.l

@@ -1,4 +1,5 @@
 %option reentrant bison-bridge bison-locations
+%option align
 %option noyywrap
 %option never-interactive
 %option stack
@@ -28,15 +29,7 @@ using namespace nix;
 
 namespace nix {
 
-static inline PosIdx makeCurPos(const YYLTYPE & loc, ParseData * data)
+#define CUR_POS state->at(*yylloc)
-{
-    return data->state.positions.add(data->origin, loc.first_line, loc.first_column);
-}
-
-#define CUR_POS makeCurPos(*yylloc, data)
-
-// backup to recover from yyless(0)
-thread_local YYLTYPE prev_yylloc;
 
 static void initLoc(YYLTYPE * loc)
 {
@@ -46,7 +39,7 @@ static void initLoc(YYLTYPE * loc)
 
 static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
 {
-    prev_yylloc = *loc;
+    loc->stash();
 
     loc->first_line = loc->last_line;
     loc->first_column = loc->last_column;
@@ -132,7 +125,7 @@ else        { return ELSE; }
 assert      { return ASSERT; }
 with        { return WITH; }
 let         { return LET; }
-in          { return IN; }
+in          { return IN_KW; }
 rec         { return REC; }
 inherit     { return INHERIT; }
 or          { return OR_KW; }
@@ -155,19 +148,19 @@ or          { return OR_KW; }
               } catch (const boost::bad_lexical_cast &) {
                   throw ParseError({
                       .msg = hintfmt("invalid integer '%1%'", yytext),
-                      .errPos = data->state.positions[CUR_POS],
+                      .errPos = state->positions[CUR_POS],
                   });
               }
-              return INT;
+              return INT_LIT;
             }
 {FLOAT}     { errno = 0;
               yylval->nf = strtod(yytext, 0);
               if (errno != 0)
                   throw ParseError({
                       .msg = hintfmt("invalid float '%1%'", yytext),
-                      .errPos = data->state.positions[CUR_POS],
+                      .errPos = state->positions[CUR_POS],
                   });
-              return FLOAT;
+              return FLOAT_LIT;
             }
 
 \$\{        { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
@@ -188,7 +181,7 @@ or          { return OR_KW; }
                 /* It is impossible to match strings ending with '$' with one
                    regex because trailing contexts are only valid at the end
                    of a rule. (A sane but undocumented limitation.) */
-                yylval->str = unescapeStr(data->symbols, yytext, yyleng);
+                yylval->str = unescapeStr(state->symbols, yytext, yyleng);
                 return STR;
               }
 <STRING>\$\{  { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
@@ -216,7 +209,7 @@ or          { return OR_KW; }
                    return IND_STR;
                  }
 <IND_STRING>\'\'\\{ANY} {
-                   yylval->str = unescapeStr(data->symbols, yytext + 2, yyleng - 2);
+                   yylval->str = unescapeStr(state->symbols, yytext + 2, yyleng - 2);
                    return IND_STR;
                  }
 <IND_STRING>\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
@@ -230,7 +223,7 @@ or          { return OR_KW; }
 {HPATH_START}\$\{ {
   PUSH_STATE(PATH_START);
   yyless(0);
-  *yylloc = prev_yylloc;
+  yylloc->unstash();
 }
 
 <PATH_START>{PATH_SEG} {
@@ -286,7 +279,7 @@ or          { return OR_KW; }
      context (it may be ')', ';', or something of that sort) */
   POP_STATE();
   yyless(0);
-  *yylloc = prev_yylloc;
+  yylloc->unstash();
   return PATH_END;
 }
 
@@ -294,7 +287,7 @@ or          { return OR_KW; }
 <INPATH_SLASH><<EOF>> {
   throw ParseError({
       .msg = hintfmt("path has a trailing slash"),
-      .errPos = data->state.positions[CUR_POS],
+      .errPos = state->positions[CUR_POS],
   });
 }
 

src/libexpr/local.mk

@@ -16,9 +16,9 @@ libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/lib
 
 libexpr_LIBS = libutil libstore libfetchers
 
-libexpr_LDFLAGS += -lboost_context -pthread
+libexpr_LDFLAGS += -lboost_context $(THREAD_LDFLAGS)
 ifdef HOST_LINUX
- libexpr_LDFLAGS += -ldl
+  libexpr_LDFLAGS += -ldl
 endif
 
 # The dependency on libgc must be propagated (i.e. meaning that

src/libexpr/nixexpr.cc

@@ -9,57 +9,9 @@
 
 namespace nix {
 
-struct PosAdapter : AbstractPos
+unsigned long Expr::nrExprs = 0;
-{
-    Pos::Origin origin;
 
-    PosAdapter(Pos::Origin origin)
+ExprBlackHole eBlackHole;
-        : origin(std::move(origin))
-    {
-    }
-
-    std::optional<std::string> getSource() const override
-    {
-        return std::visit(overloaded {
-            [](const Pos::none_tag &) -> std::optional<std::string> {
-                return std::nullopt;
-            },
-            [](const Pos::Stdin & s) -> std::optional<std::string> {
-                // Get rid of the null terminators added by the parser.
-                return std::string(s.source->c_str());
-            },
-            [](const Pos::String & s) -> std::optional<std::string> {
-                // Get rid of the null terminators added by the parser.
-                return std::string(s.source->c_str());
-            },
-            [](const SourcePath & path) -> std::optional<std::string> {
-                try {
-                    return path.readFile();
-                } catch (Error &) {
-                    return std::nullopt;
-                }
-            }
-        }, origin);
-    }
-
-    void print(std::ostream & out) const override
-    {
-        std::visit(overloaded {
-            [&](const Pos::none_tag &) { out << "«none»"; },
-            [&](const Pos::Stdin &) { out << "«stdin»"; },
-            [&](const Pos::String & s) { out << "«string»"; },
-            [&](const SourcePath & path) { out << path; }
-        }, origin);
-    }
-};
-
-Pos::operator std::shared_ptr<AbstractPos>() const
-{
-    auto pos = std::make_shared<PosAdapter>(origin);
-    pos->line = line;
-    pos->column = column;
-    return pos;
-}
 
 // FIXME: remove, because *symbols* are abstract and do not have a single
 //        textual representation; see printIdentifier()
@@ -266,17 +218,6 @@ void ExprPos::show(const SymbolTable & symbols, std::ostream & str) const
 }
 
 
-std::ostream & operator << (std::ostream & str, const Pos & pos)
-{
-    if (auto pos2 = (std::shared_ptr<AbstractPos>) pos) {
-        str << *pos2;
-    } else
-        str << "undefined position";
-
-    return str;
-}
-
-
 std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath)
 {
     std::ostringstream out;
@@ -331,6 +272,8 @@ void ExprVar::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
     if (es.debugRepl)
         es.exprEnvs.insert(std::make_pair(this, env));
 
+    fromWith = nullptr;
+
     /* Check whether the variable appears in the environment.  If so,
        set its level and displacement. */
     const StaticEnv * curEnv;
@@ -342,7 +285,6 @@ void ExprVar::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
         } else {
             auto i = curEnv->find(name);
             if (i != curEnv->vars.end()) {
-                fromWith = false;
                 this->level = level;
                 displ = i->second;
                 return;
@@ -358,7 +300,8 @@ void ExprVar::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
             .msg = hintfmt("undefined variable '%1%'", es.symbols[name]),
             .errPos = es.positions[pos]
         });
-    fromWith = true;
+    for (auto * e = env.get(); e && !fromWith; e = e->up)
+        fromWith = e->isWith;
     this->level = withLevel;
 }
 
@@ -391,7 +334,7 @@ void ExprAttrs::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv>
         es.exprEnvs.insert(std::make_pair(this, env));
 
     if (recursive) {
-        auto newEnv = std::make_shared<StaticEnv>(false, env.get(), recursive ? attrs.size() : 0);
+        auto newEnv = std::make_shared<StaticEnv>(nullptr, env.get(), recursive ? attrs.size() : 0);
 
         Displacement displ = 0;
         for (auto & i : attrs)
@@ -433,7 +376,7 @@ void ExprLambda::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv>
         es.exprEnvs.insert(std::make_pair(this, env));
 
     auto newEnv = std::make_shared<StaticEnv>(
-        false, env.get(),
+        nullptr, env.get(),
         (hasFormals() ? formals->formals.size() : 0) +
         (!arg ? 0 : 1));
 
@@ -469,7 +412,7 @@ void ExprLet::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
     if (es.debugRepl)
         es.exprEnvs.insert(std::make_pair(this, env));
 
-    auto newEnv = std::make_shared<StaticEnv>(false, env.get(), attrs->attrs.size());
+    auto newEnv = std::make_shared<StaticEnv>(nullptr, env.get(), attrs->attrs.size());
 
     Displacement displ = 0;
     for (auto & i : attrs->attrs)
@@ -488,6 +431,10 @@ void ExprWith::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
     if (es.debugRepl)
         es.exprEnvs.insert(std::make_pair(this, env));
 
+    parentWith = nullptr;
+    for (auto * e = env.get(); e && !parentWith; e = e->up)
+        parentWith = e->isWith;
+
     /* Does this `with' have an enclosing `with'?  If so, record its
        level so that `lookupVar' can look up variables in the previous
        `with' if this one doesn't contain the desired attribute. */
@@ -504,7 +451,7 @@ void ExprWith::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
         es.exprEnvs.insert(std::make_pair(this, env));
 
     attrs->bindVars(es, env);
-    auto newEnv = std::make_shared<StaticEnv>(true, env.get());
+    auto newEnv = std::make_shared<StaticEnv>(this, env.get());
     body->bindVars(es, newEnv);
 }
 

src/libexpr/nixexpr.hh

@@ -8,6 +8,7 @@
 #include "symbol-table.hh"
 #include "error.hh"
 #include "chunked-vector.hh"
+#include "position.hh"
 
 namespace nix {
 
@@ -21,25 +22,11 @@ MakeError(TypeError, EvalError);
 MakeError(UndefinedVarError, Error);
 MakeError(MissingArgumentError, EvalError);
 
-/**
+class InfiniteRecursionError : public EvalError
- * Position objects.
- */
-struct Pos
 {
-    uint32_t line;
+    friend class EvalState;
-    uint32_t column;
+public:
-
+    using EvalError::EvalError;
-    struct none_tag { };
-    struct Stdin { ref<std::string> source; };
-    struct String { ref<std::string> source; };
-
-    typedef std::variant<none_tag, Stdin, String, SourcePath> Origin;
-
-    Origin origin;
-
-    explicit operator bool() const { return line > 0; }
-
-    operator std::shared_ptr<AbstractPos>() const;
 };
 
 class PosIdx {
@@ -74,7 +61,7 @@ public:
         mutable uint32_t idx = std::numeric_limits<uint32_t>::max();
 
         // Used for searching in PosTable::[].
-        explicit Origin(uint32_t idx): idx(idx), origin{Pos::none_tag()} {}
+        explicit Origin(uint32_t idx): idx(idx), origin{std::monostate()} {}
 
     public:
         const Pos::Origin origin;
@@ -125,12 +112,11 @@ public:
 
 inline PosIdx noPos = {};
 
-std::ostream & operator << (std::ostream & str, const Pos & pos);
-
 
 struct Env;
 struct Value;
 class EvalState;
+struct ExprWith;
 struct StaticEnv;
 
 
@@ -154,6 +140,11 @@ std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath)
 
 struct Expr
 {
+    struct AstSymbols {
+        Symbol sub, lessThan, mul, div, or_, findFile, nixPath, body;
+    };
+
+
     static unsigned long nrExprs;
     Expr() {
         nrExprs++;
@@ -219,8 +210,11 @@ struct ExprVar : Expr
     Symbol name;
 
     /* Whether the variable comes from an environment (e.g. a rec, let
-       or function argument) or from a "with". */
+       or function argument) or from a "with".
-    bool fromWith;
+
+       `nullptr`: Not from a `with`.
+       Valid pointer: the nearest, innermost `with` expression to query first. */
+    ExprWith * fromWith;
 
     /* In the former case, the value is obtained by going `level`
        levels up from the current environment and getting the
@@ -292,6 +286,7 @@ struct ExprList : Expr
     std::vector<Expr *> elems;
     ExprList() { };
     COMMON_METHODS
+    Value * maybeThunk(EvalState & state, Env & env) override;
 
     PosIdx getPos() const override
     {
@@ -378,6 +373,7 @@ struct ExprWith : Expr
     PosIdx pos;
     Expr * attrs, * body;
     size_t prevWith;
+    ExprWith * parentWith;
     ExprWith(const PosIdx & pos, Expr * attrs, Expr * body) : pos(pos), attrs(attrs), body(body) { };
     PosIdx getPos() const override { return pos; }
     COMMON_METHODS
@@ -455,20 +451,30 @@ struct ExprPos : Expr
     COMMON_METHODS
 };
 
+/* only used to mark thunks as black holes. */
+struct ExprBlackHole : Expr
+{
+    void show(const SymbolTable & symbols, std::ostream & str) const override {}
+    void eval(EvalState & state, Env & env, Value & v) override;
+    void bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env) override {}
+};
+
+extern ExprBlackHole eBlackHole;
+
 
 /* Static environments are used to map variable names onto (level,
    displacement) pairs used to obtain the value of the variable at
    runtime. */
 struct StaticEnv
 {
-    bool isWith;
+    ExprWith * isWith;
     const StaticEnv * up;
 
     // Note: these must be in sorted order.
     typedef std::vector<std::pair<Symbol, Displacement>> Vars;
     Vars vars;
 
-    StaticEnv(bool isWith, const StaticEnv * up, size_t expectedSize = 0) : isWith(isWith), up(up) {
+    StaticEnv(ExprWith * isWith, const StaticEnv * up, size_t expectedSize = 0) : isWith(isWith), up(up) {
         vars.reserve(expectedSize);
     };
 

src/libexpr/parser-state.hh

@@ -0,0 +1,271 @@
+#pragma once
+///@file
+
+#include "eval.hh"
+
+namespace nix {
+
+/**
+ * @note Storing a C-style `char *` and `size_t` allows us to avoid
+ * having to define the special members that using string_view here
+ * would implicitly delete.
+ */
+struct StringToken
+{
+    const char * p;
+    size_t l;
+    bool hasIndentation;
+    operator std::string_view() const { return {p, l}; }
+};
+
+struct ParserLocation
+{
+    int first_line, first_column;
+    int last_line, last_column;
+
+    // backup to recover from yyless(0)
+    int stashed_first_line, stashed_first_column;
+    int stashed_last_line, stashed_last_column;
+
+    void stash() {
+        stashed_first_line = first_line;
+        stashed_first_column = first_column;
+        stashed_last_line = last_line;
+        stashed_last_column = last_column;
+    }
+
+    void unstash() {
+        first_line = stashed_first_line;
+        first_column = stashed_first_column;
+        last_line = stashed_last_line;
+        last_column = stashed_last_column;
+    }
+};
+
+struct ParserState
+{
+    SymbolTable & symbols;
+    PosTable & positions;
+    Expr * result;
+    SourcePath basePath;
+    PosTable::Origin origin;
+    const ref<InputAccessor> rootFS;
+    const Expr::AstSymbols & s;
+
+    void dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos);
+    void dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos);
+    void addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * e, const PosIdx pos);
+    Formals * validateFormals(Formals * formals, PosIdx pos = noPos, Symbol arg = {});
+    Expr * stripIndentation(const PosIdx pos,
+        std::vector<std::pair<PosIdx, std::variant<Expr *, StringToken>>> && es);
+    PosIdx at(const ParserLocation & loc);
+};
+
+inline void ParserState::dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos)
+{
+    throw ParseError({
+         .msg = hintfmt("attribute '%1%' already defined at %2%",
+             showAttrPath(symbols, attrPath), positions[prevPos]),
+         .errPos = positions[pos]
+    });
+}
+
+inline void ParserState::dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos)
+{
+    throw ParseError({
+        .msg = hintfmt("attribute '%1%' already defined at %2%", symbols[attr], positions[prevPos]),
+        .errPos = positions[pos]
+    });
+}
+
+inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * e, const PosIdx pos)
+{
+    AttrPath::iterator i;
+    // All attrpaths have at least one attr
+    assert(!attrPath.empty());
+    // Checking attrPath validity.
+    // ===========================
+    for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) {
+        if (i->symbol) {
+            ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol);
+            if (j != attrs->attrs.end()) {
+                if (!j->second.inherited) {
+                    ExprAttrs * attrs2 = dynamic_cast<ExprAttrs *>(j->second.e);
+                    if (!attrs2) dupAttr(attrPath, pos, j->second.pos);
+                    attrs = attrs2;
+                } else
+                    dupAttr(attrPath, pos, j->second.pos);
+            } else {
+                ExprAttrs * nested = new ExprAttrs;
+                attrs->attrs[i->symbol] = ExprAttrs::AttrDef(nested, pos);
+                attrs = nested;
+            }
+        } else {
+            ExprAttrs *nested = new ExprAttrs;
+            attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, nested, pos));
+            attrs = nested;
+        }
+    }
+    // Expr insertion.
+    // ==========================
+    if (i->symbol) {
+        ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol);
+        if (j != attrs->attrs.end()) {
+            // This attr path is already defined. However, if both
+            // e and the expr pointed by the attr path are two attribute sets,
+            // we want to merge them.
+            // Otherwise, throw an error.
+            auto ae = dynamic_cast<ExprAttrs *>(e);
+            auto jAttrs = dynamic_cast<ExprAttrs *>(j->second.e);
+            if (jAttrs && ae) {
+                for (auto & ad : ae->attrs) {
+                    auto j2 = jAttrs->attrs.find(ad.first);
+                    if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error.
+                        dupAttr(ad.first, j2->second.pos, ad.second.pos);
+                    jAttrs->attrs.emplace(ad.first, ad.second);
+                }
+                jAttrs->dynamicAttrs.insert(jAttrs->dynamicAttrs.end(), ae->dynamicAttrs.begin(), ae->dynamicAttrs.end());
+            } else {
+                dupAttr(attrPath, pos, j->second.pos);
+            }
+        } else {
+            // This attr path is not defined. Let's create it.
+            attrs->attrs.emplace(i->symbol, ExprAttrs::AttrDef(e, pos));
+            e->setName(i->symbol);
+        }
+    } else {
+        attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, e, pos));
+    }
+}
+
+inline Formals * ParserState::validateFormals(Formals * formals, PosIdx pos, Symbol arg)
+{
+    std::sort(formals->formals.begin(), formals->formals.end(),
+        [] (const auto & a, const auto & b) {
+            return std::tie(a.name, a.pos) < std::tie(b.name, b.pos);
+        });
+
+    std::optional<std::pair<Symbol, PosIdx>> duplicate;
+    for (size_t i = 0; i + 1 < formals->formals.size(); i++) {
+        if (formals->formals[i].name != formals->formals[i + 1].name)
+            continue;
+        std::pair thisDup{formals->formals[i].name, formals->formals[i + 1].pos};
+        duplicate = std::min(thisDup, duplicate.value_or(thisDup));
+    }
+    if (duplicate)
+        throw ParseError({
+            .msg = hintfmt("duplicate formal function argument '%1%'", symbols[duplicate->first]),
+            .errPos = positions[duplicate->second]
+        });
+
+    if (arg && formals->has(arg))
+        throw ParseError({
+            .msg = hintfmt("duplicate formal function argument '%1%'", symbols[arg]),
+            .errPos = positions[pos]
+        });
+
+    return formals;
+}
+
+inline Expr * ParserState::stripIndentation(const PosIdx pos,
+    std::vector<std::pair<PosIdx, std::variant<Expr *, StringToken>>> && es)
+{
+    if (es.empty()) return new ExprString("");
+
+    /* Figure out the minimum indentation.  Note that by design
+       whitespace-only final lines are not taken into account.  (So
+       the " " in "\n ''" is ignored, but the " " in "\n foo''" is.) */
+    bool atStartOfLine = true; /* = seen only whitespace in the current line */
+    size_t minIndent = 1000000;
+    size_t curIndent = 0;
+    for (auto & [i_pos, i] : es) {
+        auto * str = std::get_if<StringToken>(&i);
+        if (!str || !str->hasIndentation) {
+            /* Anti-quotations and escaped characters end the current start-of-line whitespace. */
+            if (atStartOfLine) {
+                atStartOfLine = false;
+                if (curIndent < minIndent) minIndent = curIndent;
+            }
+            continue;
+        }
+        for (size_t j = 0; j < str->l; ++j) {
+            if (atStartOfLine) {
+                if (str->p[j] == ' ')
+                    curIndent++;
+                else if (str->p[j] == '\n') {
+                    /* Empty line, doesn't influence minimum
+                       indentation. */
+                    curIndent = 0;
+                } else {
+                    atStartOfLine = false;
+                    if (curIndent < minIndent) minIndent = curIndent;
+                }
+            } else if (str->p[j] == '\n') {
+                atStartOfLine = true;
+                curIndent = 0;
+            }
+        }
+    }
+
+    /* Strip spaces from each line. */
+    auto * es2 = new std::vector<std::pair<PosIdx, Expr *>>;
+    atStartOfLine = true;
+    size_t curDropped = 0;
+    size_t n = es.size();
+    auto i = es.begin();
+    const auto trimExpr = [&] (Expr * e) {
+        atStartOfLine = false;
+        curDropped = 0;
+        es2->emplace_back(i->first, e);
+    };
+    const auto trimString = [&] (const StringToken & t) {
+        std::string s2;
+        for (size_t j = 0; j < t.l; ++j) {
+            if (atStartOfLine) {
+                if (t.p[j] == ' ') {
+                    if (curDropped++ >= minIndent)
+                        s2 += t.p[j];
+                }
+                else if (t.p[j] == '\n') {
+                    curDropped = 0;
+                    s2 += t.p[j];
+                } else {
+                    atStartOfLine = false;
+                    curDropped = 0;
+                    s2 += t.p[j];
+                }
+            } else {
+                s2 += t.p[j];
+                if (t.p[j] == '\n') atStartOfLine = true;
+            }
+        }
+
+        /* Remove the last line if it is empty and consists only of
+           spaces. */
+        if (n == 1) {
+            std::string::size_type p = s2.find_last_of('\n');
+            if (p != std::string::npos && s2.find_first_not_of(' ', p + 1) == std::string::npos)
+                s2 = std::string(s2, 0, p + 1);
+        }
+
+        es2->emplace_back(i->first, new ExprString(std::move(s2)));
+    };
+    for (; i != es.end(); ++i, --n) {
+        std::visit(overloaded { trimExpr, trimString }, i->second);
+    }
+
+    /* If this is a single string, then don't do a concatenation. */
+    if (es2->size() == 1 && dynamic_cast<ExprString *>((*es2)[0].second)) {
+        auto *const result = (*es2)[0].second;
+        delete es2;
+        return result;
+    }
+    return new ExprConcatStrings(pos, true, es2);
+}
+
+inline PosIdx ParserState::at(const ParserLocation & loc)
+{
+    return positions.add(origin, loc.first_line, loc.first_column);
+}
+
+}

src/libexpr/parser.y

@@ -5,9 +5,9 @@
 %defines
 /* %no-lines */
 %parse-param { void * scanner }
-%parse-param { nix::ParseData * data }
+%parse-param { nix::ParserState * state }
 %lex-param { void * scanner }
-%lex-param { nix::ParseData * data }
+%lex-param { nix::ParserState * state }
 %expect 1
 %expect-rr 1
 
@@ -18,6 +18,7 @@
 
 #include <variant>
 
+#include "finally.hh"
 #include "util.hh"
 #include "users.hh"
 
@@ -25,38 +26,26 @@
 #include "eval.hh"
 #include "eval-settings.hh"
 #include "globals.hh"
+#include "parser-state.hh"
+
+#define YYLTYPE ::nix::ParserLocation
+#define YY_DECL int yylex \
+    (YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParserState * state)
 
 namespace nix {
 
-    struct ParseData
+Expr * parseExprFromBuf(
-    {
+    char * text,
-        EvalState & state;
+    size_t length,
-        SymbolTable & symbols;
+    Pos::Origin origin,
-        Expr * result;
+    const SourcePath & basePath,
-        SourcePath basePath;
+    SymbolTable & symbols,
-        PosTable::Origin origin;
+    PosTable & positions,
-        std::optional<ErrorInfo> error;
+    const ref<InputAccessor> rootFS,
-    };
+    const Expr::AstSymbols & astSymbols);
-
-    struct ParserFormals {
-        std::vector<Formal> formals;
-        bool ellipsis = false;
-    };
 
 }
 
-// using C a struct allows us to avoid having to define the special
-// members that using string_view here would implicitly delete.
-struct StringToken {
-  const char * p;
-  size_t l;
-  bool hasIndentation;
-  operator std::string_view() const { return {p, l}; }
-};
-
-#define YY_DECL int yylex \
-    (YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParseData * data)
-
 #endif
 
 }
@@ -70,240 +59,15 @@ YY_DECL;
 
 using namespace nix;
 
-
+#define CUR_POS state->at(*yylocp)
-namespace nix {
 
 
-static void dupAttr(const EvalState & state, const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos)
+void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * error)
 {
     throw ParseError({
-         .msg = hintfmt("attribute '%1%' already defined at %2%",
-             showAttrPath(state.symbols, attrPath), state.positions[prevPos]),
-         .errPos = state.positions[pos]
-    });
-}
-
-static void dupAttr(const EvalState & state, Symbol attr, const PosIdx pos, const PosIdx prevPos)
-{
-    throw ParseError({
-        .msg = hintfmt("attribute '%1%' already defined at %2%", state.symbols[attr], state.positions[prevPos]),
-        .errPos = state.positions[pos]
-    });
-}
-
-
-static void addAttr(ExprAttrs * attrs, AttrPath && attrPath,
-    Expr * e, const PosIdx pos, const nix::EvalState & state)
-{
-    AttrPath::iterator i;
-    // All attrpaths have at least one attr
-    assert(!attrPath.empty());
-    // Checking attrPath validity.
-    // ===========================
-    for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) {
-        if (i->symbol) {
-            ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol);
-            if (j != attrs->attrs.end()) {
-                if (!j->second.inherited) {
-                    ExprAttrs * attrs2 = dynamic_cast<ExprAttrs *>(j->second.e);
-                    if (!attrs2) dupAttr(state, attrPath, pos, j->second.pos);
-                    attrs = attrs2;
-                } else
-                    dupAttr(state, attrPath, pos, j->second.pos);
-            } else {
-                ExprAttrs * nested = new ExprAttrs;
-                attrs->attrs[i->symbol] = ExprAttrs::AttrDef(nested, pos);
-                attrs = nested;
-            }
-        } else {
-            ExprAttrs *nested = new ExprAttrs;
-            attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, nested, pos));
-            attrs = nested;
-        }
-    }
-    // Expr insertion.
-    // ==========================
-    if (i->symbol) {
-        ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol);
-        if (j != attrs->attrs.end()) {
-            // This attr path is already defined. However, if both
-            // e and the expr pointed by the attr path are two attribute sets,
-            // we want to merge them.
-            // Otherwise, throw an error.
-            auto ae = dynamic_cast<ExprAttrs *>(e);
-            auto jAttrs = dynamic_cast<ExprAttrs *>(j->second.e);
-            if (jAttrs && ae) {
-                for (auto & ad : ae->attrs) {
-                    auto j2 = jAttrs->attrs.find(ad.first);
-                    if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error.
-                        dupAttr(state, ad.first, j2->second.pos, ad.second.pos);
-                    jAttrs->attrs.emplace(ad.first, ad.second);
-                }
-                jAttrs->dynamicAttrs.insert(jAttrs->dynamicAttrs.end(), ae->dynamicAttrs.begin(), ae->dynamicAttrs.end());
-            } else {
-                dupAttr(state, attrPath, pos, j->second.pos);
-            }
-        } else {
-            // This attr path is not defined. Let's create it.
-            attrs->attrs.emplace(i->symbol, ExprAttrs::AttrDef(e, pos));
-            e->setName(i->symbol);
-        }
-    } else {
-        attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, e, pos));
-    }
-}
-
-
-static Formals * toFormals(ParseData & data, ParserFormals * formals,
-    PosIdx pos = noPos, Symbol arg = {})
-{
-    std::sort(formals->formals.begin(), formals->formals.end(),
-        [] (const auto & a, const auto & b) {
-            return std::tie(a.name, a.pos) < std::tie(b.name, b.pos);
-        });
-
-    std::optional<std::pair<Symbol, PosIdx>> duplicate;
-    for (size_t i = 0; i + 1 < formals->formals.size(); i++) {
-        if (formals->formals[i].name != formals->formals[i + 1].name)
-            continue;
-        std::pair thisDup{formals->formals[i].name, formals->formals[i + 1].pos};
-        duplicate = std::min(thisDup, duplicate.value_or(thisDup));
-    }
-    if (duplicate)
-        throw ParseError({
-            .msg = hintfmt("duplicate formal function argument '%1%'", data.symbols[duplicate->first]),
-            .errPos = data.state.positions[duplicate->second]
-        });
-
-    Formals result;
-    result.ellipsis = formals->ellipsis;
-    result.formals = std::move(formals->formals);
-
-    if (arg && result.has(arg))
-        throw ParseError({
-            .msg = hintfmt("duplicate formal function argument '%1%'", data.symbols[arg]),
-            .errPos = data.state.positions[pos]
-        });
-
-    delete formals;
-    return new Formals(std::move(result));
-}
-
-
-static Expr * stripIndentation(const PosIdx pos, SymbolTable & symbols,
-    std::vector<std::pair<PosIdx, std::variant<Expr *, StringToken>>> && es)
-{
-    if (es.empty()) return new ExprString("");
-
-    /* Figure out the minimum indentation.  Note that by design
-       whitespace-only final lines are not taken into account.  (So
-       the " " in "\n ''" is ignored, but the " " in "\n foo''" is.) */
-    bool atStartOfLine = true; /* = seen only whitespace in the current line */
-    size_t minIndent = 1000000;
-    size_t curIndent = 0;
-    for (auto & [i_pos, i] : es) {
-        auto * str = std::get_if<StringToken>(&i);
-        if (!str || !str->hasIndentation) {
-            /* Anti-quotations and escaped characters end the current start-of-line whitespace. */
-            if (atStartOfLine) {
-                atStartOfLine = false;
-                if (curIndent < minIndent) minIndent = curIndent;
-            }
-            continue;
-        }
-        for (size_t j = 0; j < str->l; ++j) {
-            if (atStartOfLine) {
-                if (str->p[j] == ' ')
-                    curIndent++;
-                else if (str->p[j] == '\n') {
-                    /* Empty line, doesn't influence minimum
-                       indentation. */
-                    curIndent = 0;
-                } else {
-                    atStartOfLine = false;
-                    if (curIndent < minIndent) minIndent = curIndent;
-                }
-            } else if (str->p[j] == '\n') {
-                atStartOfLine = true;
-                curIndent = 0;
-            }
-        }
-    }
-
-    /* Strip spaces from each line. */
-    auto * es2 = new std::vector<std::pair<PosIdx, Expr *>>;
-    atStartOfLine = true;
-    size_t curDropped = 0;
-    size_t n = es.size();
-    auto i = es.begin();
-    const auto trimExpr = [&] (Expr * e) {
-        atStartOfLine = false;
-        curDropped = 0;
-        es2->emplace_back(i->first, e);
-    };
-    const auto trimString = [&] (const StringToken & t) {
-        std::string s2;
-        for (size_t j = 0; j < t.l; ++j) {
-            if (atStartOfLine) {
-                if (t.p[j] == ' ') {
-                    if (curDropped++ >= minIndent)
-                        s2 += t.p[j];
-                }
-                else if (t.p[j] == '\n') {
-                    curDropped = 0;
-                    s2 += t.p[j];
-                } else {
-                    atStartOfLine = false;
-                    curDropped = 0;
-                    s2 += t.p[j];
-                }
-            } else {
-                s2 += t.p[j];
-                if (t.p[j] == '\n') atStartOfLine = true;
-            }
-        }
-
-        /* Remove the last line if it is empty and consists only of
-           spaces. */
-        if (n == 1) {
-            std::string::size_type p = s2.find_last_of('\n');
-            if (p != std::string::npos && s2.find_first_not_of(' ', p + 1) == std::string::npos)
-                s2 = std::string(s2, 0, p + 1);
-        }
-
-        es2->emplace_back(i->first, new ExprString(std::move(s2)));
-    };
-    for (; i != es.end(); ++i, --n) {
-        std::visit(overloaded { trimExpr, trimString }, i->second);
-    }
-
-    /* If this is a single string, then don't do a concatenation. */
-    if (es2->size() == 1 && dynamic_cast<ExprString *>((*es2)[0].second)) {
-        auto *const result = (*es2)[0].second;
-        delete es2;
-        return result;
-    }
-    return new ExprConcatStrings(pos, true, es2);
-}
-
-
-static inline PosIdx makeCurPos(const YYLTYPE & loc, ParseData * data)
-{
-    return data->state.positions.add(data->origin, loc.first_line, loc.first_column);
-}
-
-#define CUR_POS makeCurPos(*yylocp, data)
-
-
-}
-
-
-void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * error)
-{
-    data->error = {
         .msg = hintfmt(error),
-        .errPos = data->state.positions[makeCurPos(*loc, data)]
+        .errPos = state->positions[state->at(*loc)]
-    };
+    });
 }
 
 
@@ -314,17 +78,17 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err
   nix::Expr * e;
   nix::ExprList * list;
   nix::ExprAttrs * attrs;
-  nix::ParserFormals * formals;
+  nix::Formals * formals;
   nix::Formal * formal;
   nix::NixInt n;
   nix::NixFloat nf;
-  StringToken id; // !!! -> Symbol
+  nix::StringToken id; // !!! -> Symbol
-  StringToken path;
+  nix::StringToken path;
-  StringToken uri;
+  nix::StringToken uri;
-  StringToken str;
+  nix::StringToken str;
   std::vector<nix::AttrName> * attrNames;
   std::vector<std::pair<nix::PosIdx, nix::Expr *>> * string_parts;
-  std::vector<std::pair<nix::PosIdx, std::variant<nix::Expr *, StringToken>>> * ind_string_parts;
+  std::vector<std::pair<nix::PosIdx, std::variant<nix::Expr *, nix::StringToken>>> * ind_string_parts;
 }
 
 %type <e> start expr expr_function expr_if expr_op
@@ -340,11 +104,11 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err
 %type <id> attr
 %token <id> ID
 %token <str> STR IND_STR
-%token <n> INT
+%token <n> INT_LIT
-%token <nf> FLOAT
+%token <nf> FLOAT_LIT
 %token <path> PATH HPATH SPATH PATH_END
 %token <uri> URI
-%token IF THEN ELSE ASSERT WITH LET IN REC INHERIT EQ NEQ AND OR IMPL OR_KW
+%token IF THEN ELSE ASSERT WITH LET IN_KW REC INHERIT EQ NEQ AND OR IMPL OR_KW
 %token DOLLAR_CURLY /* == ${ */
 %token IND_STRING_OPEN IND_STRING_CLOSE
 %token ELLIPSIS
@@ -364,34 +128,34 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err
 
 %%
 
-start: expr { data->result = $1; };
+start: expr { state->result = $1; };
 
 expr: expr_function;
 
 expr_function
   : ID ':' expr_function
-    { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), 0, $3); }
+    { $$ = new ExprLambda(CUR_POS, state->symbols.create($1), 0, $3); }
   | '{' formals '}' ':' expr_function
-    { $$ = new ExprLambda(CUR_POS, toFormals(*data, $2), $5); }
+    { $$ = new ExprLambda(CUR_POS, state->validateFormals($2), $5); }
   | '{' formals '}' '@' ID ':' expr_function
     {
-      auto arg = data->symbols.create($5);
+      auto arg = state->symbols.create($5);
-      $$ = new ExprLambda(CUR_POS, arg, toFormals(*data, $2, CUR_POS, arg), $7);
+      $$ = new ExprLambda(CUR_POS, arg, state->validateFormals($2, CUR_POS, arg), $7);
     }
   | ID '@' '{' formals '}' ':' expr_function
     {
-      auto arg = data->symbols.create($1);
+      auto arg = state->symbols.create($1);
-      $$ = new ExprLambda(CUR_POS, arg, toFormals(*data, $4, CUR_POS, arg), $7);
+      $$ = new ExprLambda(CUR_POS, arg, state->validateFormals($4, CUR_POS, arg), $7);
     }
   | ASSERT expr ';' expr_function
     { $$ = new ExprAssert(CUR_POS, $2, $4); }
   | WITH expr ';' expr_function
     { $$ = new ExprWith(CUR_POS, $2, $4); }
-  | LET binds IN expr_function
+  | LET binds IN_KW expr_function
     { if (!$2->dynamicAttrs.empty())
         throw ParseError({
             .msg = hintfmt("dynamic attributes not allowed in let"),
-            .errPos = data->state.positions[CUR_POS]
+            .errPos = state->positions[CUR_POS]
         });
       $$ = new ExprLet($2, $4);
     }
@@ -405,24 +169,24 @@ expr_if
 
 expr_op
   : '!' expr_op %prec NOT { $$ = new ExprOpNot($2); }
-  | '-' expr_op %prec NEGATE { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__sub")), {new ExprInt(0), $2}); }
+  | '-' expr_op %prec NEGATE { $$ = new ExprCall(CUR_POS, new ExprVar(state->s.sub), {new ExprInt(0), $2}); }
   | expr_op EQ expr_op { $$ = new ExprOpEq($1, $3); }
   | expr_op NEQ expr_op { $$ = new ExprOpNEq($1, $3); }
-  | expr_op '<' expr_op { $$ = new ExprCall(makeCurPos(@2, data), new ExprVar(data->symbols.create("__lessThan")), {$1, $3}); }
+  | expr_op '<' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.lessThan), {$1, $3}); }
-  | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprCall(makeCurPos(@2, data), new ExprVar(data->symbols.create("__lessThan")), {$3, $1})); }
+  | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprCall(state->at(@2), new ExprVar(state->s.lessThan), {$3, $1})); }
-  | expr_op '>' expr_op { $$ = new ExprCall(makeCurPos(@2, data), new ExprVar(data->symbols.create("__lessThan")), {$3, $1}); }
+  | expr_op '>' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.lessThan), {$3, $1}); }
-  | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprCall(makeCurPos(@2, data), new ExprVar(data->symbols.create("__lessThan")), {$1, $3})); }
+  | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprCall(state->at(@2), new ExprVar(state->s.lessThan), {$1, $3})); }
-  | expr_op AND expr_op { $$ = new ExprOpAnd(makeCurPos(@2, data), $1, $3); }
+  | expr_op AND expr_op { $$ = new ExprOpAnd(state->at(@2), $1, $3); }
-  | expr_op OR expr_op { $$ = new ExprOpOr(makeCurPos(@2, data), $1, $3); }
+  | expr_op OR expr_op { $$ = new ExprOpOr(state->at(@2), $1, $3); }
-  | expr_op IMPL expr_op { $$ = new ExprOpImpl(makeCurPos(@2, data), $1, $3); }
+  | expr_op IMPL expr_op { $$ = new ExprOpImpl(state->at(@2), $1, $3); }
-  | expr_op UPDATE expr_op { $$ = new ExprOpUpdate(makeCurPos(@2, data), $1, $3); }
+  | expr_op UPDATE expr_op { $$ = new ExprOpUpdate(state->at(@2), $1, $3); }
   | expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, std::move(*$3)); delete $3; }
   | expr_op '+' expr_op
-    { $$ = new ExprConcatStrings(makeCurPos(@2, data), false, new std::vector<std::pair<PosIdx, Expr *> >({{makeCurPos(@1, data), $1}, {makeCurPos(@3, data), $3}})); }
+    { $$ = new ExprConcatStrings(state->at(@2), false, new std::vector<std::pair<PosIdx, Expr *> >({{state->at(@1), $1}, {state->at(@3), $3}})); }
-  | expr_op '-' expr_op { $$ = new ExprCall(makeCurPos(@2, data), new ExprVar(data->symbols.create("__sub")), {$1, $3}); }
+  | expr_op '-' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.sub), {$1, $3}); }
-  | expr_op '*' expr_op { $$ = new ExprCall(makeCurPos(@2, data), new ExprVar(data->symbols.create("__mul")), {$1, $3}); }
+  | expr_op '*' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.mul), {$1, $3}); }
-  | expr_op '/' expr_op { $$ = new ExprCall(makeCurPos(@2, data), new ExprVar(data->symbols.create("__div")), {$1, $3}); }
+  | expr_op '/' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.div), {$1, $3}); }
-  | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists(makeCurPos(@2, data), $1, $3); }
+  | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists(state->at(@2), $1, $3); }
   | expr_app
   ;
 
@@ -445,7 +209,7 @@ expr_select
   | /* Backwards compatibility: because Nixpkgs has a rarely used
        function named ‘or’, allow stuff like ‘map or [...]’. */
     expr_simple OR_KW
-    { $$ = new ExprCall(CUR_POS, $1, {new ExprVar(CUR_POS, data->symbols.create("or"))}); }
+    { $$ = new ExprCall(CUR_POS, $1, {new ExprVar(CUR_POS, state->s.or_)}); }
   | expr_simple
   ;
 
@@ -455,25 +219,25 @@ expr_simple
       if ($1.l == s.size() && strncmp($1.p, s.data(), s.size()) == 0)
           $$ = new ExprPos(CUR_POS);
       else
-          $$ = new ExprVar(CUR_POS, data->symbols.create($1));
+          $$ = new ExprVar(CUR_POS, state->symbols.create($1));
   }
-  | INT { $$ = new ExprInt($1); }
+  | INT_LIT { $$ = new ExprInt($1); }
-  | FLOAT { $$ = new ExprFloat($1); }
+  | FLOAT_LIT { $$ = new ExprFloat($1); }
   | '"' string_parts '"' { $$ = $2; }
   | IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE {
-      $$ = stripIndentation(CUR_POS, data->symbols, std::move(*$2));
+      $$ = state->stripIndentation(CUR_POS, std::move(*$2));
       delete $2;
   }
   | path_start PATH_END
   | path_start string_parts_interpolated PATH_END {
-      $2->insert($2->begin(), {makeCurPos(@1, data), $1});
+      $2->insert($2->begin(), {state->at(@1), $1});
       $$ = new ExprConcatStrings(CUR_POS, false, $2);
   }
   | SPATH {
       std::string path($1.p + 1, $1.l - 2);
       $$ = new ExprCall(CUR_POS,
-          new ExprVar(data->symbols.create("__findFile")),
+          new ExprVar(state->s.findFile),
-          {new ExprVar(data->symbols.create("__nixPath")),
+          {new ExprVar(state->s.nixPath),
            new ExprString(std::move(path))});
   }
   | URI {
@@ -481,7 +245,7 @@ expr_simple
       if (noURLLiterals)
           throw ParseError({
               .msg = hintfmt("URL literals are disabled"),
-              .errPos = data->state.positions[CUR_POS]
+              .errPos = state->positions[CUR_POS]
           });
       $$ = new ExprString(std::string($1));
   }
@@ -489,7 +253,7 @@ expr_simple
   /* Let expressions `let {..., body = ...}' are just desugared
      into `(rec {..., body = ...}).body'. */
   | LET '{' binds '}'
-    { $3->recursive = true; $$ = new ExprSelect(noPos, $3, data->symbols.create("body")); }
+    { $3->recursive = true; $$ = new ExprSelect(noPos, $3, state->s.body); }
   | REC '{' binds '}'
     { $3->recursive = true; $$ = $3; }
   | '{' binds '}'
@@ -505,23 +269,23 @@ string_parts
 
 string_parts_interpolated
   : string_parts_interpolated STR
-  { $$ = $1; $1->emplace_back(makeCurPos(@2, data), new ExprString(std::string($2))); }
+  { $$ = $1; $1->emplace_back(state->at(@2), new ExprString(std::string($2))); }
-  | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $3); }
+  | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(state->at(@2), $3); }
-  | DOLLAR_CURLY expr '}' { $$ = new std::vector<std::pair<PosIdx, Expr *>>; $$->emplace_back(makeCurPos(@1, data), $2); }
+  | DOLLAR_CURLY expr '}' { $$ = new std::vector<std::pair<PosIdx, Expr *>>; $$->emplace_back(state->at(@1), $2); }
   | STR DOLLAR_CURLY expr '}' {
       $$ = new std::vector<std::pair<PosIdx, Expr *>>;
-      $$->emplace_back(makeCurPos(@1, data), new ExprString(std::string($1)));
+      $$->emplace_back(state->at(@1), new ExprString(std::string($1)));
-      $$->emplace_back(makeCurPos(@2, data), $3);
+      $$->emplace_back(state->at(@2), $3);
     }
   ;
 
 path_start
   : PATH {
-    Path path(absPath({$1.p, $1.l}, data->basePath.path.abs()));
+    Path path(absPath({$1.p, $1.l}, state->basePath.path.abs()));
     /* add back in the trailing '/' to the first segment */
     if ($1.p[$1.l-1] == '/' && $1.l > 1)
       path += "/";
-    $$ = new ExprPath(ref<InputAccessor>(data->state.rootFS), std::move(path));
+    $$ = new ExprPath(ref<InputAccessor>(state->rootFS), std::move(path));
   }
   | HPATH {
     if (evalSettings.pureEval) {
@@ -531,24 +295,24 @@ path_start
         );
     }
     Path path(getHome() + std::string($1.p + 1, $1.l - 1));
-    $$ = new ExprPath(ref<InputAccessor>(data->state.rootFS), std::move(path));
+    $$ = new ExprPath(ref<InputAccessor>(state->rootFS), std::move(path));
   }
   ;
 
 ind_string_parts
-  : ind_string_parts IND_STR { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $2); }
+  : ind_string_parts IND_STR { $$ = $1; $1->emplace_back(state->at(@2), $2); }
-  | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $3); }
+  | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(state->at(@2), $3); }
   | { $$ = new std::vector<std::pair<PosIdx, std::variant<Expr *, StringToken>>>; }
   ;
 
 binds
-  : binds attrpath '=' expr ';' { $$ = $1; addAttr($$, std::move(*$2), $4, makeCurPos(@2, data), data->state); delete $2; }
+  : binds attrpath '=' expr ';' { $$ = $1; state->addAttr($$, std::move(*$2), $4, state->at(@2)); delete $2; }
   | binds INHERIT attrs ';'
     { $$ = $1;
       for (auto & i : *$3) {
           if ($$->attrs.find(i.symbol) != $$->attrs.end())
-              dupAttr(data->state, i.symbol, makeCurPos(@3, data), $$->attrs[i.symbol].pos);
+              state->dupAttr(i.symbol, state->at(@3), $$->attrs[i.symbol].pos);
-          auto pos = makeCurPos(@3, data);
+          auto pos = state->at(@3);
           $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprVar(CUR_POS, i.symbol), pos, true));
       }
       delete $3;
@@ -558,48 +322,48 @@ binds
       /* !!! Should ensure sharing of the expression in $4. */
       for (auto & i : *$6) {
           if ($$->attrs.find(i.symbol) != $$->attrs.end())
-              dupAttr(data->state, i.symbol, makeCurPos(@6, data), $$->attrs[i.symbol].pos);
+              state->dupAttr(i.symbol, state->at(@6), $$->attrs[i.symbol].pos);
-          $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), makeCurPos(@6, data)));
+          $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), state->at(@6)));
       }
       delete $6;
     }
-  | { $$ = new ExprAttrs(makeCurPos(@0, data)); }
+  | { $$ = new ExprAttrs(state->at(@0)); }
   ;
 
 attrs
-  : attrs attr { $$ = $1; $1->push_back(AttrName(data->symbols.create($2))); }
+  : attrs attr { $$ = $1; $1->push_back(AttrName(state->symbols.create($2))); }
   | attrs string_attr
     { $$ = $1;
       ExprString * str = dynamic_cast<ExprString *>($2);
       if (str) {
-          $$->push_back(AttrName(data->symbols.create(str->s)));
+          $$->push_back(AttrName(state->symbols.create(str->s)));
           delete str;
       } else
           throw ParseError({
               .msg = hintfmt("dynamic attributes not allowed in inherit"),
-              .errPos = data->state.positions[makeCurPos(@2, data)]
+              .errPos = state->positions[state->at(@2)]
           });
     }
   | { $$ = new AttrPath; }
   ;
 
 attrpath
-  : attrpath '.' attr { $$ = $1; $1->push_back(AttrName(data->symbols.create($3))); }
+  : attrpath '.' attr { $$ = $1; $1->push_back(AttrName(state->symbols.create($3))); }
   | attrpath '.' string_attr
     { $$ = $1;
       ExprString * str = dynamic_cast<ExprString *>($3);
       if (str) {
-          $$->push_back(AttrName(data->symbols.create(str->s)));
+          $$->push_back(AttrName(state->symbols.create(str->s)));
           delete str;
       } else
           $$->push_back(AttrName($3));
     }
-  | attr { $$ = new std::vector<AttrName>; $$->push_back(AttrName(data->symbols.create($1))); }
+  | attr { $$ = new std::vector<AttrName>; $$->push_back(AttrName(state->symbols.create($1))); }
   | string_attr
     { $$ = new std::vector<AttrName>;
       ExprString *str = dynamic_cast<ExprString *>($1);
       if (str) {
-          $$->push_back(AttrName(data->symbols.create(str->s)));
+          $$->push_back(AttrName(state->symbols.create(str->s)));
           delete str;
       } else
           $$->push_back(AttrName($1));
@@ -625,226 +389,52 @@ formals
   : formal ',' formals
     { $$ = $3; $$->formals.emplace_back(*$1); delete $1; }
   | formal
-    { $$ = new ParserFormals; $$->formals.emplace_back(*$1); $$->ellipsis = false; delete $1; }
+    { $$ = new Formals; $$->formals.emplace_back(*$1); $$->ellipsis = false; delete $1; }
   |
-    { $$ = new ParserFormals; $$->ellipsis = false; }
+    { $$ = new Formals; $$->ellipsis = false; }
   | ELLIPSIS
-    { $$ = new ParserFormals; $$->ellipsis = true; }
+    { $$ = new Formals; $$->ellipsis = true; }
   ;
 
 formal
-  : ID { $$ = new Formal{CUR_POS, data->symbols.create($1), 0}; }
+  : ID { $$ = new Formal{CUR_POS, state->symbols.create($1), 0}; }
-  | ID '?' expr { $$ = new Formal{CUR_POS, data->symbols.create($1), $3}; }
+  | ID '?' expr { $$ = new Formal{CUR_POS, state->symbols.create($1), $3}; }
   ;
 
 %%
 
-
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <unistd.h>
-
 #include "eval.hh"
-#include "filetransfer.hh"
-#include "tarball.hh"
-#include "store-api.hh"
-#include "flake/flake.hh"
-#include "fs-input-accessor.hh"
-#include "memory-input-accessor.hh"
 
 
 namespace nix {
 
-unsigned long Expr::nrExprs = 0;
+Expr * parseExprFromBuf(
-
-Expr * EvalState::parse(
     char * text,
     size_t length,
     Pos::Origin origin,
     const SourcePath & basePath,
-    std::shared_ptr<StaticEnv> & staticEnv)
+    SymbolTable & symbols,
+    PosTable & positions,
+    const ref<InputAccessor> rootFS,
+    const Expr::AstSymbols & astSymbols)
 {
     yyscan_t scanner;
-    ParseData data {
+    ParserState state {
-        .state = *this,
         .symbols = symbols,
+        .positions = positions,
         .basePath = basePath,
         .origin = {origin},
+        .rootFS = rootFS,
+        .s = astSymbols,
     };
 
     yylex_init(&scanner);
+    Finally _destroy([&] { yylex_destroy(scanner); });
+
     yy_scan_buffer(text, length, scanner);
-    int res = yyparse(scanner, &data);
+    yyparse(scanner, &state);
-    yylex_destroy(scanner);
 
-    if (res) throw ParseError(data.error.value());
+    return state.result;
-
-    data.result->bindVars(*this, staticEnv);
-
-    return data.result;
-}
-
-
-SourcePath resolveExprPath(SourcePath path)
-{
-    unsigned int followCount = 0, maxFollow = 1024;
-
-    /* If `path' is a symlink, follow it.  This is so that relative
-       path references work. */
-    while (!path.path.isRoot()) {
-        // Basic cycle/depth limit to avoid infinite loops.
-        if (++followCount >= maxFollow)
-            throw Error("too many symbolic links encountered while traversing the path '%s'", path);
-        auto p = path.parent().resolveSymlinks() + path.baseName();
-        if (p.lstat().type != InputAccessor::tSymlink) break;
-        path = {path.accessor, CanonPath(p.readLink(), path.path.parent().value_or(CanonPath::root))};
-    }
-
-    /* If `path' refers to a directory, append `/default.nix'. */
-    if (path.resolveSymlinks().lstat().type == InputAccessor::tDirectory)
-        return path + "default.nix";
-
-    return path;
-}
-
-
-Expr * EvalState::parseExprFromFile(const SourcePath & path)
-{
-    return parseExprFromFile(path, staticBaseEnv);
-}
-
-
-Expr * EvalState::parseExprFromFile(const SourcePath & path, std::shared_ptr<StaticEnv> & staticEnv)
-{
-    auto buffer = path.resolveSymlinks().readFile();
-    // readFile hopefully have left some extra space for terminators
-    buffer.append("\0\0", 2);
-    return parse(buffer.data(), buffer.size(), Pos::Origin(path), path.parent(), staticEnv);
-}
-
-
-Expr * EvalState::parseExprFromString(std::string s_, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv)
-{
-    auto s = make_ref<std::string>(std::move(s_));
-    s->append("\0\0", 2);
-    return parse(s->data(), s->size(), Pos::String{.source = s}, basePath, staticEnv);
-}
-
-
-Expr * EvalState::parseExprFromString(std::string s, const SourcePath & basePath)
-{
-    return parseExprFromString(std::move(s), basePath, staticBaseEnv);
-}
-
-
-Expr * EvalState::parseStdin()
-{
-    //Activity act(*logger, lvlTalkative, "parsing standard input");
-    auto buffer = drainFD(0);
-    // drainFD should have left some extra space for terminators
-    buffer.append("\0\0", 2);
-    auto s = make_ref<std::string>(std::move(buffer));
-    return parse(s->data(), s->size(), Pos::Stdin{.source = s}, rootPath(CanonPath::fromCwd()), staticBaseEnv);
-}
-
-
-SourcePath EvalState::findFile(const std::string_view path)
-{
-    return findFile(searchPath, path);
-}
-
-
-SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos)
-{
-    for (auto & i : searchPath.elements) {
-        auto suffixOpt = i.prefix.suffixIfPotentialMatch(path);
-
-        if (!suffixOpt) continue;
-        auto suffix = *suffixOpt;
-
-        auto rOpt = resolveSearchPathPath(i.path);
-        if (!rOpt) continue;
-        auto r = *rOpt;
-
-        Path res = suffix == "" ? r : concatStrings(r, "/", suffix);
-        if (pathExists(res)) return rootPath(CanonPath(canonPath(res)));
-    }
-
-    if (hasPrefix(path, "nix/"))
-        return {corepkgsFS, CanonPath(path.substr(3))};
-
-    debugThrow(ThrownError({
-        .msg = hintfmt(evalSettings.pureEval
-            ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)"
-            : "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)",
-            path),
-        .errPos = positions[pos]
-    }), 0, 0);
-}
-
-
-std::optional<std::string> EvalState::resolveSearchPathPath(const SearchPath::Path & value0, bool initAccessControl)
-{
-    auto & value = value0.s;
-    auto i = searchPathResolved.find(value);
-    if (i != searchPathResolved.end()) return i->second;
-
-    std::optional<std::string> res;
-
-    if (EvalSettings::isPseudoUrl(value)) {
-        try {
-            auto storePath = fetchers::downloadTarball(
-                store, EvalSettings::resolvePseudoUrl(value), "source", false).storePath;
-            res = { store->toRealPath(storePath) };
-        } catch (FileTransferError & e) {
-            logWarning({
-                .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value)
-            });
-        }
-    }
-
-    else if (hasPrefix(value, "flake:")) {
-        experimentalFeatureSettings.require(Xp::Flakes);
-        auto flakeRef = parseFlakeRef(value.substr(6), {}, true, false);
-        debug("fetching flake search path element '%s''", value);
-        auto storePath = flakeRef.resolve(store).fetchTree(store).first;
-        res = { store->toRealPath(storePath) };
-    }
-
-    else {
-        auto path = absPath(value);
-
-        /* Allow access to paths in the search path. */
-        if (initAccessControl) {
-            allowPath(path);
-            if (store->isInStore(path)) {
-                try {
-                    StorePathSet closure;
-                    store->computeFSClosure(store->toStorePath(path).first, closure);
-                    for (auto & p : closure)
-                        allowPath(p);
-                } catch (InvalidPath &) { }
-            }
-        }
-
-        if (pathExists(path))
-            res = { path };
-        else {
-            logWarning({
-                .msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", value)
-            });
-            res = std::nullopt;
-        }
-    }
-
-    if (res)
-        debug("resolved search path element '%s' to '%s'", value, *res);
-    else
-        debug("failed to resolve search path element '%s'", value);
-
-    searchPathResolved.emplace(value, res);
-    return res;
 }
 
 

src/libexpr/primops.cc

@@ -16,6 +16,7 @@
 #include "value-to-xml.hh"
 #include "primops.hh"
 #include "fs-input-accessor.hh"
+#include "fetch-to-store.hh"
 
 #include <boost/container/small_vector.hpp>
 #include <nlohmann/json.hpp>
@@ -84,14 +85,14 @@ StringMap EvalState::realiseContext(const NixStringContext & context)
     /* Build/substitute the context. */
     std::vector<DerivedPath> buildReqs;
     for (auto & d : drvs) buildReqs.emplace_back(DerivedPath { d });
-    store->buildPaths(buildReqs);
+    buildStore->buildPaths(buildReqs, bmNormal, store);
+
+    StorePathSet outputsToCopyAndAllow;
 
     for (auto & drv : drvs) {
-        auto outputs = resolveDerivedPath(*store, drv);
+        auto outputs = resolveDerivedPath(*buildStore, drv, &*store);
         for (auto & [outputName, outputPath] : outputs) {
-            /* Add the output of this derivations to the allowed
+            outputsToCopyAndAllow.insert(outputPath);
-               paths. */
-            allowPath(store->toRealPath(outputPath));
 
             /* Get all the output paths corresponding to the placeholders we had */
             if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
@@ -101,12 +102,19 @@ StringMap EvalState::realiseContext(const NixStringContext & context)
                             .drvPath = drv.drvPath,
                             .output = outputName,
                         }).render(),
-                    store->printStorePath(outputPath)
+                    buildStore->printStorePath(outputPath)
                 );
             }
         }
     }
 
+    if (store != buildStore) copyClosure(*buildStore, *store, outputsToCopyAndAllow);
+    for (auto & outputPath : outputsToCopyAndAllow) {
+        /* Add the output of this derivations to the allowed
+           paths. */
+        allowPath(outputPath);
+    }
+
     return res;
 }
 
@@ -214,7 +222,7 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v
             Env * env = &state.allocEnv(vScope->attrs->size());
             env->up = &state.baseEnv;
 
-            auto staticEnv = std::make_shared<StaticEnv>(false, state.staticBaseEnv.get(), vScope->attrs->size());
+            auto staticEnv = std::make_shared<StaticEnv>(nullptr, state.staticBaseEnv.get(), vScope->attrs->size());
 
             unsigned int displ = 0;
             for (auto & attr : *vScope->attrs) {
@@ -438,9 +446,7 @@ static RegisterPrimOp primop_isNull({
     .doc = R"(
       Return `true` if *e* evaluates to `null`, and `false` otherwise.
 
-      > **Warning**
+      This is equivalent to `e == null`.
-      >
-      > This function is *deprecated*; just write `e == null` instead.
     )",
     .fun = prim_isNull,
 });
@@ -586,7 +592,7 @@ struct CompareValues
                 case nFloat:
                     return v1->fpoint < v2->fpoint;
                 case nString:
-                    return v1->string_view().compare(v2->string_view()) < 0;
+                    return strcmp(v1->c_str(), v2->c_str()) < 0;
                 case nPath:
                     // Note: we don't take the accessor into account
                     // since it's not obvious how to compare them in a
@@ -991,7 +997,7 @@ static void prim_trace(EvalState & state, const PosIdx pos, Value * * args, Valu
     if (args[0]->type() == nString)
         printError("trace: %1%", args[0]->string_view());
     else
-        printError("trace: %1%", printValue(state, *args[0]));
+        printError("trace: %1%", ValuePrinter(state, *args[0]));
     state.forceValue(*args[1], pos);
     v = *args[1];
 }
@@ -1872,7 +1878,7 @@ static RegisterPrimOp primop_outputOf({
       For instance,
       ```nix
       builtins.outputOf
-        (builtins.outputOf myDrv "out)
+        (builtins.outputOf myDrv "out")
         "out"
       ```
       will return a placeholder for the output of the output of `myDrv`.
@@ -2072,8 +2078,14 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Val
     }
 
     auto storePath = settings.readOnlyMode
-        ? state.store->computeStorePathForText(name, contents, refs)
+        ? state.store->makeFixedOutputPathFromCA(name, TextInfo {
-        : state.store->addTextToStore(name, contents, refs, state.repair);
+            .hash = hashString(HashAlgorithm::SHA256, contents),
+            .references = std::move(refs),
+        })
+        : ({
+            StringSource s { contents };
+            state.store->addToStoreFromDump(s, name, TextIngestionMethod {}, HashAlgorithm::SHA256, refs, state.repair);
+        });
 
     /* Note: we don't need to add `context' to the context of the
        result, since `storePath' itself has references to the paths
@@ -2229,7 +2241,7 @@ static void addPath(
             });
 
         if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
-            auto dstPath = path.fetchToStore(state.store, name, method, filter.get(), state.repair);
+            auto dstPath = fetchToStore(*state.store, path.resolveSymlinks(), name, method, filter.get(), state.repair);
             if (expectedHash && expectedStorePath != dstPath)
                 state.debugThrowLastTrace(Error("store path mismatch in (possibly filtered) path added from '%s'", path));
             state.allowAndSetStorePathString(dstPath, v);
@@ -2401,7 +2413,7 @@ static void prim_attrNames(EvalState & state, const PosIdx pos, Value * * args,
         (v.listElems()[n++] = state.allocValue())->mkString(state.symbols[i.name]);
 
     std::sort(v.listElems(), v.listElems() + n,
-              [](Value * v1, Value * v2) { return v1->string_view().compare(v2->string_view()) < 0; });
+              [](Value * v1, Value * v2) { return strcmp(v1->c_str(), v2->c_str()) < 0; });
 }
 
 static RegisterPrimOp primop_attrNames({
@@ -3700,9 +3712,6 @@ static RegisterPrimOp primop_toString({
 static void prim_substring(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
     int start = state.forceInt(*args[0], pos, "while evaluating the first argument (the start offset) passed to builtins.substring");
-    int len = state.forceInt(*args[1], pos, "while evaluating the second argument (the substring length) passed to builtins.substring");
-    NixStringContext context;
-    auto s = state.coerceToString(pos, *args[2], context, "while evaluating the third argument (the string) passed to builtins.substring");
 
     if (start < 0)
         state.debugThrowLastTrace(EvalError({
@@ -3710,6 +3719,22 @@ static void prim_substring(EvalState & state, const PosIdx pos, Value * * args,
             .errPos = state.positions[pos]
         }));
 
+
+    int len = state.forceInt(*args[1], pos, "while evaluating the second argument (the substring length) passed to builtins.substring");
+
+    // Special-case on empty substring to avoid O(n) strlen
+    // This allows for the use of empty substrings to efficently capture string context
+    if (len == 0) {
+        state.forceValue(*args[2], pos);
+        if (args[2]->type() == nString) {
+            v.mkString("", args[2]->context());
+            return;
+        }
+    }
+
+    NixStringContext context;
+    auto s = state.coerceToString(pos, *args[2], context, "while evaluating the third argument (the string) passed to builtins.substring");
+
     v.mkString((unsigned int) start >= s->size() ? "" : s->substr(start, len), context);
 }
 
@@ -4383,13 +4408,16 @@ void EvalState::createBaseEnv()
         .impureOnly = true,
     });
 
-    if (!evalSettings.pureEval) {
+    if (!evalSettings.pureEval)
-        v.mkString(settings.thisSystem.get());
+        v.mkString(evalSettings.getCurrentSystem());
-    }
     addConstant("__currentSystem", v, {
         .type = nString,
         .doc = R"(
-          The value of the [`system` configuration option](@docroot@/command-ref/conf-file.md#conf-system).
+          The value of the
+          [`eval-system`](@docroot@/command-ref/conf-file.md#conf-eval-system)
+          or else
+          [`system`](@docroot@/command-ref/conf-file.md#conf-system)
+          configuration option.
 
           It can be used to set the `system` attribute for [`builtins.derivation`](@docroot@/language/derivations.md) such that the resulting derivation can be built on the same system that evaluates the Nix expression:
 

src/libexpr/primops/fetchTree.cc

@@ -1,3 +1,4 @@
+#include "libfetchers/attrs.hh"
 #include "primops.hh"
 #include "eval-inline.hh"
 #include "eval-settings.hh"
@@ -25,7 +26,7 @@ void emitTreeAttrs(
 {
     assert(input.isLocked());
 
-    auto attrs = state.buildBindings(10);
+    auto attrs = state.buildBindings(100);
 
     state.mkStorePathString(storePath, attrs.alloc(state.sOutPath));
 
@@ -135,6 +136,10 @@ static void fetchTree(
                     state.symbols[attr.name], showType(*attr.value)));
         }
 
+        if (params.isFetchGit && !attrs.contains("exportIgnore") && (!attrs.contains("submodules") || !*fetchers::maybeGetBoolAttr(attrs, "submodules"))) {
+            attrs.emplace("exportIgnore", Explicit<bool>{true});
+        }
+
         if (!params.allowNameArgument)
             if (auto nameIter = attrs.find("name"); nameIter != attrs.end())
                 state.debugThrowLastTrace(EvalError({
@@ -152,6 +157,9 @@ static void fetchTree(
             fetchers::Attrs attrs;
             attrs.emplace("type", "git");
             attrs.emplace("url", fixGitURL(url));
+            if (!attrs.contains("exportIgnore") && (!attrs.contains("submodules") || !*fetchers::maybeGetBoolAttr(attrs, "submodules"))) {
+                attrs.emplace("exportIgnore", Explicit<bool>{true});
+            }
             input = fetchers::Input::fromAttrs(std::move(attrs));
         } else {
             if (!experimentalFeatureSettings.isEnabled(Xp::Flakes))
@@ -166,8 +174,12 @@ static void fetchTree(
     if (!evalSettings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes))
         input = lookupInRegistries(state.store, input).first;
 
-    if (evalSettings.pureEval && !input.isLocked())
+    if (evalSettings.pureEval && !input.isLocked()) {
-        state.debugThrowLastTrace(EvalError("in pure evaluation mode, 'fetchTree' requires a locked input, at %s", state.positions[pos]));
+        if (params.isFetchGit)
+            state.debugThrowLastTrace(EvalError("in pure evaluation mode, 'fetchGit' requires a locked input, at %s", state.positions[pos]));
+        else
+            state.debugThrowLastTrace(EvalError("in pure evaluation mode, 'fetchTree' requires a locked input, at %s", state.positions[pos]));
+    }
 
     state.checkURI(input.toURLString());
 
@@ -593,10 +605,16 @@ static RegisterPrimOp primop_fetchGit({
 
         A Boolean parameter that specifies whether submodules should be checked out.
 
+      - `exportIgnore` (default: `true`)
+
+        A Boolean parameter that specifies whether `export-ignore` from `.gitattributes` should be applied.
+        This approximates part of the `git archive` behavior.
+
+        Enabling this option is not recommended because it is unknown whether the Git developers commit to the reproducibility of `export-ignore` in newer Git versions.
+
       - `shallow` (default: `false`)
 
-        A Boolean parameter that specifies whether fetching from a shallow remote repository is allowed.
+        Make a shallow clone when fetching the Git tree.
-        This still performs a full clone of what is available on the remote.
 
       - `allRefs`
 

src/libexpr/print-ambiguous.cc

@@ -0,0 +1,101 @@
+#include "print-ambiguous.hh"
+#include "print.hh"
+#include "signals.hh"
+#include "eval.hh"
+
+namespace nix {
+
+// See: https://github.com/NixOS/nix/issues/9730
+void printAmbiguous(
+    Value &v,
+    const SymbolTable &symbols,
+    std::ostream &str,
+    std::set<const void *> *seen,
+    int depth)
+{
+    checkInterrupt();
+
+    if (depth <= 0) {
+        str << "«too deep»";
+        return;
+    }
+    switch (v.type()) {
+    case nInt:
+        str << v.integer;
+        break;
+    case nBool:
+        printLiteralBool(str, v.boolean);
+        break;
+    case nString:
+        printLiteralString(str, v.string_view());
+        break;
+    case nPath:
+        str << v.path().to_string(); // !!! escaping?
+        break;
+    case nNull:
+        str << "null";
+        break;
+    case nAttrs: {
+        if (seen && !v.attrs->empty() && !seen->insert(v.attrs).second)
+            str << "«repeated»";
+        else {
+            str << "{ ";
+            for (auto & i : v.attrs->lexicographicOrder(symbols)) {
+                str << symbols[i->name] << " = ";
+                printAmbiguous(*i->value, symbols, str, seen, depth - 1);
+                str << "; ";
+            }
+            str << "}";
+        }
+        break;
+    }
+    case nList:
+        if (seen && v.listSize() && !seen->insert(v.listElems()).second)
+            str << "«repeated»";
+        else {
+            str << "[ ";
+            for (auto v2 : v.listItems()) {
+                if (v2)
+                    printAmbiguous(*v2, symbols, str, seen, depth - 1);
+                else
+                    str << "(nullptr)";
+                str << " ";
+            }
+            str << "]";
+        }
+        break;
+    case nThunk:
+        if (!v.isBlackhole()) {
+            str << "<CODE>";
+        } else {
+            // Although we know for sure that it's going to be an infinite recursion
+            // when this value is accessed _in the current context_, it's likely
+            // that the user will misinterpret a simpler «infinite recursion» output
+            // as a definitive statement about the value, while in fact it may be
+            // a valid value after `builtins.trace` and perhaps some other steps
+            // have completed.
+            str << "«potential infinite recursion»";
+        }
+        break;
+    case nFunction:
+        if (v.isLambda()) {
+            str << "<LAMBDA>";
+        } else if (v.isPrimOp()) {
+            str << "<PRIMOP>";
+        } else if (v.isPrimOpApp()) {
+            str << "<PRIMOP-APP>";
+        }
+        break;
+    case nExternal:
+        str << *v.external;
+        break;
+    case nFloat:
+        str << v.fpoint;
+        break;
+    default:
+        printError("Nix evaluator internal error: printAmbiguous: invalid value type");
+        abort();
+    }
+}
+
+}

src/libexpr/print-ambiguous.hh

@@ -0,0 +1,24 @@
+#pragma once
+
+#include "value.hh"
+
+namespace nix {
+
+/**
+ * Print a value in the deprecated format used by `nix-instantiate --eval` and
+ * `nix-env` (for manifests).
+ *
+ * This output can't be changed because it's part of the `nix-instantiate` API,
+ * but it produces ambiguous output; unevaluated thunks and lambdas (and a few
+ * other types) are printed as Nix path syntax like `<CODE>`.
+ *
+ * See: https://github.com/NixOS/nix/issues/9730
+ */
+void printAmbiguous(
+    Value &v,
+    const SymbolTable &symbols,
+    std::ostream &str,
+    std::set<const void *> *seen,
+    int depth);
+
+}

src/libexpr/print-options.hh

@@ -0,0 +1,70 @@
+#pragma once
+/**
+ * @file
+ * @brief Options for printing Nix values.
+ */
+
+#include <limits>
+
+namespace nix {
+
+/**
+ * Options for printing Nix values.
+ */
+struct PrintOptions
+{
+    /**
+     * If true, output ANSI color sequences.
+     */
+    bool ansiColors = false;
+    /**
+     * If true, force values.
+     */
+    bool force = false;
+    /**
+     * If true and `force` is set, print derivations as
+     * `«derivation /nix/store/...»` instead of as attribute sets.
+     */
+    bool derivationPaths = false;
+    /**
+     * If true, track which values have been printed and skip them on
+     * subsequent encounters. Useful for self-referential values.
+     */
+    bool trackRepeated = true;
+    /**
+     * Maximum depth to evaluate to.
+     */
+    size_t maxDepth = std::numeric_limits<size_t>::max();
+    /**
+     * Maximum number of attributes in attribute sets to print.
+     *
+     * Note that this is a limit for the entire print invocation, not for each
+     * attribute set encountered.
+     */
+    size_t maxAttrs = std::numeric_limits<size_t>::max();
+    /**
+     * Maximum number of list items to print.
+     *
+     * Note that this is a limit for the entire print invocation, not for each
+     * list encountered.
+     */
+    size_t maxListItems = std::numeric_limits<size_t>::max();
+    /**
+     * Maximum string length to print.
+     */
+    size_t maxStringLength = std::numeric_limits<size_t>::max();
+};
+
+/**
+ * `PrintOptions` for unknown and therefore potentially large values in error messages,
+ * to avoid printing "too much" output.
+ */
+static PrintOptions errorPrintOptions = PrintOptions {
+    .ansiColors = true,
+    .maxDepth = 10,
+    .maxAttrs = 10,
+    .maxListItems = 10,
+    .maxStringLength = 1024
+};
+
+}

src/libexpr/print.cc

@@ -1,24 +1,67 @@
-#include "print.hh"
+#include <limits>
 #include <unordered_set>
 
+#include "print.hh"
+#include "ansicolor.hh"
+#include "signals.hh"
+#include "store-api.hh"
+#include "terminal.hh"
+#include "english.hh"
+#include "eval.hh"
+
 namespace nix {
 
-std::ostream &
+void printElided(
-printLiteralString(std::ostream & str, const std::string_view string)
+    std::ostream & output,
+    unsigned int value,
+    const std::string_view single,
+    const std::string_view plural,
+    bool ansiColors)
 {
+    if (ansiColors)
+        output << ANSI_FAINT;
+    output << "«";
+    pluralize(output, value, single, plural);
+    output << " elided»";
+    if (ansiColors)
+        output << ANSI_NORMAL;
+}
+
+
+std::ostream &
+printLiteralString(std::ostream & str, const std::string_view string, size_t maxLength, bool ansiColors)
+{
+    size_t charsPrinted = 0;
+    if (ansiColors)
+        str << ANSI_MAGENTA;
     str << "\"";
     for (auto i = string.begin(); i != string.end(); ++i) {
+        if (charsPrinted >= maxLength) {
+            str << "\" ";
+            printElided(str, string.length() - charsPrinted, "byte", "bytes", ansiColors);
+            return str;
+        }
+
         if (*i == '\"' || *i == '\\') str << "\\" << *i;
         else if (*i == '\n') str << "\\n";
         else if (*i == '\r') str << "\\r";
         else if (*i == '\t') str << "\\t";
         else if (*i == '$' && *(i+1) == '{') str << "\\" << *i;
         else str << *i;
+        charsPrinted++;
     }
     str << "\"";
+    if (ansiColors)
+        str << ANSI_NORMAL;
     return str;
 }
 
+std::ostream &
+printLiteralString(std::ostream & str, const std::string_view string)
+{
+    return printLiteralString(str, string, std::numeric_limits<size_t>::max(), false);
+}
+
 std::ostream &
 printLiteralBool(std::ostream & str, bool boolean)
 {
@@ -90,5 +133,382 @@ printAttributeName(std::ostream & str, std::string_view name) {
     return str;
 }
 
+bool isImportantAttrName(const std::string& attrName)
+{
+    return attrName == "type" || attrName == "_type";
+}
+
+typedef std::pair<std::string, Value *> AttrPair;
+
+struct ImportantFirstAttrNameCmp
+{
+
+    bool operator()(const AttrPair& lhs, const AttrPair& rhs) const
+    {
+        auto lhsIsImportant = isImportantAttrName(lhs.first);
+        auto rhsIsImportant = isImportantAttrName(rhs.first);
+        return std::forward_as_tuple(!lhsIsImportant, lhs.first)
+            < std::forward_as_tuple(!rhsIsImportant, rhs.first);
+    }
+};
+
+typedef std::set<Value *> ValuesSeen;
+
+class Printer
+{
+private:
+    std::ostream & output;
+    EvalState & state;
+    PrintOptions options;
+    std::optional<ValuesSeen> seen;
+    size_t attrsPrinted = 0;
+    size_t listItemsPrinted = 0;
+
+    void printRepeated()
+    {
+        if (options.ansiColors)
+            output << ANSI_MAGENTA;
+        output << "«repeated»";
+        if (options.ansiColors)
+            output << ANSI_NORMAL;
+    }
+
+    void printNullptr()
+    {
+        if (options.ansiColors)
+            output << ANSI_MAGENTA;
+        output << "«nullptr»";
+        if (options.ansiColors)
+            output << ANSI_NORMAL;
+    }
+
+    void printElided(unsigned int value, const std::string_view single, const std::string_view plural)
+    {
+        ::nix::printElided(output, value, single, plural, options.ansiColors);
+    }
+
+    void printInt(Value & v)
+    {
+        if (options.ansiColors)
+            output << ANSI_CYAN;
+        output << v.integer;
+        if (options.ansiColors)
+            output << ANSI_NORMAL;
+    }
+
+    void printFloat(Value & v)
+    {
+        if (options.ansiColors)
+            output << ANSI_CYAN;
+        output << v.fpoint;
+        if (options.ansiColors)
+            output << ANSI_NORMAL;
+    }
+
+    void printBool(Value & v)
+    {
+        if (options.ansiColors)
+            output << ANSI_CYAN;
+        printLiteralBool(output, v.boolean);
+        if (options.ansiColors)
+            output << ANSI_NORMAL;
+    }
+
+    void printString(Value & v)
+    {
+        printLiteralString(output, v.string_view(), options.maxStringLength, options.ansiColors);
+    }
+
+    void printPath(Value & v)
+    {
+        if (options.ansiColors)
+            output << ANSI_GREEN;
+        output << v.path().to_string(); // !!! escaping?
+        if (options.ansiColors)
+            output << ANSI_NORMAL;
+    }
+
+    void printNull()
+    {
+        if (options.ansiColors)
+            output << ANSI_CYAN;
+        output << "null";
+        if (options.ansiColors)
+            output << ANSI_NORMAL;
+    }
+
+    void printDerivation(Value & v)
+    {
+        try {
+            Bindings::iterator i = v.attrs->find(state.sDrvPath);
+            NixStringContext context;
+            std::string storePath;
+            if (i != v.attrs->end())
+                storePath = state.store->printStorePath(state.coerceToStorePath(i->pos, *i->value, context, "while evaluating the drvPath of a derivation"));
+
+            if (options.ansiColors)
+                output << ANSI_GREEN;
+            output << "«derivation";
+            if (!storePath.empty()) {
+                output << " " << storePath;
+            }
+            output << "»";
+            if (options.ansiColors)
+                output << ANSI_NORMAL;
+        } catch (BaseError & e) {
+            printError_(e);
+        }
+    }
+
+    void printAttrs(Value & v, size_t depth)
+    {
+        if (seen && !seen->insert(&v).second) {
+            printRepeated();
+            return;
+        }
+
+        if (options.force && options.derivationPaths && state.isDerivation(v)) {
+            printDerivation(v);
+        } else if (depth < options.maxDepth) {
+            output << "{ ";
+
+            std::vector<std::pair<std::string, Value *>> sorted;
+            for (auto & i : *v.attrs)
+                sorted.emplace_back(std::pair(state.symbols[i.name], i.value));
+
+            if (options.maxAttrs == std::numeric_limits<size_t>::max())
+                std::sort(sorted.begin(), sorted.end());
+            else
+                std::sort(sorted.begin(), sorted.end(), ImportantFirstAttrNameCmp());
+
+            for (auto & i : sorted) {
+                if (attrsPrinted >= options.maxAttrs) {
+                    printElided(sorted.size() - attrsPrinted, "attribute", "attributes");
+                    break;
+                }
+
+                printAttributeName(output, i.first);
+                output << " = ";
+                print(*i.second, depth + 1);
+                output << "; ";
+                attrsPrinted++;
+            }
+
+            output << "}";
+        } else
+            output << "{ ... }";
+    }
+
+    void printList(Value & v, size_t depth)
+    {
+        if (seen && v.listSize() && !seen->insert(&v).second) {
+            printRepeated();
+            return;
+        }
+
+        output << "[ ";
+        if (depth < options.maxDepth) {
+            for (auto elem : v.listItems()) {
+                if (listItemsPrinted >= options.maxListItems) {
+                    printElided(v.listSize() - listItemsPrinted, "item", "items");
+                    break;
+                }
+
+                if (elem) {
+                    print(*elem, depth + 1);
+                } else {
+                    printNullptr();
+                }
+                output << " ";
+                listItemsPrinted++;
+            }
+        }
+        else
+            output << "... ";
+        output << "]";
+    }
+
+    void printFunction(Value & v)
+    {
+        if (options.ansiColors)
+            output << ANSI_BLUE;
+        output << "«";
+
+        if (v.isLambda()) {
+            output << "lambda";
+            if (v.lambda.fun) {
+                if (v.lambda.fun->name) {
+                    output << " " << state.symbols[v.lambda.fun->name];
+                }
+
+                std::ostringstream s;
+                s << state.positions[v.lambda.fun->pos];
+                output << " @ " << filterANSIEscapes(s.str());
+            }
+        } else if (v.isPrimOp()) {
+            if (v.primOp)
+                output << *v.primOp;
+            else
+                output << "primop";
+        } else if (v.isPrimOpApp()) {
+            output << "partially applied ";
+            auto primOp = v.primOpAppPrimOp();
+            if (primOp)
+                output << *primOp;
+            else
+                output << "primop";
+        } else {
+            abort();
+        }
+
+        output << "»";
+        if (options.ansiColors)
+            output << ANSI_NORMAL;
+    }
+
+    void printThunk(Value & v)
+    {
+        if (v.isBlackhole()) {
+            // Although we know for sure that it's going to be an infinite recursion
+            // when this value is accessed _in the current context_, it's likely
+            // that the user will misinterpret a simpler «infinite recursion» output
+            // as a definitive statement about the value, while in fact it may be
+            // a valid value after `builtins.trace` and perhaps some other steps
+            // have completed.
+            if (options.ansiColors)
+                output << ANSI_RED;
+            output << "«potential infinite recursion»";
+            if (options.ansiColors)
+                output << ANSI_NORMAL;
+        } else if (v.isThunk() || v.isApp()) {
+            if (options.ansiColors)
+                    output << ANSI_MAGENTA;
+            output << "«thunk»";
+            if (options.ansiColors)
+                    output << ANSI_NORMAL;
+        } else {
+            abort();
+        }
+    }
+
+    void printExternal(Value & v)
+    {
+        v.external->print(output);
+    }
+
+    void printUnknown()
+    {
+        if (options.ansiColors)
+            output << ANSI_RED;
+        output << "«unknown»";
+        if (options.ansiColors)
+            output << ANSI_NORMAL;
+    }
+
+    void printError_(BaseError & e)
+    {
+        if (options.ansiColors)
+            output << ANSI_RED;
+        output << "«" << e.msg() << "»";
+        if (options.ansiColors)
+            output << ANSI_NORMAL;
+    }
+
+    void print(Value & v, size_t depth)
+    {
+        output.flush();
+        checkInterrupt();
+
+        if (options.force) {
+            try {
+                state.forceValue(v, v.determinePos(noPos));
+            } catch (BaseError & e) {
+                printError_(e);
+                return;
+            }
+        }
+
+        switch (v.type()) {
+
+        case nInt:
+            printInt(v);
+            break;
+
+        case nFloat:
+            printFloat(v);
+            break;
+
+        case nBool:
+            printBool(v);
+            break;
+
+        case nString:
+            printString(v);
+            break;
+
+        case nPath:
+            printPath(v);
+            break;
+
+        case nNull:
+            printNull();
+            break;
+
+        case nAttrs:
+            printAttrs(v, depth);
+            break;
+
+        case nList:
+            printList(v, depth);
+            break;
+
+        case nFunction:
+            printFunction(v);
+            break;
+
+        case nThunk:
+            printThunk(v);
+            break;
+
+        case nExternal:
+            printExternal(v);
+            break;
+
+        default:
+            printUnknown();
+            break;
+        }
+    }
+
+public:
+    Printer(std::ostream & output, EvalState & state, PrintOptions options)
+        : output(output), state(state), options(options) { }
+
+    void print(Value & v)
+    {
+        attrsPrinted = 0;
+        listItemsPrinted = 0;
+
+        if (options.trackRepeated) {
+            seen.emplace();
+        } else {
+            seen.reset();
+        }
+
+        ValuesSeen seen;
+        print(v, 0);
+    }
+};
+
+void printValue(EvalState & state, std::ostream & output, Value & v, PrintOptions options)
+{
+    Printer(output, state, options).print(v);
+}
+
+std::ostream & operator<<(std::ostream & output, const ValuePrinter & printer)
+{
+    printValue(printer.state, output, printer.value, printer.options);
+    return output;
+}
 
 }

src/libexpr/print.hh

@@ -9,46 +9,73 @@
 
 #include <iostream>
 
+#include "print-options.hh"
+
 namespace nix {
-    /**
-     * Print a string as a Nix string literal.
-     *
-     * Quotes and fairly minimal escaping are added.
-     *
-     * @param s The logical string
-     */
-    std::ostream & printLiteralString(std::ostream & o, std::string_view s);
-    inline std::ostream & printLiteralString(std::ostream & o, const char * s) {
-        return printLiteralString(o, std::string_view(s));
-    }
-    inline std::ostream & printLiteralString(std::ostream & o, const std::string & s) {
-        return printLiteralString(o, std::string_view(s));
-    }
 
-    /** Print `true` or `false`. */
+class EvalState;
-    std::ostream & printLiteralBool(std::ostream & o, bool b);
+struct Value;
 
-    /**
+/**
-     * Print a string as an attribute name in the Nix expression language syntax.
+ * Print a string as a Nix string literal.
-     *
+ *
-     * Prints a quoted string if necessary.
+ * Quotes and fairly minimal escaping are added.
-     */
+ *
-    std::ostream & printAttributeName(std::ostream & o, std::string_view s);
+ * @param o The output stream to print to
-
+ * @param s The logical string
-    /**
+ */
-     * Returns `true' is a string is a reserved keyword which requires quotation
+std::ostream & printLiteralString(std::ostream & o, std::string_view s);
-     * when printing attribute set field names.
+inline std::ostream & printLiteralString(std::ostream & o, const char * s) {
-     */
+    return printLiteralString(o, std::string_view(s));
-    bool isReservedKeyword(const std::string_view str);
+}
-
+inline std::ostream & printLiteralString(std::ostream & o, const std::string & s) {
-    /**
+    return printLiteralString(o, std::string_view(s));
-     * Print a string as an identifier in the Nix expression language syntax.
+}
-     *
+
-     * FIXME: "identifier" is ambiguous. Identifiers do not have a single
+/** Print `true` or `false`. */
-     *        textual representation. They can be used in variable references,
+std::ostream & printLiteralBool(std::ostream & o, bool b);
-     *        let bindings, left-hand sides or attribute names in a select
+
-     *        expression, or something else entirely, like JSON. Use one of the
+/**
-     *        `print*` functions instead.
+ * Print a string as an attribute name in the Nix expression language syntax.
-     */
+ *
-    std::ostream & printIdentifier(std::ostream & o, std::string_view s);
+ * Prints a quoted string if necessary.
+ */
+std::ostream & printAttributeName(std::ostream & o, std::string_view s);
+
+/**
+ * Returns `true' is a string is a reserved keyword which requires quotation
+ * when printing attribute set field names.
+ */
+bool isReservedKeyword(const std::string_view str);
+
+/**
+ * Print a string as an identifier in the Nix expression language syntax.
+ *
+ * FIXME: "identifier" is ambiguous. Identifiers do not have a single
+ *        textual representation. They can be used in variable references,
+ *        let bindings, left-hand sides or attribute names in a select
+ *        expression, or something else entirely, like JSON. Use one of the
+ *        `print*` functions instead.
+ */
+std::ostream & printIdentifier(std::ostream & o, std::string_view s);
+
+void printValue(EvalState & state, std::ostream & str, Value & v, PrintOptions options = PrintOptions {});
+
+/**
+ * A partially-applied form of `printValue` which can be formatted using `<<`
+ * without allocating an intermediate string.
+ */
+class ValuePrinter {
+    friend std::ostream & operator << (std::ostream & output, const ValuePrinter & printer);
+private:
+    EvalState & state;
+    Value & value;
+    PrintOptions options;
+
+public:
+    ValuePrinter(EvalState & state, Value & value, PrintOptions options = PrintOptions {})
+        : state(state), value(value), options(options) { }
+};
+
+std::ostream & operator<<(std::ostream & output, const ValuePrinter & printer);
 }

src/libexpr/value.hh

@@ -8,6 +8,8 @@
 #include "symbol-table.hh"
 #include "value/context.hh"
 #include "input-accessor.hh"
+#include "source-path.hh"
+#include "print-options.hh"
 
 #if HAVE_BOEHMGC
 #include <gc/gc_allocator.h>
@@ -32,7 +34,6 @@ typedef enum {
     tThunk,
     tApp,
     tLambda,
-    tBlackhole,
     tPrimOp,
     tPrimOpApp,
     tExternal,
@@ -62,6 +63,7 @@ class Bindings;
 struct Env;
 struct Expr;
 struct ExprLambda;
+struct ExprBlackHole;
 struct PrimOp;
 class Symbol;
 class PosIdx;
@@ -69,7 +71,7 @@ struct Pos;
 class StorePath;
 class EvalState;
 class XMLWriter;
-
+class Printer;
 
 typedef int64_t NixInt;
 typedef double NixFloat;
@@ -81,6 +83,7 @@ typedef double NixFloat;
 class ExternalValueBase
 {
     friend std::ostream & operator << (std::ostream & str, const ExternalValueBase & v);
+    friend class Printer;
     protected:
     /**
      * Print out the value
@@ -138,11 +141,9 @@ private:
 
     friend std::string showType(const Value & v);
 
-    void print(const SymbolTable &symbols, std::ostream &str, std::set<const void *> *seen, int depth) const;
-
 public:
 
-    void print(const SymbolTable &symbols, std::ostream &str, bool showRepeated = false, int depth = INT_MAX) const;
+    void print(EvalState &state, std::ostream &str, PrintOptions options = PrintOptions {});
 
     // Functions needed to distinguish the type
     // These should be removed eventually, by putting the functionality that's
@@ -151,7 +152,7 @@ public:
     // type() == nThunk
     inline bool isThunk() const { return internalType == tThunk; };
     inline bool isApp() const { return internalType == tApp; };
-    inline bool isBlackhole() const { return internalType == tBlackhole; };
+    inline bool isBlackhole() const;
 
     // type() == nFunction
     inline bool isLambda() const { return internalType == tLambda; };
@@ -248,7 +249,7 @@ public:
             case tLambda: case tPrimOp: case tPrimOpApp: return nFunction;
             case tExternal: return nExternal;
             case tFloat: return nFloat;
-            case tThunk: case tApp: case tBlackhole: return nThunk;
+            case tThunk: case tApp: return nThunk;
         }
         if (invalidIsThunk)
             return nThunk;
@@ -356,21 +357,22 @@ public:
         lambda.fun = f;
     }
 
-    inline void mkBlackhole()
+    inline void mkBlackhole();
-    {
-        internalType = tBlackhole;
-        // Value will be overridden anyways
-    }
 
     void mkPrimOp(PrimOp * p);
 
     inline void mkPrimOpApp(Value * l, Value * r)
     {
         internalType = tPrimOpApp;
-        app.left = l;
+        primOpApp.left = l;
-        app.right = r;
+        primOpApp.right = r;
     }
 
+    /**
+     * For a `tPrimOpApp` value, get the original `PrimOp` value.
+     */
+    PrimOp * primOpAppPrimOp() const;
+
     inline void mkExternal(ExternalValueBase * e)
     {
         clearValue();
@@ -447,6 +449,20 @@ public:
 };
 
 
+extern ExprBlackHole eBlackHole;
+
+bool Value::isBlackhole() const
+{
+    return internalType == tThunk && thunk.expr == (Expr*) &eBlackHole;
+}
+
+void Value::mkBlackhole()
+{
+    internalType = tThunk;
+    thunk.expr = (Expr*) &eBlackHole;
+}
+
+
 #if HAVE_BOEHMGC
 typedef std::vector<Value *, traceable_allocator<Value *>> ValueVector;
 typedef std::map<Symbol, Value *, std::less<Symbol>, traceable_allocator<std::pair<const Symbol, Value *>>> ValueMap;

src/libfetchers/cache.cc

@@ -106,7 +106,7 @@ struct CacheImpl : Cache
     }
 
     void add(
-        ref<Store> store,
+        Store & store,
         const Attrs & inAttrs,
         const Attrs & infoAttrs,
         const StorePath & storePath,
@@ -115,13 +115,13 @@ struct CacheImpl : Cache
         _state.lock()->add.use()
             (attrsToJSON(inAttrs).dump())
             (attrsToJSON(infoAttrs).dump())
-            (store->printStorePath(storePath))
+            (store.printStorePath(storePath))
             (locked)
             (time(0)).exec();
     }
 
     std::optional<std::pair<Attrs, StorePath>> lookup(
-        ref<Store> store,
+        Store & store,
         const Attrs & inAttrs) override
     {
         if (auto res = lookupExpired(store, inAttrs)) {
@@ -134,7 +134,7 @@ struct CacheImpl : Cache
     }
 
     std::optional<Result> lookupExpired(
-        ref<Store> store,
+        Store & store,
         const Attrs & inAttrs) override
     {
         auto state(_state.lock());
@@ -148,19 +148,19 @@ struct CacheImpl : Cache
         }
 
         auto infoJSON = stmt.getStr(0);
-        auto storePath = store->parseStorePath(stmt.getStr(1));
+        auto storePath = store.parseStorePath(stmt.getStr(1));
         auto locked = stmt.getInt(2) != 0;
         auto timestamp = stmt.getInt(3);
 
-        store->addTempRoot(storePath);
+        store.addTempRoot(storePath);
-        if (!store->isValidPath(storePath)) {
+        if (!store.isValidPath(storePath)) {
             // FIXME: we could try to substitute 'storePath'.
             debug("ignoring disappeared cache entry '%s'", inAttrsJSON);
             return {};
         }
 
         debug("using cache entry '%s' -> '%s', '%s'",
-            inAttrsJSON, infoJSON, store->printStorePath(storePath));
+            inAttrsJSON, infoJSON, store.printStorePath(storePath));
 
         return Result {
             .expired = !locked && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)),

src/libfetchers/cache.hh

@@ -50,14 +50,14 @@ struct Cache
 
     /* Old cache for things that have a store path. */
     virtual void add(
-        ref<Store> store,
+        Store & store,
         const Attrs & inAttrs,
         const Attrs & infoAttrs,
         const StorePath & storePath,
         bool locked) = 0;
 
     virtual std::optional<std::pair<Attrs, StorePath>> lookup(
-        ref<Store> store,
+        Store & store,
         const Attrs & inAttrs) = 0;
 
     struct Result
@@ -68,7 +68,7 @@ struct Cache
     };
 
     virtual std::optional<Result> lookupExpired(
-        ref<Store> store,
+        Store & store,
         const Attrs & inAttrs) = 0;
 };
 

src/libfetchers/fetch-to-store.cc

@@ -0,0 +1,68 @@
+#include "fetch-to-store.hh"
+#include "fetchers.hh"
+#include "cache.hh"
+
+namespace nix {
+
+StorePath fetchToStore(
+    Store & store,
+    const SourcePath & path,
+    std::string_view name,
+    ContentAddressMethod method,
+    PathFilter * filter,
+    RepairFlag repair)
+{
+    // FIXME: add an optimisation for the case where the accessor is
+    // an FSInputAccessor pointing to a store path.
+
+    std::optional<fetchers::Attrs> cacheKey;
+
+    if (!filter && path.accessor->fingerprint) {
+        cacheKey = fetchers::Attrs{
+            {"_what", "fetchToStore"},
+            {"store", store.storeDir},
+            {"name", std::string(name)},
+            {"fingerprint", *path.accessor->fingerprint},
+            {
+                "method",
+                std::visit(overloaded {
+                    [](const TextIngestionMethod &) {
+                        return "text";
+                    },
+                    [](const FileIngestionMethod & fim) {
+                        switch (fim) {
+                        case FileIngestionMethod::Flat: return "flat";
+                        case FileIngestionMethod::Recursive: return "nar";
+                        default: assert(false);
+                        }
+                    },
+                }, method.raw),
+            },
+            {"path", path.path.abs()}
+        };
+        if (auto res = fetchers::getCache()->lookup(store, *cacheKey)) {
+            debug("store path cache hit for '%s'", path);
+            return res->second;
+        }
+    } else
+        debug("source path '%s' is uncacheable", path);
+
+    Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", path));
+
+    auto filter2 = filter ? *filter : defaultPathFilter;
+
+    auto storePath =
+        settings.readOnlyMode
+        ? store.computeStorePath(
+            name, *path.accessor, path.path, method, HashAlgorithm::SHA256, {}, filter2).first
+        : store.addToStore(
+            name, *path.accessor, path.path, method, HashAlgorithm::SHA256, {}, filter2, repair);
+
+    if (cacheKey)
+        fetchers::getCache()->add(store, *cacheKey, {}, storePath, true);
+
+    return storePath;
+}
+
+
+}

src/libfetchers/fetch-to-store.hh

@@ -0,0 +1,22 @@
+#pragma once
+
+#include "source-path.hh"
+#include "store-api.hh"
+#include "file-system.hh"
+#include "repair-flag.hh"
+#include "file-content-address.hh"
+
+namespace nix {
+
+/**
+ * Copy the `path` to the Nix store.
+ */
+StorePath fetchToStore(
+    Store & store,
+    const SourcePath & path,
+    std::string_view name = "source",
+    ContentAddressMethod method = FileIngestionMethod::Recursive,
+    PathFilter * filter = nullptr,
+    RepairFlag repair = NoRepair);
+
+}