diff --git a/.version b/.version index ad2261920..5827d9bfd 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.24.0 +2.24.2 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 38f5d43b7..12423366a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -41,9 +41,9 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). There are many open pull requests that might already do what you intend to work on. You can use [labels](https://github.com/NixOS/nix/labels) to filter for relevant topics. -3. Check the [Nix reference manual](https://nixos.org/manual/nix/unstable/contributing/hacking.html) for information on building Nix and running its tests. +3. Check the [Nix reference manual](https://nix.dev/manual/nix/development/development/building.html) for information on building Nix and running its tests. - For contributions to the command line interface, please check the [CLI guidelines](https://nixos.org/manual/nix/unstable/contributing/cli-guideline.html). + For contributions to the command line interface, please check the [CLI guidelines](https://nix.dev/manual/nix/development/development/cli-guideline.html). 4. Make your change! @@ -69,7 +69,7 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). - [ ] API documentation in header files - [ ] Code and comments are self-explanatory - [ ] Commit message explains **why** the change was made - - [ ] New feature or incompatible change: [add a release note](https://nixos.org/manual/nix/stable/contributing/hacking#add-a-release-note) + - [ ] New feature or incompatible change: [add a release note](https://nix.dev/manual/nix/development/development/contributing.html#add-a-release-note) 7. If you need additional feedback or help to getting pull request into shape, ask other contributors using [@mentions](https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax#mentioning-people-and-teams). @@ -78,7 +78,7 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). The Nix reference manual is hosted on https://nixos.org/manual/nix. The underlying source files are located in [`doc/manual/src`](./doc/manual/src). For small changes you can [use GitHub to edit these files](https://docs.github.com/en/repositories/working-with-files/managing-files/editing-files) -For larger changes see the [Nix reference manual](https://nixos.org/manual/nix/unstable/contributing/hacking.html). +For larger changes see the [Nix reference manual](https://nix.dev/manual/nix/development/development/contributing.html). ## Getting help diff --git a/Makefile b/Makefile index bb64a104e..dbf510a3e 100644 --- a/Makefile +++ b/Makefile @@ -54,6 +54,7 @@ ifeq ($(ENABLE_FUNCTIONAL_TESTS), yes) ifdef HOST_UNIX makefiles += \ tests/functional/local.mk \ + tests/functional/flakes/local.mk \ tests/functional/ca/local.mk \ tests/functional/git-hashing/local.mk \ tests/functional/dyn-drv/local.mk \ @@ -92,7 +93,7 @@ ifdef HOST_WINDOWS GLOBAL_LDFLAGS += -Wl,--export-all-symbols endif -GLOBAL_CXXFLAGS += -g -Wall -Wdeprecated-copy -Wignored-qualifiers -Wimplicit-fallthrough -Werror=unused-result -include $(buildprefix)config.h -std=c++2a -I src +GLOBAL_CXXFLAGS += -g -Wall -Wdeprecated-copy -Wignored-qualifiers -Wimplicit-fallthrough -Werror=unused-result -Werror=suggest-override -include $(buildprefix)config.h -std=c++2a -I src # Include the main lib, causing rules to be defined diff --git a/README.md b/README.md index 931a60bba..021e54a3b 100644 --- a/README.md +++ b/README.md @@ -4,19 +4,18 @@ [![Test](https://github.com/NixOS/nix/workflows/Test/badge.svg)](https://github.com/NixOS/nix/actions) Nix is a powerful package manager for Linux and other Unix systems that makes package -management reliable and reproducible. Please refer to the [Nix manual](https://nixos.org/nix/manual) +management reliable and reproducible. Please refer to the [Nix manual](https://nix.dev/reference/nix-manual) for more details. ## Installation and first steps Visit [nix.dev](https://nix.dev) for [installation instructions](https://nix.dev/tutorials/install-nix) and [beginner tutorials](https://nix.dev/tutorials/first-steps). -Full reference documentation can be found in the [Nix manual](https://nixos.org/nix/manual). +Full reference documentation can be found in the [Nix manual](https://nix.dev/reference/nix-manual). ## Building and developing -See our [Hacking guide](https://nixos.org/manual/nix/unstable/contributing/hacking.html) in our manual for instruction on how to - set up a development environment and build Nix from source. +Follow instructions in the Nix reference manual to [set up a development environment and build Nix from source](https://nix.dev/manual/nix/development/building.html). ## Contributing diff --git a/build-utils-meson/diagnostics/meson.build b/build-utils-meson/diagnostics/meson.build index 2b79f6566..30eedfc13 100644 --- a/build-utils-meson/diagnostics/meson.build +++ b/build-utils-meson/diagnostics/meson.build @@ -1,13 +1,11 @@ add_project_arguments( - '-Wno-deprecated-declarations', - '-Wimplicit-fallthrough', + '-Wdeprecated-copy', + '-Werror=suggest-override', '-Werror=switch', '-Werror=switch-enum', '-Werror=unused-result', - '-Wdeprecated-copy', '-Wignored-qualifiers', - # Enable assertions in libstdc++ by default. Harmless on libc++. Benchmarked - # at ~1% overhead in `nix search`. - # + '-Wimplicit-fallthrough', + '-Wno-deprecated-declarations', language : 'cpp', ) diff --git a/configure.ac b/configure.ac index caeb88b67..cd931b87d 100644 --- a/configure.ac +++ b/configure.ac @@ -340,13 +340,6 @@ AC_CHECK_HEADERS([aws/s3/S3Client.h], AC_SUBST(ENABLE_S3, [$enable_s3]) AC_LANG_POP(C++) -if test -n "$enable_s3"; then - declare -a aws_version_tokens=($(printf '#include \nAWS_SDK_VERSION_STRING' | $CPP $CPPFLAGS - | grep -v '^#.*' | sed 's/"//g' | tr '.' ' ')) - AC_DEFINE_UNQUOTED([AWS_VERSION_MAJOR], ${aws_version_tokens@<:@0@:>@}, [Major version of aws-sdk-cpp.]) - AC_DEFINE_UNQUOTED([AWS_VERSION_MINOR], ${aws_version_tokens@<:@1@:>@}, [Minor version of aws-sdk-cpp.]) - AC_DEFINE_UNQUOTED([AWS_VERSION_PATCH], ${aws_version_tokens@<:@2@:>@}, [Patch version of aws-sdk-cpp.]) -fi - # Whether to use the Boehm garbage collector. AC_ARG_ENABLE(gc, AS_HELP_STRING([--enable-gc],[enable garbage collection in the Nix expression evaluator (requires Boehm GC) [default=yes]]), diff --git a/doc/manual/generate-builtins.nix b/doc/manual/generate-builtins.nix index 13de6c397..37ed12a43 100644 --- a/doc/manual/generate-builtins.nix +++ b/doc/manual/generate-builtins.nix @@ -12,7 +12,7 @@ let experimentalNotice = optionalString (experimental-feature != null) '' > **Note** > - > This function is only available if the [`${experimental-feature}` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-${experimental-feature}) is enabled. + > This function is only available if the [`${experimental-feature}` experimental feature](@docroot@/development/experimental-features.md#xp-feature-${experimental-feature}) is enabled. > > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): > diff --git a/doc/manual/generate-settings.nix b/doc/manual/generate-settings.nix index 504cda362..93a8e093e 100644 --- a/doc/manual/generate-settings.nix +++ b/doc/manual/generate-settings.nix @@ -33,10 +33,10 @@ let > **Warning** > > This setting is part of an - > [experimental feature](@docroot@/contributing/experimental-features.md). + > [experimental feature](@docroot@/development/experimental-features.md). > > To change this setting, make sure the - > [`${experimentalFeature}` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}) + > [`${experimentalFeature}` experimental feature](@docroot@/development/experimental-features.md#xp-feature-${experimentalFeature}) > is enabled. > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): > diff --git a/doc/manual/generate-store-info.nix b/doc/manual/generate-store-info.nix index c311c3c39..cc3704124 100644 --- a/doc/manual/generate-store-info.nix +++ b/doc/manual/generate-store-info.nix @@ -32,10 +32,10 @@ let > **Warning** > > This store is part of an - > [experimental feature](@docroot@/contributing/experimental-features.md). + > [experimental feature](@docroot@/development/experimental-features.md). > > To use this store, make sure the - > [`${experimentalFeature}` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}) + > [`${experimentalFeature}` experimental feature](@docroot@/development/experimental-features.md#xp-feature-${experimentalFeature}) > is enabled. > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): > diff --git a/doc/manual/generate-xp-features-shortlist.nix b/doc/manual/generate-xp-features-shortlist.nix index ec09f4b75..eb735ba5f 100644 --- a/doc/manual/generate-xp-features-shortlist.nix +++ b/doc/manual/generate-xp-features-shortlist.nix @@ -4,6 +4,6 @@ with import ; let showExperimentalFeature = name: doc: '' - - [`${name}`](@docroot@/contributing/experimental-features.md#xp-feature-${name}) + - [`${name}`](@docroot@/development/experimental-features.md#xp-feature-${name}) ''; in xps: indent " " (concatStrings (attrValues (mapAttrs showExperimentalFeature xps))) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index d4cba066b..76d09654c 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -95,7 +95,7 @@ $(d)/nix-profiles.5: $(d)/src/command-ref/files/profiles.md $(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@ @rm $^.tmp -$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md +$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/development/experimental-feature-descriptions.md @cp $< $@ @$(call process-includes,$@,$@) @@ -124,7 +124,7 @@ $(d)/conf-file.json: $(doc_nix) $(trace-gen) $(dummy-env) $(doc_nix) config show --json > $@.tmp @mv $@.tmp $@ -$(d)/src/contributing/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(doc_nix) +$(d)/src/development/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(doc_nix) @rm -rf $@ $@.tmp $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features.nix (builtins.fromJSON (builtins.readFile $<))' @mv $@.tmp $@ @@ -207,11 +207,11 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli done @touch $@ -# the `! -name 'contributing.md'` filter excludes the one place where +# the `! -name 'documentation.md'` filter excludes the one place where # `@docroot@` is to be preserved for documenting the mechanism # FIXME: maybe contributing guides should live right next to the code # instead of in the manual -$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/release-notes/rl-next.md $(d)/src/figures $(d)/src/favicon.png $(d)/src/favicon.svg +$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/development/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/release-notes/rl-next.md $(d)/src/figures $(d)/src/favicon.png $(d)/src/favicon.svg $(trace-gen) \ tmp="$$(mktemp -d)"; \ cp -r doc/manual "$$tmp"; \ diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index 0f9f91b03..beef6ef4a 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -143,7 +143,7 @@ const redirects = { "opt-timeout": "command-ref/opt-common.html#opt-timeout", "sec-common-options": "command-ref/opt-common.html", "ch-utilities": "command-ref/utilities.html", - "chap-hacking": "contributing/hacking.html", + "chap-hacking": "development/building.html", "adv-attr-allowSubstitutes": "language/advanced-attributes.html#adv-attr-allowSubstitutes", "adv-attr-allowedReferences": "language/advanced-attributes.html#adv-attr-allowedReferences", "adv-attr-allowedRequisites": "language/advanced-attributes.html#adv-attr-allowedRequisites", @@ -350,7 +350,7 @@ const redirects = { "macos": "uninstall.html#macos", "uninstalling": "uninstall.html", }, - "contributing/hacking.html": { + "development/building.html": { "nix-with-flakes": "#building-nix-with-flakes", "classic-nix": "#building-nix", "running-tests": "testing.html#running-tests", @@ -361,7 +361,12 @@ const redirects = { "installer-tests": "testing.html#installer-tests", "one-time-setup": "testing.html#one-time-setup", "using-the-ci-generated-installer-for-manual-testing": "testing.html#using-the-ci-generated-installer-for-manual-testing", - "characterization-testing": "#characterisation-testing-unit", + "characterization-testing": "testing.html#characterisation-testing-unit", + "add-a-release-note": "contributing.html#add-a-release-note", + "add-an-entry": "contributing.html#add-an-entry", + "build-process": "contributing.html#build-process", + "reverting": "contributing.html#reverting", + "branches": "contributing.html#branches", }, "glossary.html": { "gloss-local-store": "store/types/local-store.html", diff --git a/doc/manual/rl-next/drop-vendored-toml11.md b/doc/manual/rl-next/drop-vendored-toml11.md deleted file mode 100644 index d1feeb703..000000000 --- a/doc/manual/rl-next/drop-vendored-toml11.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -synopsis: Stop vendoring toml11 ---- - -We don't apply any patches to it, and vendoring it locks users into -bugs (it hasn't been updated since its introduction in late 2021). diff --git a/doc/manual/rl-next/harden-user-sandboxing.md b/doc/manual/rl-next/harden-user-sandboxing.md deleted file mode 100644 index a647acf25..000000000 --- a/doc/manual/rl-next/harden-user-sandboxing.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -synopsis: Harden the user sandboxing -significance: significant -issues: ---- - -The build directory has been hardened against interference with the outside world by nesting it inside another directory owned by (and only readable by) the daemon user. diff --git a/doc/manual/rl-next/nix-shell-looks-for-shell-nix.md b/doc/manual/rl-next/nix-shell-looks-for-shell-nix.md deleted file mode 100644 index 99be4148b..000000000 --- a/doc/manual/rl-next/nix-shell-looks-for-shell-nix.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -synopsis: "`nix-shell ` looks for `shell.nix`" -significance: significant -issues: -- 496 -- 2279 -- 4529 -- 5431 -- 11053 -prs: -- 11057 ---- - -`nix-shell $x` now looks for `$x/shell.nix` when `$x` resolves to a directory. - -Although this might be seen as a breaking change, its primarily interactive usage makes it a minor issue. -This adjustment addresses a commonly reported problem. - -This also applies to `nix-shell` shebang scripts. Consider the following example: - -```shell -#!/usr/bin/env nix-shell -#!nix-shell -i bash -``` - -This will now load `shell.nix` from the script's directory, if it exists; `default.nix` otherwise. - -The old behavior can be opted into by setting the option [`nix-shell-always-looks-for-shell-nix`](@docroot@/command-ref/conf-file.md#conf-nix-shell-always-looks-for-shell-nix) to `false`. diff --git a/doc/manual/rl-next/repl-doc-renders-doc-comments.md b/doc/manual/rl-next/repl-doc-renders-doc-comments.md deleted file mode 100644 index 05023697c..000000000 --- a/doc/manual/rl-next/repl-doc-renders-doc-comments.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -synopsis: "`nix-repl`'s `:doc` shows documentation comments" -significance: significant -issues: -- 3904 -- 10771 -prs: -- 1652 -- 9054 -- 11072 ---- - -`nix repl` has a `:doc` command that previously only rendered documentation for internally defined functions. -This feature has been extended to also render function documentation comments, in accordance with [RFC 145]. - -Example: - -``` -nix-repl> :doc lib.toFunction -Function toFunction - … defined at /home/user/h/nixpkgs/lib/trivial.nix:1072:5 - - Turns any non-callable values into constant functions. Returns - callable values as is. - -Inputs - - v - - : Any value - -Examples - - :::{.example} - -## lib.trivial.toFunction usage example - - | nix-repl> lib.toFunction 1 2 - | 1 - | - | nix-repl> lib.toFunction (x: x + 1) 2 - | 3 - - ::: -``` - -Known limitations: -- It does not render documentation for "formals", such as `{ /** the value to return */ x, ... }: x`. -- Some extensions to markdown are not yet supported, as you can see in the example above. - -We'd like to acknowledge Yingchi Long for proposing a proof of concept for this functionality in [#9054](https://github.com/NixOS/nix/pull/9054), as well as @sternenseemann and Johannes Kirschbauer for their contributions, proposals, and their work on [RFC 145]. - -[RFC 145]: https://github.com/NixOS/rfcs/pull/145 diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 56e0dbeec..b6d5b3c44 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -28,6 +28,7 @@ - [Data Types](language/types.md) - [String context](language/string-context.md) - [Syntax and semantics](language/syntax.md) + - [Identifiers](language/identifiers.md) - [Scoping rules](language/scope.md) - [String interpolation](language/string-interpolation.md) - [Lookup path](language/constructs/lookup-path.md) @@ -116,16 +117,18 @@ - [Flake Schemas](protocols/flake-schemas.md) - [C API](c-api.md) - [Glossary](glossary.md) -- [Contributing](contributing/index.md) - - [Hacking](contributing/hacking.md) - - [Testing](contributing/testing.md) - - [Documentation](contributing/documentation.md) - - [Experimental Features](contributing/experimental-features.md) - - [CLI guideline](contributing/cli-guideline.md) - - [JSON guideline](contributing/json-guideline.md) - - [C++ style guide](contributing/cxx.md) +- [Development](development/index.md) + - [Building](development/building.md) + - [Testing](development/testing.md) + - [Documentation](development/documentation.md) + - [CLI guideline](development/cli-guideline.md) + - [JSON guideline](development/json-guideline.md) + - [C++ style guide](development/cxx.md) + - [Experimental Features](development/experimental-features.md) + - [Contributing](development/contributing.md) - [Releases](release-notes/index.md) {{#include ./SUMMARY-rl-next.md}} + - [Release 2.24 (2024-07-31)](release-notes/rl-2.24.md) - [Release 2.23 (2024-06-03)](release-notes/rl-2.23.md) - [Release 2.22 (2024-04-23)](release-notes/rl-2.22.md) - [Release 2.21 (2024-03-11)](release-notes/rl-2.21.md) diff --git a/doc/manual/src/_redirects b/doc/manual/src/_redirects index 578c48f06..07b3130f9 100644 --- a/doc/manual/src/_redirects +++ b/doc/manual/src/_redirects @@ -20,7 +20,15 @@ /command-ref/command-ref /command-ref 301! -/contributing/contributing /contributing 301! +/contributing/contributing /development 301! +/contributing /development 301! +/contributing/hacking /development/building 301! +/contributing/testing /development/testing 301! +/contributing/documentation /development/documentation 301! +/contributing/experimental-features /development/experimental-features 301! +/contributing/cli-guideline /development/cli-guideline 301! +/contributing/json-guideline /development/json-guideline 301! +/contributing/cxx /development/cxx 301! /expressions/expression-language /language/ 301! /expressions/language-constructs /language/constructs 301! diff --git a/doc/manual/src/c-api.md b/doc/manual/src/c-api.md index 29df0b644..0cdd83832 100644 --- a/doc/manual/src/c-api.md +++ b/doc/manual/src/c-api.md @@ -10,7 +10,7 @@ See: - [Matrix Room *Nix Bindings*](https://matrix.to/#/#nix-bindings:nixos.org) for discussion and questions. - [Stabilisation Milestone](https://github.com/NixOS/nix/milestone/52) - [Other C API PRs and issues](https://github.com/NixOS/nix/labels/c%20api) -- [Contributing C API Documentation](contributing/documentation.md#c-api-documentation), including how to build it locally. +- [Contributing C API Documentation](development/documentation.md#c-api-documentation), including how to build it locally. [Getting Started]: https://hydra.nixos.org/job/nix/master/external-api-docs/latest/download-by-type/doc/external-api-docs [Index]: https://hydra.nixos.org/job/nix/master/external-api-docs/latest/download-by-type/doc/external-api-docs/globals.html diff --git a/doc/manual/src/command-ref/env-common.md b/doc/manual/src/command-ref/env-common.md index d3f5f9c14..0b5017882 100644 --- a/doc/manual/src/command-ref/env-common.md +++ b/doc/manual/src/command-ref/env-common.md @@ -9,22 +9,26 @@ Most Nix commands interpret the following environment variables: - [`NIX_PATH`](#env-NIX_PATH) - A colon-separated list of directories used to look up the location of Nix - expressions using [paths](@docroot@/language/types.md#type-path) - enclosed in angle brackets (i.e., ``), - e.g. `/home/eelco/Dev:/etc/nixos`. It can be extended using the - [`-I` option](@docroot@/command-ref/opt-common.md#opt-I). + A colon-separated list of search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). - If `NIX_PATH` is not set at all, Nix will fall back to the following list in [impure](@docroot@/command-ref/conf-file.md#conf-pure-eval) and [unrestricted](@docroot@/command-ref/conf-file.md#conf-restrict-eval) evaluation mode: + This environment variable overrides the value of the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path). - 1. `$HOME/.nix-defexpr/channels` - 2. `nixpkgs=/nix/var/nix/profiles/per-user/root/channels/nixpkgs` - 3. `/nix/var/nix/profiles/per-user/root/channels` + It can be extended using the [`-I` option](@docroot@/command-ref/opt-common.md#opt-I). + + > **Example** + > + > ```bash + > $ export NIX_PATH=`/home/eelco/Dev:nixos-config=/etc/nixos + > ``` If `NIX_PATH` is set to an empty string, resolving search paths will always fail. - For example, attempting to use `` will produce: - error: file 'nixpkgs' was not found in the Nix search path + > **Example** + > + > ```bash + > $ NIX_PATH= nix-instantiate --eval '' + > error: file 'nixpkgs' was not found in the Nix search path (add it using $NIX_PATH or -I) + > ``` - [`NIX_IGNORE_SYMLINK_STORE`](#env-NIX_IGNORE_SYMLINK_STORE) diff --git a/doc/manual/src/command-ref/experimental-commands.md b/doc/manual/src/command-ref/experimental-commands.md index 286ddc6d6..1190729a2 100644 --- a/doc/manual/src/command-ref/experimental-commands.md +++ b/doc/manual/src/command-ref/experimental-commands.md @@ -1,6 +1,6 @@ # Experimental Commands -This section lists [experimental commands](@docroot@/contributing/experimental-features.md#xp-feature-nix-command). +This section lists [experimental commands](@docroot@/development/experimental-features.md#xp-feature-nix-command). > **Warning** > diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index ddec30f5b..69a711bd5 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -297,3 +297,8 @@ with import {}; runCommand "dummy" { buildInputs = [ python pythonPackages.prettytable ]; } "" ``` + +The script's file name is passed as the first argument to the interpreter specified by the `-i` flag. + +Aside from the very first line, which is a directive to the operating system, the additional `#! nix-shell` lines do not need to be at the beginning of the file. +This allows wrapping them in block comments for languages where `#` does not start a comment, such as ECMAScript, Erlang, PHP, or Ruby. diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index e30b351a4..a899758df 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -32,7 +32,7 @@ If no substitutes are available and no store derivation is given, realisation fa [store objects]: @docroot@/store/store-object.md [closure]: @docroot@/glossary.md#gloss-closure [substituters]: @docroot@/command-ref/conf-file.md#conf-substituters -[content-addressed derivations]: @docroot@/contributing/experimental-features.md#xp-feature-ca-derivations +[content-addressed derivations]: @docroot@/development/experimental-features.md#xp-feature-ca-derivations [Nix database]: @docroot@/glossary.md#gloss-nix-database The resulting paths are printed on standard output. diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index a42909e2d..69a700207 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -37,7 +37,7 @@ Most Nix commands accept the following command-line options: Print even more informational messages. - `4` “Debug” - + Print debug information. - `5` “Vomit” @@ -187,11 +187,12 @@ Most Nix commands accept the following command-line options: For `nix-shell`, this option is commonly used to give you a shell in which you can build the packages returned by the expression. If you want to get a shell which contain the *built* packages ready for use, give your expression to the `nix-shell --packages ` convenience flag instead. -- [`-I`](#opt-I) *path* +- [`-I` / `--include`](#opt-I) *path* - Add an entry to the [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path). + Add an entry to the list of search paths used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). This option may be given multiple times. - Paths added through `-I` take precedence over [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH). + + Paths added through `-I` take precedence over the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path) and the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH). - [`--option`](#opt-option) *name* *value* diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/development/building.md similarity index 65% rename from doc/manual/src/contributing/hacking.md rename to doc/manual/src/development/building.md index 451b38976..8b7046377 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/development/building.md @@ -1,18 +1,4 @@ -# Hacking - -This section provides some notes on how to hack on Nix. To get the -latest version of Nix from GitHub: - -```console -$ git clone https://github.com/NixOS/nix.git -$ cd nix -``` - -The following instructions assume you already have some version of Nix installed locally, so that you can use it to set up the development environment. If you don't have it installed, follow the [installation instructions]. - -[installation instructions]: ../installation/index.md - -## Building Nix with flakes +## Building Nix To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: @@ -61,50 +47,6 @@ $ nix build You can also build Nix for one of the [supported platforms](#platforms). -## Building Nix - -To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: - -```console -$ nix-shell -``` - -To get a shell with one of the other [supported compilation environments](#compilation-environments): - -```console -$ nix-shell --attr devShells.x86_64-linux.native-clangStdenvPackages -``` - -> **Note** -> -> You can use `native-ccacheStdenvPackages` to drastically improve rebuild time. -> By default, [ccache](https://ccache.dev) keeps artifacts in `~/.cache/ccache/`. - -To build Nix itself in this shell: - -```console -[nix-shell]$ autoreconfPhase -[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out -[nix-shell]$ make -j $NIX_BUILD_CORES -``` - -To install it in `$(pwd)/outputs` and test it: - -```console -[nix-shell]$ make install -[nix-shell]$ make installcheck -j $NIX_BUILD_CORES -[nix-shell]$ ./outputs/out/bin/nix --version -nix (Nix) 2.12 -``` - -To build a release version of Nix for the current operating system and CPU architecture: - -```console -$ nix build -``` - -You can also build Nix for one of the [supported platforms](#platforms). - ## Makefile variables You may need `profiledir=$out/etc/profile.d` and `sysconfdir=$out/etc` to run `make install`. @@ -274,81 +216,3 @@ If it fails, run `git add --patch` to approve the suggestions _and commit again_ To refresh pre-commit hook's config file, do the following: 1. Exit the development shell and start it again by running `nix develop`. 2. If you also use the pre-commit hook, also run `pre-commit-hooks-install` again. - -## Add a release note - -`doc/manual/rl-next` contains release notes entries for all unreleased changes. - -User-visible changes should come with a release note. - -### Add an entry - -Here's what a complete entry looks like. The file name is not incorporated in the document. - -``` ---- -synopsis: Basically a title -issues: 1234 -prs: 1238 ---- - -Here's one or more paragraphs that describe the change. - -- It's markdown -- Add references to the manual using @docroot@ -``` - -Significant changes should add the following header, which moves them to the top. - -``` -significance: significant -``` - - -See also the [format documentation](https://github.com/haskell/cabal/blob/master/CONTRIBUTING.md#changelog). - -### Build process - -Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`. - -## Branches - -- [`master`](https://github.com/NixOS/nix/commits/master) - - The main development branch. All changes are approved and merged here. - When developing a change, create a branch based on the latest `master`. - - Maintainers try to [keep it in a release-worthy state](#reverting). - -- [`maintenance-*.*`](https://github.com/NixOS/nix/branches/all?query=maintenance) - - These branches are the subject of backports only, and are - also [kept](#reverting) in a release-worthy state. - - See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) - -- [`latest-release`](https://github.com/NixOS/nix/tree/latest-release) - - The latest patch release of the latest minor version. - - See [`maintainers/release-process.md`](https://github.com/NixOS/nix/blob/master/maintainers/release-process.md) - -- [`backport-*-to-*`](https://github.com/NixOS/nix/branches/all?query=backport) - - Generally branches created by the backport action. - - See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) - -- [_other_](https://github.com/NixOS/nix/branches/all) - - Branches that do not conform to the above patterns should be feature branches. - -## Reverting - -If a change turns out to be merged by mistake, or contain a regression, it may be reverted. -A revert is not a rejection of the contribution, but merely part of an effective development process. -It makes sure that development keeps running smoothly, with minimal uncertainty, and less overhead. -If maintainers have to worry too much about avoiding reverts, they would not be able to merge as much. -By embracing reverts as a good part of the development process, everyone wins. - -However, taking a step back may be frustrating, so maintainers will be extra supportive on the next try. diff --git a/doc/manual/src/contributing/cli-guideline.md b/doc/manual/src/development/cli-guideline.md similarity index 100% rename from doc/manual/src/contributing/cli-guideline.md rename to doc/manual/src/development/cli-guideline.md diff --git a/doc/manual/src/development/contributing.md b/doc/manual/src/development/contributing.md new file mode 100644 index 000000000..7de7489dc --- /dev/null +++ b/doc/manual/src/development/contributing.md @@ -0,0 +1,79 @@ +# Contributing + +## Add a release note + +`doc/manual/rl-next` contains release notes entries for all unreleased changes. + +User-visible changes should come with a release note. + +### Add an entry + +Here's what a complete entry looks like. The file name is not incorporated in the document. + +``` +--- +synopsis: Basically a title +issues: 1234 +prs: 1238 +--- + +Here's one or more paragraphs that describe the change. + +- It's markdown +- Add references to the manual using @docroot@ +``` + +Significant changes should add the following header, which moves them to the top. + +``` +significance: significant +``` + + +See also the [format documentation](https://github.com/haskell/cabal/blob/master/CONTRIBUTING.md#changelog). + +### Build process + +Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`. + +## Branches + +- [`master`](https://github.com/NixOS/nix/commits/master) + + The main development branch. All changes are approved and merged here. + When developing a change, create a branch based on the latest `master`. + + Maintainers try to [keep it in a release-worthy state](#reverting). + +- [`maintenance-*.*`](https://github.com/NixOS/nix/branches/all?query=maintenance) + + These branches are the subject of backports only, and are + also [kept](#reverting) in a release-worthy state. + + See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) + +- [`latest-release`](https://github.com/NixOS/nix/tree/latest-release) + + The latest patch release of the latest minor version. + + See [`maintainers/release-process.md`](https://github.com/NixOS/nix/blob/master/maintainers/release-process.md) + +- [`backport-*-to-*`](https://github.com/NixOS/nix/branches/all?query=backport) + + Generally branches created by the backport action. + + See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) + +- [_other_](https://github.com/NixOS/nix/branches/all) + + Branches that do not conform to the above patterns should be feature branches. + +## Reverting + +If a change turns out to be merged by mistake, or contain a regression, it may be reverted. +A revert is not a rejection of the contribution, but merely part of an effective development process. +It makes sure that development keeps running smoothly, with minimal uncertainty, and less overhead. +If maintainers have to worry too much about avoiding reverts, they would not be able to merge as much. +By embracing reverts as a good part of the development process, everyone wins. + +However, taking a step back may be frustrating, so maintainers will be extra supportive on the next try. diff --git a/doc/manual/src/contributing/cxx.md b/doc/manual/src/development/cxx.md similarity index 100% rename from doc/manual/src/contributing/cxx.md rename to doc/manual/src/development/cxx.md diff --git a/doc/manual/src/contributing/documentation.md b/doc/manual/src/development/documentation.md similarity index 99% rename from doc/manual/src/contributing/documentation.md rename to doc/manual/src/development/documentation.md index a14ecedd6..63f574ab7 100644 --- a/doc/manual/src/contributing/documentation.md +++ b/doc/manual/src/development/documentation.md @@ -24,7 +24,7 @@ nix build .#^doc and open `./result-doc/share/doc/nix/manual/index.html`. -To build the manual incrementally, [enter the development shell](./hacking.md) and run: +To build the manual incrementally, [enter the development shell](./building.md) and run: ```console make manual-html-open -j $NIX_BUILD_CORES diff --git a/doc/manual/src/contributing/experimental-features.md b/doc/manual/src/development/experimental-features.md similarity index 100% rename from doc/manual/src/contributing/experimental-features.md rename to doc/manual/src/development/experimental-features.md diff --git a/doc/manual/src/contributing/index.md b/doc/manual/src/development/index.md similarity index 77% rename from doc/manual/src/contributing/index.md rename to doc/manual/src/development/index.md index 4d55c17a4..6403c3e66 100644 --- a/doc/manual/src/contributing/index.md +++ b/doc/manual/src/development/index.md @@ -5,4 +5,4 @@ Check the [contributing guide](https://github.com/NixOS/nix/blob/master/CONTRIBU This chapter is a collection of guides for making changes to the code and documentation. -If you're not sure where to start, try to [compile Nix from source](./hacking.md) and consider [making improvements to documentation](./documentation.md). +If you're not sure where to start, try to [compile Nix from source](./building.md) and consider [making improvements to documentation](./documentation.md). diff --git a/doc/manual/src/contributing/json-guideline.md b/doc/manual/src/development/json-guideline.md similarity index 100% rename from doc/manual/src/contributing/json-guideline.md rename to doc/manual/src/development/json-guideline.md diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/development/testing.md similarity index 100% rename from doc/manual/src/contributing/testing.md rename to doc/manual/src/development/testing.md diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index f65ada63a..877c4668b 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -168,7 +168,7 @@ - [impure derivation]{#gloss-impure-derivation} - [An experimental feature](#@docroot@/contributing/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, + [An experimental feature](#@docroot@/development/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, so that they are always rebuilt, and their outputs not reused by subsequent calls to realise them. - [Nix database]{#gloss-nix-database} @@ -353,7 +353,7 @@ Not yet stabilized functionality guarded by named experimental feature flags. These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting. - See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/contributing/experimental-features.md). + See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/development/experimental-features.md). [Nix language]: ./language/index.md diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index 385008d8c..6a168ff3d 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -77,7 +77,7 @@ $ su root # Installing from a binary tarball You can also download a binary tarball that contains Nix and all its dependencies: -- Choose a [version](https://releases.nixos.org/?prefix=nix/) and [system type](../contributing/hacking.md#platforms) +- Choose a [version](https://releases.nixos.org/?prefix=nix/) and [system type](../development/building.md#platforms) - Download and unpack the tarball - Run the installer diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index e916c7360..51b83fc8a 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -113,7 +113,7 @@ Derivations can declare some infrequently used optional attributes. > `nix-build`. If the [`configurable-impure-env` experimental - feature](@docroot@/contributing/experimental-features.md#xp-feature-configurable-impure-env) + feature](@docroot@/development/experimental-features.md#xp-feature-configurable-impure-env) is enabled, these environment variables can also be controlled through the [`impure-env`](@docroot@/command-ref/conf-file.md#conf-impure-env) @@ -226,7 +226,7 @@ Derivations can declare some infrequently used optional attributes. - [`__contentAddressed`]{#adv-attr-__contentAddressed} > **Warning** - > This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md). + > This attribute is part of an [experimental feature](@docroot@/development/experimental-features.md). > > To use this attribute, you must enable the > [`ca-derivations`][xp-feature-ca-derivations] experimental feature. @@ -370,6 +370,6 @@ Derivations can declare some infrequently used optional attributes. ensures that the derivation can only be built on a machine with the `kvm` feature. -[xp-feature-ca-derivations]: @docroot@/contributing/experimental-features.md#xp-feature-ca-derivations -[xp-feature-dynamic-derivations]: @docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations -[xp-feature-git-hashing]: @docroot@/contributing/experimental-features.md#xp-feature-git-hashing +[xp-feature-ca-derivations]: @docroot@/development/experimental-features.md#xp-feature-ca-derivations +[xp-feature-dynamic-derivations]: @docroot@/development/experimental-features.md#xp-feature-dynamic-derivations +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/doc/manual/src/language/identifiers.md b/doc/manual/src/language/identifiers.md new file mode 100644 index 000000000..c9e981da6 --- /dev/null +++ b/doc/manual/src/language/identifiers.md @@ -0,0 +1,50 @@ +# Identifiers + +An *identifier* is an [ASCII](https://en.wikipedia.org/wiki/ASCII) character sequence that: +- Starts with a letter (`a-z`, `A-Z`) or underscore (`_`) +- Can contain any number of: + - Letters (`a-z`, `A-Z`) + - Digits (`0-9`) + - Underscores (`_`) + - Apostrophes (`'`) + - Hyphens (`-`) +- Is not one of the [keywords](#keywords) + +> **Syntax** +> +> *identifier* ~ `[A-Za-z_][A-Za-z0-9_'-]*` + +# Names + +A name can be an [identifier](#identifier) or a [string literal](./syntax.md#string-literal). + +> **Syntax** +> +> *name* → *identifier* | *string* + +Names are used in [attribute sets](./syntax.md#attrs-literal), [`let` bindings](./syntax.md#let-expressions), and [`inherit`](./syntax.md#inheriting attributes). + +# Keywords + +These keywords are reserved and cannot be used as [identifiers](#identifiers): + +- [`assert`](./syntax.md#assertions) +- [`else`][if] +- [`if`][if] +- [`in`][let] +- [`inherit`](./syntax.md#inheriting-attributes) +- [`let`][let] +- [`or`](./operators.md#attribute-selection) (see note) +- [`rec`](./syntax.md#recursive-sets) +- [`then`][if] +- [`with`](./syntax.md#with-expressions) + +[if]: ./syntax.md#conditionals +[let]: ./syntax.md#let-expressions + +> **Note** +> +> The Nix language evaluator currently allows `or` to be used as a name in some contexts, for backwards compatibility reasons. +> Users are advised not to rely on this. +> +> There are long-standing issues with how `or` is parsed as a name, which can't be resolved without making a breaking change to the language. diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index 9660a764d..e1c020781 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -26,13 +26,17 @@ | Logical conjunction (`AND`) | *bool* `&&` *bool* | left | 12 | | Logical disjunction (`OR`) | *bool* \|\| *bool* | left | 13 | | [Logical implication] | *bool* `->` *bool* | right | 14 | +| [Pipe operator] (experimental) | *expr* `\|>` *func* | left | 15 | +| [Pipe operator] (experimental) | *func* `<\|` *expr* | right | 15 | [string]: ./types.md#type-string [path]: ./types.md#type-path -[number]: ./types.md#type-float +[number]: ./types.md#type-float [list]: ./types.md#list [attribute set]: ./types.md#attribute-set + + ## Attribute selection > **Syntax** @@ -42,12 +46,6 @@ Select the attribute denoted by attribute path *attrpath* from [attribute set] *attrset*. If the attribute doesn’t exist, return the *expr* after `or` if provided, otherwise abort evaluation. -An attribute path is a dot-separated list of [attribute names](./types.md#attribute-set). - -> **Syntax** -> -> *attrpath* = *name* [ `.` *name* ]... - [Attribute selection]: #attribute-selection ## Has attribute @@ -182,3 +180,34 @@ Equivalent to `!`*b1* `||` *b2*. [Logical implication]: #logical-implication +## Pipe operators + +- *a* `|>` *b* is equivalent to *b* *a* +- *a* `<|` *b* is equivalent to *a* *b* + +> **Example** +> +> ``` +> nix-repl> 1 |> builtins.add 2 |> builtins.mul 3 +> 9 +> +> nix-repl> builtins.add 1 <| builtins.mul 2 <| 3 +> 7 +> ``` + +> **Warning** +> +> This syntax is part of an +> [experimental feature](@docroot@/development/experimental-features.md) +> and may change in future releases. +> +> To use this syntax, make sure the +> [`pipe-operators` experimental feature](@docroot@/development/experimental-features.md#xp-feature-pipe-operators) +> is enabled. +> For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): +> +> ``` +> extra-experimental-features = pipe-operators +> ``` + +[Pipe operator]: #pipe-operators diff --git a/doc/manual/src/language/syntax.md b/doc/manual/src/language/syntax.md index 238c502f9..6108bacd6 100644 --- a/doc/manual/src/language/syntax.md +++ b/doc/manual/src/language/syntax.md @@ -190,18 +190,13 @@ This section covers syntax and semantics of the Nix language. ### Path {#path-literal} - *Paths* are distinct from strings and can be expressed by path literals such as `./builder.sh`. - - Paths are suitable for referring to local files, and are often preferable over strings. - - Path values do not contain trailing slashes, `.` and `..`, as they are resolved when evaluating a path literal. - - Path literals are automatically resolved relative to their [base directory](@docroot@/glossary.md#gloss-base-directory). - - The files referred to by path values are automatically copied into the Nix store when used in a string interpolation or concatenation. - - Tooling can recognize path literals and provide additional features, such as autocompletion, refactoring automation and jump-to-file. + *Paths* can be expressed by path literals such as `./builder.sh`. A path literal must contain at least one slash to be recognised as such. For instance, `builder.sh` is not a path: it's parsed as an expression that selects the attribute `sh` from the variable `builder`. + Path literals are resolved relative to their [base directory](@docroot@/glossary.md#gloss-base-directory). Path literals may also refer to absolute paths by starting with a slash. > **Note** @@ -215,15 +210,6 @@ This section covers syntax and semantics of the Nix language. For example, `~/foo` would be equivalent to `/home/edolstra/foo` for a user whose home directory is `/home/edolstra`. Path literals that start with `~` are not allowed in [pure](@docroot@/command-ref/conf-file.md#conf-pure-eval) evaluation. - Paths can be used in [string interpolation] and string concatenation. - For instance, evaluating `"${./foo.txt}"` will cause `foo.txt` from the same directory to be copied into the Nix store and result in the string `"/nix/store/-foo.txt"`. - - Note that the Nix language assumes that all input files will remain _unchanged_ while evaluating a Nix expression. - For example, assume you used a file path in an interpolated string during a `nix repl` session. - Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new [store path], since Nix might not re-read the file contents. Use `:r` to reset the repl as needed. - - [store path]: @docroot@/store/store-path.md - Path literals can also include [string interpolation], besides being [interpolated into other expressions]. [interpolated into other expressions]: ./string-interpolation.md#interpolated-expressions @@ -261,37 +247,76 @@ Elements in a list can be accessed using [`builtins.elemAt`](./builtins.md#built ## Attribute Set {#attrs-literal} -An attribute set is a collection of name-value-pairs (called *attributes*) enclosed in curly brackets (`{ }`). +An attribute set is a collection of name-value-pairs called *attributes*. -An attribute name can be an identifier or a [string](#string). -An identifier must start with a letter (`a-z`, `A-Z`) or underscore (`_`), and can otherwise contain letters (`a-z`, `A-Z`), numbers (`0-9`), underscores (`_`), apostrophes (`'`), or dashes (`-`). +Attribute sets are written enclosed in curly brackets (`{ }`). +Attribute names and attribute values are separated by an equal sign (`=`). +Each value can be an arbitrary expression, terminated by a semicolon (`;`) + +An attribute name is a string without context, and is denoted by a [name] (an [identifier](./identifiers.md#identifiers) or [string literal](#string-literal)). + +[name]: ./identifiers.md#names > **Syntax** > -> *name* = *identifier* | *string* \ -> *identifier* ~ `[a-zA-Z_][a-zA-Z0-9_'-]*` - -Names and values are separated by an equal sign (`=`). -Each value is an arbitrary expression terminated by a semicolon (`;`). - -> **Syntax** -> -> *attrset* = `{` [ *name* `=` *expr* `;` ]... `}` +> *attrset* → `{` { *name* `=` *expr* `;` } `}` Attributes can appear in any order. -An attribute name may only occur once. +An attribute name may only occur once in each attribute set. -Example: +> **Example** +> +> This defines an attribute set with attributes named: +> - `x` with the value `123`, an integer +> - `text` with the value `"Hello"`, a string +> - `y` where the value is the result of applying the function `f` to the attribute set `{ bla = 456; }` +> +> ```nix +> { +> x = 123; +> text = "Hello"; +> y = f { bla = 456; }; +> } +> ``` -```nix -{ - x = 123; - text = "Hello"; - y = f { bla = 456; }; -} -``` +Attributes in nested attribute sets can be written using *attribute paths*. -This defines a set with attributes named `x`, `text`, `y`. +> **Syntax** +> +> *attrset* → `{` { *attrpath* `=` *expr* `;` } `}` + +An attribute path is a dot-separated list of [names][name]. + +> **Syntax** +> +> *attrpath* = *name* { `.` *name* } + + + +> **Example** +> +> ```nix +> { a.b.c = 1; a.b.d = 2; } +> ``` +> +> { +> a = { +> b = { +> c = 1; +> d = 2; +> }; +> }; +> } + +Attribute names can also be set implicitly by using the [`inherit` keyword](#inheriting-attributes). + +> **Example** +> +> ```nix +> { inherit (builtins) true; } +> ``` +> +> { true = true; } Attributes can be accessed with the [`.` operator](./operators.md#attribute-selection). diff --git a/doc/manual/src/language/types.md b/doc/manual/src/language/types.md index c6cfb3c69..229756e6b 100644 --- a/doc/manual/src/language/types.md +++ b/doc/manual/src/language/types.md @@ -50,8 +50,37 @@ The function [`builtins.isString`](builtins.md#builtins-isString) can be used to ### Path {#type-path} - +A _path_ in the Nix language is an immutable, finite-length sequence of bytes starting with `/`, representing a POSIX-style, canonical file system path. +Path values are distinct from string values, even if they contain the same sequence of bytes. +Operations that produce paths will simplify the result as the standard C function [`realpath`] would, except that there is no symbolic link resolution. +[`realpath`]: https://pubs.opengroup.org/onlinepubs/9699919799/functions/realpath.html + +Paths are suitable for referring to local files, and are often preferable over strings. +- Path values do not contain trailing or duplicate slashes, `.`, or `..`. +- Relative path literals are automatically resolved relative to their [base directory]. +- Tooling can recognize path literals and provide additional features, such as autocompletion, refactoring automation and jump-to-file. + +[base directory]: @docroot@/glossary.md#gloss-base-directory + +A file is not required to exist at a given path in order for that path value to be valid, but a path that is converted to a string with [string interpolation] or [string-and-path concatenation] must resolve to a readable file or directory which will be copied into the Nix store. +For instance, evaluating `"${./foo.txt}"` will cause `foo.txt` from the same directory to be copied into the Nix store and result in the string `"/nix/store/-foo.txt"`. +Operations such as [`import`] can also expect a path to resolve to a readable file or directory. + +[string interpolation]: string-interpolation.md#interpolated-expression +[string-and-path concatenation]: operators.md#string-and-path-concatenation +[`import`]: builtins.md#builtins-import + +> **Note** +> +> The Nix language assumes that all input files will remain _unchanged_ while evaluating a Nix expression. +> For example, assume you used a file path in an interpolated string during a `nix repl` session. +> Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new [store path], since Nix might not re-read the file contents. +> Use `:r` to reset the repl as needed. + +[store path]: @docroot@/store/store-path.md + +Path values can be expressed as [path literals](syntax.md#path-literal). The function [`builtins.isPath`](builtins.md#builtins-isPath) can be used to determine if a value is a path. ### Null {#type-null} diff --git a/doc/manual/src/protocols/derivation-aterm.md b/doc/manual/src/protocols/derivation-aterm.md index e58b602a3..1ba757ae0 100644 --- a/doc/manual/src/protocols/derivation-aterm.md +++ b/doc/manual/src/protocols/derivation-aterm.md @@ -14,6 +14,6 @@ Derivations are serialised in one of the following formats: DrvWithVersion(, ...) ``` - The only `version-string`s that are in use today are for [experimental features](@docroot@/contributing/experimental-features.md): + The only `version-string`s that are in use today are for [experimental features](@docroot@/development/experimental-features.md): - - `"xp-dyn-drv"` for the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. + - `"xp-dyn-drv"` for the [`dynamic-derivations`](@docroot@/development/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. diff --git a/doc/manual/src/protocols/tarball-fetcher.md b/doc/manual/src/protocols/tarball-fetcher.md index 24ec7ae14..5cff05d66 100644 --- a/doc/manual/src/protocols/tarball-fetcher.md +++ b/doc/manual/src/protocols/tarball-fetcher.md @@ -41,4 +41,30 @@ Link: ///archive/.tar.gz +``` + +> **Example** +> +> +> ```nix +> # flake.nix +> { +> inputs = { +> foo.url = "https://gitea.example.org/some-person/some-flake/archive/main.tar.gz"; +> bar.url = "https://gitea.example.org/some-other-person/other-flake/archive/442793d9ec0584f6a6e82fa253850c8085bb150a.tar.gz"; +> qux = { +> url = "https://forgejo.example.org/another-person/some-non-flake-repo/archive/development.tar.gz"; +> flake = false; +> }; +> }; +> outputs = { foo, bar, qux }: { /* ... */ }; +> } +``` + [Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive diff --git a/doc/manual/src/release-notes/rl-2.18.md b/doc/manual/src/release-notes/rl-2.18.md index 4bbc52b50..eb26fc9e7 100644 --- a/doc/manual/src/release-notes/rl-2.18.md +++ b/doc/manual/src/release-notes/rl-2.18.md @@ -13,7 +13,7 @@ - The `discard-references` feature has been stabilized. This means that the - [unsafeDiscardReferences](@docroot@/contributing/experimental-features.md#xp-feature-discard-references) + [unsafeDiscardReferences](@docroot@/development/experimental-features.md#xp-feature-discard-references) attribute is no longer guarded by an experimental flag and can be used freely. @@ -21,7 +21,7 @@ This only affects `nix-build --json` when "building" non-derivation things like fetched sources, which is a no-op. - A new builtin [`outputOf`](@docroot@/language/builtins.md#builtins-outputOf) has been added. - It is part of the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. + It is part of the [`dynamic-derivations`](@docroot@/development/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. - Flake follow paths at depths greater than 2 are now handled correctly, preventing "follows a non-existent input" errors. diff --git a/doc/manual/src/release-notes/rl-2.19.md b/doc/manual/src/release-notes/rl-2.19.md index ba6eb9c64..e2e2f85cc 100644 --- a/doc/manual/src/release-notes/rl-2.19.md +++ b/doc/manual/src/release-notes/rl-2.19.md @@ -17,8 +17,8 @@ - `nix-shell` shebang lines now support single-quoted arguments. -- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). - This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). +- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/development/experimental-features.md#xp-fetch-tree). + This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/development/experimental-features.md#xp-fetch-tree). - The interface for creating and updating lock files has been overhauled: @@ -33,7 +33,7 @@ - The flake-specific flags `--recreate-lock-file` and `--update-input` have been removed from all commands operating on installables. They are superceded by `nix flake update`. -- Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). +- Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches). - [`nix path-info --json`](@docroot@/command-ref/new-cli/nix3-path-info.md) (experimental) now returns a JSON map rather than JSON list. diff --git a/doc/manual/src/release-notes/rl-2.23.md b/doc/manual/src/release-notes/rl-2.23.md index 3c59b8583..ac842fdc0 100644 --- a/doc/manual/src/release-notes/rl-2.23.md +++ b/doc/manual/src/release-notes/rl-2.23.md @@ -14,7 +14,7 @@ - Modify `nix derivation {add,show}` JSON format [#9866](https://github.com/NixOS/nix/issues/9866) [#10722](https://github.com/NixOS/nix/pull/10722) - The JSON format for derivations has been slightly revised to better conform to our [JSON guidelines](@docroot@/contributing/cli-guideline.md#returning-future-proof-json). + The JSON format for derivations has been slightly revised to better conform to our [JSON guidelines](@docroot@/development/cli-guideline.md#returning-future-proof-json). In particular, the hash algorithm and content addressing method of content-addresed derivation outputs are now separated into two fields `hashAlgo` and `method`, rather than one field with an arcane `:`-separated format. @@ -89,7 +89,7 @@ This makes records of this sort more self-describing, and easier to consume programmatically. We will follow this design principle going forward; - the [JSON guidelines](@docroot@/contributing/json-guideline.md) in the contributing section have been updated accordingly. + the [JSON guidelines](@docroot@/development/json-guideline.md) in the contributing section have been updated accordingly. - Large path warnings [#10661](https://github.com/NixOS/nix/pull/10661) diff --git a/doc/manual/src/release-notes/rl-2.24.md b/doc/manual/src/release-notes/rl-2.24.md new file mode 100644 index 000000000..5bcc1d79c --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.24.md @@ -0,0 +1,324 @@ +# Release 2.24.0 (2024-07-31) + +### Significant changes + +- Harden user sandboxing + + The build directory has been hardened against interference with the outside world by nesting it inside another directory owned by (and only readable by) the daemon user. + + This is a low severity security fix, [CVE-2024-38531](https://www.cve.org/CVERecord?id=CVE-2024-38531). + + Credit: [**@alois31**](https://github.com/alois31), [**Linus Heckemann (@lheckemann)**](https://github.com/lheckemann) + Co-authors: [**@edolstra**](https://github.com/edolstra) + +- `nix-shell ` looks for `shell.nix` [#496](https://github.com/NixOS/nix/issues/496) [#2279](https://github.com/NixOS/nix/issues/2279) [#4529](https://github.com/NixOS/nix/issues/4529) [#5431](https://github.com/NixOS/nix/issues/5431) [#11053](https://github.com/NixOS/nix/issues/11053) [#11057](https://github.com/NixOS/nix/pull/11057) + + `nix-shell $x` now looks for `$x/shell.nix` when `$x` resolves to a directory. + + Although this might be seen as a breaking change, its primarily interactive usage makes it a minor issue. + This adjustment addresses a commonly reported problem. + + This also applies to `nix-shell` shebang scripts. Consider the following example: + + ```shell + #!/usr/bin/env nix-shell + #!nix-shell -i bash + ``` + + This will now load `shell.nix` from the script's directory, if it exists; `default.nix` otherwise. + + The old behavior can be opted into by setting the option [`nix-shell-always-looks-for-shell-nix`](@docroot@/command-ref/conf-file.md#conf-nix-shell-always-looks-for-shell-nix) to `false`. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + +- `nix-repl`'s `:doc` shows documentation comments [#3904](https://github.com/NixOS/nix/issues/3904) [#10771](https://github.com/NixOS/nix/issues/10771) [#1652](https://github.com/NixOS/nix/pull/1652) [#9054](https://github.com/NixOS/nix/pull/9054) [#11072](https://github.com/NixOS/nix/pull/11072) + + `nix repl` has a `:doc` command that previously only rendered documentation for internally defined functions. + This feature has been extended to also render function documentation comments, in accordance with [RFC 145]. + + Example: + + ``` + nix-repl> :doc lib.toFunction + Function toFunction + … defined at /home/user/h/nixpkgs/lib/trivial.nix:1072:5 + + Turns any non-callable values into constant functions. Returns + callable values as is. + + Inputs + + v + + : Any value + + Examples + + :::{.example} + + ## lib.trivial.toFunction usage example + + | nix-repl> lib.toFunction 1 2 + | 1 + | + | nix-repl> lib.toFunction (x: x + 1) 2 + | 3 + + ::: + ``` + + Known limitations: + - It does not render documentation for "formals", such as `{ /** the value to return */ x, ... }: x`. + - Some extensions to markdown are not yet supported, as you can see in the example above. + + We'd like to acknowledge [Yingchi Long (@inclyc)](https://github.com/inclyc) for proposing a proof of concept for this functionality in [#9054](https://github.com/NixOS/nix/pull/9054), as well as [@sternenseemann](https://github.com/sternenseemann) and [Johannes Kirschbauer (@hsjobeki)](https://github.com/hsjobeki) for their contributions, proposals, and their work on [RFC 145]. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + + [RFC 145]: https://github.com/NixOS/rfcs/pull/145 + +### Other changes + +- Solve `cached failure of attribute X` [#9165](https://github.com/NixOS/nix/issues/9165) [#10513](https://github.com/NixOS/nix/issues/10513) [#10564](https://github.com/NixOS/nix/pull/10564) + + This eliminates all "cached failure of attribute X" messages by forcing evaluation of the original value when needed to show the exception to the user. This enhancement improves error reporting by providing the underlying message and stack trace. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Run the flake regressions test suite [#10603](https://github.com/NixOS/nix/pull/10603) + + This update introduces a GitHub action to run a subset of the [flake regressions test suite](https://github.com/NixOS/flake-regressions), which includes 259 flakes with their expected evaluation results. Currently, the action runs the first 25 flakes due to the full test suite's extensive runtime. A manually triggered action may be implemented later to run the entire test suite. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Support unit prefixes in configuration settings [#10668](https://github.com/NixOS/nix/pull/10668) + + Configuration settings in Nix now support unit prefixes, allowing for more intuitive and readable configurations. For example, you can now specify [`--min-free 1G`](@docroot@/command-ref/opt-common.md#opt-min-free) to set the minimum free space to 1 gigabyte. + + This enhancement was extracted from [#7851](https://github.com/NixOS/nix/pull/7851) and is also useful for PR [#10661](https://github.com/NixOS/nix/pull/10661). + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- `nix build`: show all FOD errors with `--keep-going` [#10734](https://github.com/NixOS/nix/pull/10734) + + The [`nix build`](@docroot@/command-ref/new-cli/nix3-build.md) command has been updated to improve the behavior of the [`--keep-going`] flag. Now, when `--keep-going` is used, all hash-mismatch errors of failing fixed-output derivations (FODs) are displayed, similar to the behavior for other build failures. This enhancement ensures that all relevant build errors are shown, making it easier for users to update multiple derivations at once or to diagnose and fix issues. + + Author: [**Jörg Thalheim (@Mic92)**](https://github.com/Mic92), [**Maximilian Bosch (@Ma27)**](https://github.com/Ma27) + + [`--keep-going`](@docroot@/command-ref/opt-common.md#opt-keep-going) + +- Build with Meson [#2503](https://github.com/NixOS/nix/issues/2503) [#10378](https://github.com/NixOS/nix/pull/10378) [#10855](https://github.com/NixOS/nix/pull/10855) [#10904](https://github.com/NixOS/nix/pull/10904) [#10908](https://github.com/NixOS/nix/pull/10908) [#10914](https://github.com/NixOS/nix/pull/10914) [#10933](https://github.com/NixOS/nix/pull/10933) [#10936](https://github.com/NixOS/nix/pull/10936) [#10954](https://github.com/NixOS/nix/pull/10954) [#10955](https://github.com/NixOS/nix/pull/10955) [#10963](https://github.com/NixOS/nix/pull/10963) [#10967](https://github.com/NixOS/nix/pull/10967) [#10973](https://github.com/NixOS/nix/pull/10973) [#11034](https://github.com/NixOS/nix/pull/11034) [#11054](https://github.com/NixOS/nix/pull/11054) [#11055](https://github.com/NixOS/nix/pull/11055) [#11060](https://github.com/NixOS/nix/pull/11060) [#11064](https://github.com/NixOS/nix/pull/11064) [#11155](https://github.com/NixOS/nix/pull/11155) + + These changes aim to replace the use of autotools and `make` with Meson for building various components of Nix. Additionally, each library is built in its own derivation, leveraging Meson's "subprojects" feature to allow a single development shell for building all libraries while also supporting separate builds. This approach aims to improve productivity and build modularity, compared to both make and a monolithic Meson-based derivation. + + Special thanks to everyone who has contributed to the Meson port, particularly [**@p01arst0rm**](https://github.com/p01arst0rm) and [**@Qyriad**](https://github.com/Qyriad). + + Authors: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314), [**Tom Bereknyei**](https://github.com/tomberek), [**Théophane Hufschmitt (@thufschmitt)**](https://github.com/thufschmitt), [**Valentin Gagarin (@fricklerhandwerk)**](https://github.com/fricklerhandwerk), [**Robert Hensing (@roberth)**](https://github.com/roberth) + Co-authors: [**@p01arst0rm**](https://github.com/p01arst0rm), [**@Qyriad**](https://github.com/Qyriad) + +- Evaluation cache: fix cache regressions [#10570](https://github.com/NixOS/nix/issues/10570) [#11086](https://github.com/NixOS/nix/pull/11086) + + This update addresses two bugs in the evaluation cache system: + + 1. Regression in #10570: The evaluation cache was not being persisted in `nix develop`. + 2. Nix could sometimes try to commit the evaluation cache SQLite transaction without there being an active transaction, resulting in non-error errors being printed. + + Author: [**Lexi Mattick (@kognise)**](https://github.com/kognise) + +- Introduce `libnixflake` [#9063](https://github.com/NixOS/nix/pull/9063) + + A new library, `libnixflake`, has been introduced to better separate the Flakes layer within Nix. This change refactors the codebase to encapsulate Flakes-specific functionality within its own library. + + See the commits in the pull request for detailed changes, with the only significant code modifications happening in the initial commit. + + This change was alluded to in [RFC 134](https://github.com/nixos/rfcs/blob/master/rfcs/0134-nix-store-layer.md) and is a step towards a more modular and maintainable codebase. + + Author: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314) + +- CLI options `--arg-from-file` and `--arg-from-stdin` [#9913](https://github.com/NixOS/nix/pull/9913) + +- The `--debugger` now prints source location information, instead of the + pointers of source location information. Before: + + ``` + nix-repl> :bt + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + 0x600001522598 + ``` + + After: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + /nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27 + + 131| + 132| bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs; + | ^ + 133| in + ``` + +- Stop vendoring `toml11` + + We don't apply any patches to it, and vendoring it locks users into + bugs (it hasn't been updated since its introduction in late 2021). + + Author: [**Winter (@winterqt)**](https://github.com/winterqt) + +- Rename hash format `base32` to `nix32` [#8678](https://github.com/NixOS/nix/pull/8678) + + Hash format `base32` was renamed to `nix32` since it used a special nix-specific character set for + [Base32](https://en.wikipedia.org/wiki/Base32). + + **Deprecation**: Use `nix32` instead of `base32` as `toHashFormat` + + For the builtin `convertHash`, the `toHashFormat` parameter now accepts the same hash formats as the `--to`/`--from` + parameters of the `nix hash conert` command: `"base16"`, `"nix32"`, `"base64"`, and `"sri"`. The former `"base32"` value + remains as a deprecated alias for `"nix32"`. Please convert your code from: + + ```nix + builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base32";} + ``` + + to + + ```nix + builtins.convertHash { inherit hash hashAlgo; toHashFormat = "nix32";} + ``` + +- Add `pipe-operators` experimental feature [#11131](https://github.com/NixOS/nix/pull/11131) + + This is a draft implementation of [RFC 0148](https://github.com/NixOS/rfcs/pull/148). + + The `pipe-operators` experimental feature adds [`<|` and `|>` operators][pipe operators] to the Nix language. + *a* `|>` *b* is equivalent to the function application *b* *a*, and + *a* `<|` *b* is equivalent to the function application *a* *b*. + + For example: + + ``` + nix-repl> 1 |> builtins.add 2 |> builtins.mul 3 + 9 + + nix-repl> builtins.add 1 <| builtins.mul 2 <| 3 + 7 + ``` + + `<|` and `|>` are right and left associative, respectively, and have lower precedence than any other operator. + These properties may change in future releases. + + See [the RFC](https://github.com/NixOS/rfcs/pull/148) for more examples and rationale. + + [pipe operators]: @docroot@/language/operators.md#pipe-operators + +- `nix-shell` shebang uses relative path [#4232](https://github.com/NixOS/nix/issues/4232) [#5088](https://github.com/NixOS/nix/pull/5088) [#11058](https://github.com/NixOS/nix/pull/11058) + + + Relative [path](@docroot@/language/types.md#type-path) literals in `nix-shell` shebang scripts' options are now resolved relative to the [script's location](@docroot@/glossary.md?highlight=base%20directory#gloss-base-directory). + Previously they were resolved relative to the current working directory. + + For example, consider the following script in `~/myproject/say-hi`: + + ```shell + #!/usr/bin/env nix-shell + #!nix-shell --expr 'import ./shell.nix' + #!nix-shell --arg toolset './greeting-tools.nix' + #!nix-shell -i bash + hello + ``` + + Older versions of `nix-shell` would resolve `shell.nix` relative to the current working directory, such as the user's home directory in this example: + + ```console + [hostname:~]$ ./myproject/say-hi + error: + … while calling the 'import' builtin + at «string»:1:2: + 1| (import ./shell.nix) + | ^ + + error: path '/home/user/shell.nix' does not exist + ``` + + Since this release, `nix-shell` resolves `shell.nix` relative to the script's location, and `~/myproject/shell.nix` is used. + + ```console + $ ./myproject/say-hi + Hello, world! + ``` + + **Opt-out** + + This is technically a breaking change, so we have added an option so you can adapt independently of your Nix update. + The old behavior can be opted into by setting the option [`nix-shell-shebang-arguments-relative-to-script`](@docroot@/command-ref/conf-file.md#conf-nix-shell-shebang-arguments-relative-to-script) to `false`. + This option will be removed in a future release. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + +- Improve handling of tarballs that don't consist of a single top-level directory [#11195](https://github.com/NixOS/nix/pull/11195) + + In previous Nix releases, the tarball fetcher (used by `builtins.fetchTarball`) erroneously merged top-level directories into a single directory, and silently discarded top-level files that are not directories. This is no longer the case. The new behaviour is that *only* if the tarball consists of a single directory, the top-level path component of the files in the tarball is removed (similar to `tar`'s `--strip-components=1`). + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Improve handling of tarballs that don't consist of a single top-level directory [#11195](https://github.com/NixOS/nix/pull/11195) + + In previous Nix releases, the tarball fetcher (used by `builtins.fetchTarball`) erroneously merged top-level directories into a single directory, and silently discarded top-level files that are not directories. This is no longer the case. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Setting to warn about large paths [#10778](https://github.com/NixOS/nix/pull/10778) + + Nix can now warn when evaluation of a Nix expression causes a large + path to be copied to the Nix store. The threshold for this warning can + be configured using the `warn-large-path-threshold` setting, + e.g. `--warn-large-path-threshold 100M`. + + +# Contributors + +This release was made possible by the following 43 contributors: + +- Andreas Rammhold [**(@andir)**](https://github.com/andir) +- Andrew Marshall [**(@amarshall)**](https://github.com/amarshall) +- Brian McKenna [**(@puffnfresh)**](https://github.com/puffnfresh) +- Cameron [**(@SkamDart)**](https://github.com/SkamDart) +- Cole Helbling [**(@cole-h)**](https://github.com/cole-h) +- Corbin Simpson [**(@MostAwesomeDude)**](https://github.com/MostAwesomeDude) +- Eelco Dolstra [**(@edolstra)**](https://github.com/edolstra) +- Emily [**(@emilazy)**](https://github.com/emilazy) +- Enno Richter [**(@elohmeier)**](https://github.com/elohmeier) +- Farid Zakaria [**(@fzakaria)**](https://github.com/fzakaria) +- HaeNoe [**(@haenoe)**](https://github.com/haenoe) +- Hamir Mahal [**(@hamirmahal)**](https://github.com/hamirmahal) +- Harmen [**(@alicebob)**](https://github.com/alicebob) +- Ivan Trubach [**(@tie)**](https://github.com/tie) +- Jared Baur [**(@jmbaur)**](https://github.com/jmbaur) +- John Ericson [**(@Ericson2314)**](https://github.com/Ericson2314) +- Jonathan De Troye [**(@detroyejr)**](https://github.com/detroyejr) +- Jörg Thalheim [**(@Mic92)**](https://github.com/Mic92) +- Klemens Nanni [**(@klemensn)**](https://github.com/klemensn) +- Las Safin [**(@L-as)**](https://github.com/L-as) +- Lexi Mattick [**(@kognise)**](https://github.com/kognise) +- Matthew Bauer [**(@matthewbauer)**](https://github.com/matthewbauer) +- Max “Goldstein” Siling [**(@GoldsteinE)**](https://github.com/GoldsteinE) +- Mingye Wang [**(@Artoria2e5)**](https://github.com/Artoria2e5) +- Philip Taron [**(@philiptaron)**](https://github.com/philiptaron) +- Pierre Bourdon [**(@delroth)**](https://github.com/delroth) +- Pino Toscano [**(@pinotree)**](https://github.com/pinotree) +- RTUnreal [**(@RTUnreal)**](https://github.com/RTUnreal) +- Robert Hensing [**(@roberth)**](https://github.com/roberth) +- Romain Neil [**(@romain-neil)**](https://github.com/romain-neil) +- Ryan Hendrickson [**(@rhendric)**](https://github.com/rhendric) +- Sergei Trofimovich [**(@trofi)**](https://github.com/trofi) +- Shogo Takata [**(@pineapplehunter)**](https://github.com/pineapplehunter) +- Siddhant Kumar [**(@siddhantk232)**](https://github.com/siddhantk232) +- Silvan Mosberger [**(@infinisil)**](https://github.com/infinisil) +- Théophane Hufschmitt [**(@thufschmitt)**](https://github.com/thufschmitt) +- Valentin Gagarin [**(@fricklerhandwerk)**](https://github.com/fricklerhandwerk) +- Winter [**(@winterqt)**](https://github.com/winterqt) +- jade [**(@lf-)**](https://github.com/lf-) +- kirillrdy [**(@kirillrdy)**](https://github.com/kirillrdy) +- pennae [**(@pennae)**](https://github.com/pennae) +- poweredbypie [**(@poweredbypie)**](https://github.com/poweredbypie) +- tomberek [**(@tomberek)**](https://github.com/tomberek) diff --git a/doc/manual/src/release-notes/rl-2.4.md b/doc/manual/src/release-notes/rl-2.4.md index 8b566fc7b..1201e53b6 100644 --- a/doc/manual/src/release-notes/rl-2.4.md +++ b/doc/manual/src/release-notes/rl-2.4.md @@ -23,7 +23,7 @@ more than 2800 commits from 195 contributors since release 2.3. * The **`nix` command** has seen a lot of work and is now almost at feature parity with the old command-line interface (the `nix-*` commands). It aims to be [more modern, consistent and pleasant to - use](../contributing/cli-guideline.md) than the old CLI. It is still + use](../development/cli-guideline.md) than the old CLI. It is still marked as experimental but its interface should not change much anymore in future releases. diff --git a/doc/manual/src/store/file-system-object/content-address.md b/doc/manual/src/store/file-system-object/content-address.md index 1c63c52eb..410d7fb7c 100644 --- a/doc/manual/src/store/file-system-object/content-address.md +++ b/doc/manual/src/store/file-system-object/content-address.md @@ -82,4 +82,4 @@ In the future, we may support a Git-like hash for such file system objects, or w [file system object]: ../file-system-object.md [store object]: ../store-object.md -[xp-feature-git-hashing]: @docroot@/contributing/experimental-features.md#xp-feature-git-hashing +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/doc/manual/src/store/store-object/content-address.md b/doc/manual/src/store/store-object/content-address.md index f6f982035..02dce2836 100644 --- a/doc/manual/src/store/store-object/content-address.md +++ b/doc/manual/src/store/store-object/content-address.md @@ -92,4 +92,4 @@ becomes more widespread, this restriction will be revisited. [fso-ca]: ../file-system-object/content-address.md [sp-spec]: @docroot@/protocols/store-path.md -[xp-feature-git-hashing]: @docroot@/contributing/experimental-features.md#xp-feature-git-hashing +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/flake.lock b/flake.lock index 8ea495401..1a6e4f7a1 100644 --- a/flake.lock +++ b/flake.lock @@ -3,11 +3,11 @@ "flake-compat": { "flake": false, "locked": { - "lastModified": 1673956053, - "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", "owner": "edolstra", "repo": "flake-compat", - "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", "type": "github" }, "original": { @@ -23,11 +23,11 @@ ] }, "locked": { - "lastModified": 1712014858, - "narHash": "sha256-sB4SWl2lX95bExY2gMFG5HIzvva5AVMJd4Igm+GpZNw=", + "lastModified": 1719994518, + "narHash": "sha256-pQMhCCHyQGRzdfAkdJ4cIWiw+JNuWsTX7f0ZYSyz0VY=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "9126214d0a59633752a136528f5f3b9aa8565b7d", + "rev": "9227223f6d922fee3c7b190b2cc238a99527bbb7", "type": "github" }, "original": { @@ -51,49 +51,60 @@ "type": "github" } }, - "flake-utils": { + "git-hooks-nix": { + "inputs": { + "flake-compat": [], + "gitignore": [], + "nixpkgs": [ + "nixpkgs" + ], + "nixpkgs-stable": [ + "nixpkgs" + ] + }, "locked": { - "lastModified": 1667395993, - "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f", + "lastModified": 1721042469, + "narHash": "sha256-6FPUl7HVtvRHCCBQne7Ylp4p+dpP3P/OYuzjztZ4s70=", + "owner": "cachix", + "repo": "git-hooks.nix", + "rev": "f451c19376071a90d8c58ab1a953c6e9840527fd", "type": "github" }, "original": { - "owner": "numtide", - "repo": "flake-utils", + "owner": "cachix", + "repo": "git-hooks.nix", "type": "github" } }, "libgit2": { "flake": false, "locked": { - "lastModified": 1697646580, - "narHash": "sha256-oX4Z3S9WtJlwvj0uH9HlYcWv+x1hqp8mhXl7HsLu2f0=", + "lastModified": 1715853528, + "narHash": "sha256-J2rCxTecyLbbDdsyBWn9w7r3pbKRMkI9E7RvRgAqBdY=", "owner": "libgit2", "repo": "libgit2", - "rev": "45fd9ed7ae1a9b74b957ef4f337bc3c8b3df01b5", + "rev": "36f7e21ad757a3dacc58cf7944329da6bc1d6e96", "type": "github" }, "original": { "owner": "libgit2", + "ref": "v1.8.1", "repo": "libgit2", "type": "github" } }, "nixpkgs": { "locked": { - "lastModified": 1717432640, - "narHash": "sha256-+f9c4/ZX5MWDOuB1rKoWj+lBNm0z0rs4CK47HBLxy1o=", + "lastModified": 1721548954, + "narHash": "sha256-7cCC8+Tdq1+3OPyc3+gVo9dzUNkNIQfwSDJ2HSi2u3o=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "88269ab3044128b7c2f4c7d68448b2fb50456870", + "rev": "63d37ccd2d178d54e7fb691d7ec76000740ea24a", "type": "github" }, "original": { "owner": "NixOS", - "ref": "release-24.05", + "ref": "nixos-24.05", "repo": "nixpkgs", "type": "github" } @@ -130,42 +141,16 @@ "type": "github" } }, - "pre-commit-hooks": { - "inputs": { - "flake-compat": [], - "flake-utils": "flake-utils", - "gitignore": [], - "nixpkgs": [ - "nixpkgs" - ], - "nixpkgs-stable": [ - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1712897695, - "narHash": "sha256-nMirxrGteNAl9sWiOhoN5tIHyjBbVi5e2tgZUgZlK3Y=", - "owner": "cachix", - "repo": "pre-commit-hooks.nix", - "rev": "40e6053ecb65fcbf12863338a6dcefb3f55f1bf8", - "type": "github" - }, - "original": { - "owner": "cachix", - "repo": "pre-commit-hooks.nix", - "type": "github" - } - }, "root": { "inputs": { "flake-compat": "flake-compat", "flake-parts": "flake-parts", "flake-schemas": "flake-schemas", + "git-hooks-nix": "git-hooks-nix", "libgit2": "libgit2", "nixpkgs": "nixpkgs", "nixpkgs-23-11": "nixpkgs-23-11", - "nixpkgs-regression": "nixpkgs-regression", - "pre-commit-hooks": "pre-commit-hooks" + "nixpkgs-regression": "nixpkgs-regression" } } }, diff --git a/flake.nix b/flake.nix index 256ff66cd..fe5907f77 100644 --- a/flake.nix +++ b/flake.nix @@ -1,32 +1,30 @@ { description = "The purely functional package manager"; - # TODO switch to nixos-23.11-small - # https://nixpk.gs/pr-tracker.html?pr=291954 - inputs.nixpkgs.url = "github:NixOS/nixpkgs/release-24.05"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446"; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; - inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; }; + inputs.libgit2 = { url = "github:libgit2/libgit2/v1.8.1"; flake = false; }; inputs.flake-schemas.url = "github:DeterminateSystems/flake-schemas"; # dev tooling inputs.flake-parts.url = "github:hercules-ci/flake-parts"; - inputs.pre-commit-hooks.url = "github:cachix/pre-commit-hooks.nix"; + inputs.git-hooks-nix.url = "github:cachix/git-hooks.nix"; # work around https://github.com/NixOS/nix/issues/7730 inputs.flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; - inputs.pre-commit-hooks.inputs.nixpkgs.follows = "nixpkgs"; - inputs.pre-commit-hooks.inputs.nixpkgs-stable.follows = "nixpkgs"; + inputs.git-hooks-nix.inputs.nixpkgs.follows = "nixpkgs"; + inputs.git-hooks-nix.inputs.nixpkgs-stable.follows = "nixpkgs"; # work around 7730 and https://github.com/NixOS/nix/issues/7807 - inputs.pre-commit-hooks.inputs.flake-compat.follows = ""; - inputs.pre-commit-hooks.inputs.gitignore.follows = ""; + inputs.git-hooks-nix.inputs.flake-compat.follows = ""; + inputs.git-hooks-nix.inputs.gitignore.follows = ""; outputs = inputs@{ self, nixpkgs, nixpkgs-regression, libgit2, flake-schemas, ... }: let inherit (nixpkgs) lib; - officialRelease = false; + officialRelease = true; version = lib.fileContents ./.version + versionSuffix; versionSuffix = @@ -145,6 +143,7 @@ nix_noTests = final.nix.override { doInstallCheck = false; + doCheck = false; }; # See https://github.com/NixOS/nixpkgs/pull/214409 @@ -280,6 +279,7 @@ in "-D${prefix}:${rest}"; havePerl = stdenv.buildPlatform == stdenv.hostPlatform && stdenv.hostPlatform.isUnix; + ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags; in { pname = "shell-for-" + attrs.pname; @@ -311,10 +311,12 @@ }; mesonFlags = - map (transformFlag "libutil") pkgs.nixComponents.nix-util.mesonFlags - ++ map (transformFlag "libstore") pkgs.nixComponents.nix-store.mesonFlags - ++ map (transformFlag "libfetchers") pkgs.nixComponents.nix-fetchers.mesonFlags - ++ lib.optionals havePerl (map (transformFlag "perl") pkgs.nixComponents.nix-perl-bindings.mesonFlags) + map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents.nix-util.mesonFlags) + ++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents.nix-store.mesonFlags) + ++ map (transformFlag "libfetchers") (ignoreCrossFile pkgs.nixComponents.nix-fetchers.mesonFlags) + ++ lib.optionals havePerl (map (transformFlag "perl") (ignoreCrossFile pkgs.nixComponents.nix-perl-bindings.mesonFlags)) + ++ map (transformFlag "libexpr") (ignoreCrossFile pkgs.nixComponents.nix-expr.mesonFlags) + ++ map (transformFlag "libcmd") (ignoreCrossFile pkgs.nixComponents.nix-cmd.mesonFlags) ; nativeBuildInputs = attrs.nativeBuildInputs or [] @@ -324,8 +326,17 @@ ++ lib.optionals havePerl pkgs.nixComponents.nix-perl-bindings.nativeBuildInputs ++ pkgs.nixComponents.nix-internal-api-docs.nativeBuildInputs ++ pkgs.nixComponents.nix-external-api-docs.nativeBuildInputs + ++ lib.optional + (!stdenv.buildPlatform.canExecute stdenv.hostPlatform + # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479 + && !(stdenv.hostPlatform.isWindows && stdenv.buildPlatform.isDarwin) + && stdenv.hostPlatform.emulatorAvailable pkgs.buildPackages + && lib.meta.availableOn stdenv.buildPlatform (stdenv.hostPlatform.emulator pkgs.buildPackages)) + pkgs.buildPackages.mesonEmulatorHook ++ [ pkgs.buildPackages.cmake + pkgs.buildPackages.shellcheck + pkgs.buildPackages.changelog-d modular.pre-commit.settings.package (pkgs.writeScriptBin "pre-commit-hooks-install" modular.pre-commit.settings.installationScript) diff --git a/maintainers/README.md b/maintainers/README.md index 2a718e283..b92833497 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -50,7 +50,7 @@ The team meets twice a week (times are denoted in the [Europe/Amsterdam](https:/ - mark it as draft if it is blocked on the contributor - escalate it back to the team by moving it to To discuss, and leaving a comment as to why the issue needs to be discussed again. -- Work meeting: [Mondays 13:00-15:00 Europe/Amsterdam](https://www.google.com/calendar/event?eid=Ym52NDdzYnRic2NzcDcybjZiNDhpNzhpa3NfMjAyNDA1MTNUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) +- Work meeting: [Mondays 14:00-16:00 Europe/Amsterdam](https://www.google.com/calendar/event?eid=Ym52NDdzYnRic2NzcDcybjZiNDhpNzhpa3NfMjAyNDA1MTNUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) 1. Code review on pull requests from [In review](#in-review). 2. Other chores and tasks. diff --git a/maintainers/data/release-credits-email-to-handle.json b/maintainers/data/release-credits-email-to-handle.json new file mode 100644 index 000000000..cddc1a6e7 --- /dev/null +++ b/maintainers/data/release-credits-email-to-handle.json @@ -0,0 +1,52 @@ +{ + "bogus": "bogus", + "edolstra@gmail.com": "edolstra", + "roberth@users.noreply.github.com": "roberth", + "toscano.pino@tiscali.it": "pinotree", + "valentin@gagarin.work": "fricklerhandwerk", + "mr.trubach@icloud.com": "tie", + "robert@roberthensing.nl": "roberth", + "lix@jade.fyi": "lf-", + "cole.e.helbling@outlook.com": "cole-h", + "joerg@thalheim.io": "Mic92", + "John.Ericson@Obsidian.Systems": "Ericson2314", + "ryan.hendrickson@alum.mit.edu": "rhendric", + "67135060+poweredbypie@users.noreply.github.com": "poweredbypie", + "detroyejr@outlook.com": "detroyejr", + "silvan.mosberger@tweag.io": "infinisil", + "vcs@emily.moe": "emilazy", + "farid.m.zakaria@gmail.com": "fzakaria", + "22859658+RTUnreal@users.noreply.github.com": "RTUnreal", + "me@las.rs": "L-as", + "philip.taron@gmail.com": "philiptaron", + "root@goldstein.rs": "GoldsteinE", + "tomberek@users.noreply.github.com": "tomberek", + "lexi.mattick@neuralink.com": "kognise", + "andrew@johnandrewmarshall.com": "amarshall", + "contact@romain-neil.fr": "romain-neil", + "Mic92@users.noreply.github.com": "Mic92", + "valentin.gagarin@tweag.io": "fricklerhandwerk", + "siddhantk232@gmail.com": "siddhantk232", + "kn@openbsd.org": "klemensn", + "slyich@gmail.com": "trofi", + "theophane.hufschmitt@tweag.io": "thufschmitt", + "alicebob@lijzij.de": "alicebob", + "winter@winter.cafe": "winterqt", + "brian@brianmckenna.org": "puffnfresh", + "git@haenoe.party": "haenoe", + "peshogo@gmail.com": "pineapplehunter", + "poweredbypie@users.noreply.github.com": "poweredbypie", + "arthur200126@gmail.com": "Artoria2e5", + "tomberek@gmail.com": "tomberek", + "jaredbaur@fastmail.com": "jmbaur", + "andreas@rammhold.de": "andir", + "hamirmahal@gmail.com": "hamirmahal", + "git@JohnEricson.me": "Ericson2314", + "8763518+SkamDart@users.noreply.github.com": "SkamDart", + "kirillrdy@gmail.com": "kirillrdy", + "pennae@lix.systems": "pennae", + "delroth@gmail.com": "delroth", + "enno@nerdworks.de": "elohmeier", + "mjbauer95@gmail.com": "matthewbauer", + "MostAwesomeDude@gmail.com": "MostAwesomeDude" +} \ No newline at end of file diff --git a/maintainers/data/release-credits-handle-to-name.json b/maintainers/data/release-credits-handle-to-name.json new file mode 100644 index 000000000..abf9ed05b --- /dev/null +++ b/maintainers/data/release-credits-handle-to-name.json @@ -0,0 +1,45 @@ +{ + "fzakaria": "Farid Zakaria", + "kognise": "Lexi Mattick", + "L-as": "Las Safin", + "haenoe": "HaeNoe", + "andir": "Andreas Rammhold", + "matthewbauer": "Matthew Bauer", + "emilazy": "Emily", + "pineapplehunter": "Shogo Takata", + "RTUnreal": null, + "jmbaur": "Jared Baur", + "Ericson2314": "John Ericson", + "pinotree": "Pino Toscano", + "tie": "Ivan Trubach", + "poweredbypie": null, + "fricklerhandwerk": "Valentin Gagarin", + "Mic92": "J\u00f6rg Thalheim", + "alicebob": "Harmen", + "elohmeier": "Enno Richter", + "delroth": "Pierre Bourdon", + "kirillrdy": null, + "thufschmitt": "Th\u00e9ophane Hufschmitt", + "detroyejr": "Jonathan De Troye", + "klemensn": "Klemens Nanni", + "tomberek": null, + "rhendric": "Ryan Hendrickson", + "philiptaron": "Philip Taron", + "puffnfresh": "Brian McKenna", + "lf-": "jade", + "romain-neil": "Romain Neil", + "hamirmahal": "Hamir Mahal", + "edolstra": "Eelco Dolstra", + "Artoria2e5": "Mingye Wang", + "SkamDart": "Cameron", + "roberth": "Robert Hensing", + "amarshall": "Andrew Marshall", + "trofi": "Sergei Trofimovich", + "cole-h": "Cole Helbling", + "infinisil": "Silvan Mosberger", + "siddhantk232": "Siddhant Kumar", + "winterqt": "Winter", + "GoldsteinE": "Max \u201cGoldstein\u201d Siling", + "pennae": null, + "MostAwesomeDude": "Corbin Simpson" +} \ No newline at end of file diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index 66bfcf609..be91df536 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -2,7 +2,7 @@ { imports = [ - inputs.pre-commit-hooks.flakeModule + inputs.git-hooks-nix.flakeModule ]; perSystem = { config, pkgs, ... }: { @@ -495,7 +495,6 @@ excludes = [ # We haven't linted these files yet ''^config/install-sh$'' - ''^misc/systemv/nix-daemon$'' ''^misc/bash/completion\.sh$'' ''^misc/fish/completion\.fish$'' ''^misc/zsh/completion\.zsh$'' diff --git a/maintainers/release-credits b/maintainers/release-credits new file mode 100755 index 000000000..7a5c87d7d --- /dev/null +++ b/maintainers/release-credits @@ -0,0 +1,184 @@ +#!/usr/bin/env nix +# vim: set filetype=python: +#!nix develop --impure --expr +#!nix `` +#!nix let flake = builtins.getFlake ("git+file://" + toString ../.); +#!nix pkgs = flake.inputs.nixpkgs.legacyPackages.${builtins.currentSystem}; +#!nix in pkgs.mkShell { nativeBuildInputs = [ +#!nix (pkgs.python3.withPackages (ps: with ps; [ requests ])) +#!nix ]; } +#!nix `` --command python3 + +# This script lists out the contributors for a given release. +# It must be run from the root of the Nix repository. + +import os +import sys +import json +import requests + +github_token = os.environ.get("GITHUB_TOKEN") +if not github_token: + print("GITHUB_TOKEN is not set. If you hit the rate limit, set it", file=sys.stderr) + # Might be ok, as we have a cache. + # raise ValueError("GITHUB_TOKEN must be set") + +# 1. Read the current version in .version +version = os.environ.get("VERSION") +if not version: + version = open(".version").read().strip() + +print(f"Generating release credits for Nix {version}", file=sys.stderr) + +# 2. Compute previous version +vcomponents = version.split(".") +if len(vcomponents) >= 2: + prev_version = f"{vcomponents[0]}.{int(vcomponents[1])-1}.0" +else: + raise ValueError(".version must have at least two components") + +# For unreleased versions +endref = "HEAD" +# For older releases +# endref = version + +# 2. Find the merge base between the current version and the previous version +mergeBase = os.popen(f"git merge-base {prev_version} {endref}").read().strip() +print(f"Merge base between {prev_version} and {endref} is {mergeBase}", file=sys.stderr) + +# 3. Find the date of the merge base +mergeBaseDate = os.popen(f"git show -s --format=%ci {mergeBase}").read().strip()[0:10] +print(f"Merge base date is {mergeBaseDate}", file=sys.stderr) + +# 4. Get the commits between the merge base and the current version + +def get_commits(): + raw = os.popen(f"git log --pretty=format:'%H\t%an\t%ae' {mergeBase}..{endref}").read().strip() + lines = raw.split("\n") + return [ { "hash": items[0], "author": items[1], "email": items[2] } + for line in lines + for items in (line.split("\t"),) + ] + +def commits_to_first_commit_by_email(commits): + by_email = dict() + for commit in commits: + email = commit["email"] + if email not in by_email: + by_email[email] = commit + return by_email + + +samples = commits_to_first_commit_by_email(get_commits()) + +# For quick testing, only pick two samples from the dict +# samples = dict(list(samples.items())[:2]) + +# Query the GitHub API to get handle +def get_github_commit(commit): + url = f"https://api.github.com/repos/NixOS/nix/commits/{commit['hash']}" + headers = {'Authorization': f'token {github_token}'} + response = requests.get(url, headers=headers) + response.raise_for_status() + return response.json() + +class Cache: + def __init__(self, filename, require = True): + self.filename = filename + try: + with open(filename, "r") as f: + self.values = json.load(f) + except FileNotFoundError: + if require: + raise + self.values = dict() + def save(self): + with open(self.filename, "w") as f: + json.dump(self.values, f, indent=4) + print(f"Saved cache to {self.filename}", file=sys.stderr) + +# The email to handle cache maps email addresses to either +# - a handle (string) +# - None (if no handle was found) +email_to_handle_cache = Cache("maintainers/data/release-credits-email-to-handle.json") + +handles = set() +emails = dict() + +for sample in samples: + s = samples[sample] + email = s["email"] + if not email in email_to_handle_cache.values: + print(f"Querying GitHub API for {s['hash']}, to get handle for {s['email']}") + ghc = get_github_commit(samples[sample]) + gha = ghc["author"] + if gha and gha["login"]: + handle = gha["login"] + print(f"Handle: {handle}") + email_to_handle_cache.values[email] = handle + else: + print(f"Found no handle for {s['email']}") + email_to_handle_cache.values[email] = None + handle = email_to_handle_cache.values[email] + if handle is not None: + handles.add(handle) + else: + emails[email] = s["author"] + +# print(email_to_handle_cache.values) + +email_to_handle_cache.save() + +handle_to_name_cache = Cache("maintainers/data/release-credits-handle-to-name.json") + +print(f"Found {len(handles)} handles", file=sys.stderr) + +for handle in handles: + if not handle in handle_to_name_cache.values: + print(f"Querying GitHub API for {handle}, to get name", file=sys.stderr) + url = f"https://api.github.com/users/{handle}" + headers = {'Authorization': f'token {github_token}'} + response = requests.get(url, headers=headers) + response.raise_for_status() + user = response.json() + name = user["name"] + print(f"Name: {name}", file=sys.stderr) + handle_to_name_cache.values[handle] = name + +handle_to_name_cache.save() + +entries = list() + +def is_bot(handle): + return handle.endswith("[bot]") + +for handle in handles: + name = handle_to_name_cache.values[handle] + if name is None: + # This way it looks more regular + name = handle + + # Note to bots: I'm only doing this because it would distract humans, and + # therefore delay your creation. I'm sorry; it's for your benefit. + if not is_bot(handle): + entries += [ f"- {name} [**(@{handle})**](https://github.com/{handle})" ] + +def shuffle(entries): + salt = os.urandom(16) + return sorted(entries, key=lambda x: hash((x, salt))) + +# Fair ordering is undecidable +entries = shuffle(entries) + +# For a sanity check, we could sort the entries by handle instead. +# entries = sorted(entries) + +print("") +print(f"This release was made possible by the following {len(entries)} contributors:") +print("") + +for entry in entries: + print(entry) + +for email in emails: + print(f"- {emails[email]}") diff --git a/maintainers/release-notes b/maintainers/release-notes index 0fca5abf2..c0c4ee734 100755 --- a/maintainers/release-notes +++ b/maintainers/release-notes @@ -1,4 +1,5 @@ #!/usr/bin/env nix +# vim: set filetype=bash: #!nix shell .#changelog-d --command bash # --- CONFIGURATION --- @@ -151,6 +152,13 @@ section_title="Release $version_full ($DATE)" echo "# $section_title" echo changelog-d doc/manual/rl-next | sed -e 's/ *$//' + + if ! $IS_PATCH; then + echo + echo "# Contributors" + echo + VERSION=$version_full ./maintainers/release-credits + fi ) | tee -a $file log "Wrote $file" diff --git a/maintainers/release-process.md b/maintainers/release-process.md index da6886ea9..7a2b3c0a7 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -39,6 +39,10 @@ release: * Proof-read / edit / rearrange the release notes if needed. Breaking changes and highlights should go to the top. +* Run `maintainers/release-credits` to make sure the credits script works + and produces a sensible output. Some emails might not automatically map to + a GitHub handle. + * Push. ```console diff --git a/meson.build b/meson.build index e6bdc2eac..1554244ab 100644 --- a/meson.build +++ b/meson.build @@ -26,9 +26,12 @@ subproject('external-api-docs') subproject('libutil-c') subproject('libstore-c') subproject('libexpr-c') +subproject('libmain-c') # Language Bindings -subproject('perl') +if not meson.is_cross_build() + subproject('perl') +endif # Testing subproject('nix-util-test-support') diff --git a/misc/systemv/nix-daemon b/misc/systemv/nix-daemon index fea537167..e8326f947 100755 --- a/misc/systemv/nix-daemon +++ b/misc/systemv/nix-daemon @@ -34,6 +34,7 @@ else fi # Source function library. +# shellcheck source=/dev/null . /etc/init.d/functions LOCKFILE=/var/lock/subsys/nix-daemon @@ -41,14 +42,20 @@ RUNDIR=/var/run/nix PIDFILE=${RUNDIR}/nix-daemon.pid RETVAL=0 -base=${0##*/} +# https://www.shellcheck.net/wiki/SC3004 +# Check if gettext exists +if ! type gettext > /dev/null 2>&1 +then + # If not, create a dummy function that returns the input verbatim + gettext() { printf '%s' "$1"; } +fi start() { mkdir -p ${RUNDIR} chown ${NIX_DAEMON_USER}:${NIX_DAEMON_USER} ${RUNDIR} - echo -n $"Starting nix daemon... " + printf '%s' "$(gettext 'Starting nix daemon... ')" daemonize -u $NIX_DAEMON_USER -p ${PIDFILE} $NIX_DAEMON_BIN $NIX_DAEMON_OPTS RETVAL=$? @@ -58,7 +65,7 @@ start() { } stop() { - echo -n $"Shutting down nix daemon: " + printf '%s' "$(gettext 'Shutting down nix daemon: ')" killproc -p ${PIDFILE} $NIX_DAEMON_BIN RETVAL=$? [ $RETVAL -eq 0 ] && rm -f ${LOCKFILE} ${PIDFILE} @@ -67,7 +74,7 @@ stop() { } reload() { - echo -n $"Reloading nix daemon... " + printf '%s' "$(gettext 'Reloading nix daemon... ')" killproc -p ${PIDFILE} $NIX_DAEMON_BIN -HUP RETVAL=$? echo @@ -105,7 +112,7 @@ case "$1" in fi ;; *) - echo $"Usage: $0 {start|stop|status|restart|condrestart}" + printf '%s' "$(gettext "Usage: $0 {start|stop|status|restart|condrestart}")" exit 2 ;; esac diff --git a/mk/platform.mk b/mk/platform.mk index fe960dedf..22c114a20 100644 --- a/mk/platform.mk +++ b/mk/platform.mk @@ -29,4 +29,8 @@ ifdef HOST_OS HOST_SOLARIS = 1 HOST_UNIX = 1 endif + ifeq ($(HOST_KERNEL), gnu) + HOST_HURD = 1 + HOST_UNIX = 1 + endif endif diff --git a/mk/run-test.sh b/mk/run-test.sh index 543c845e1..7f9f1d5f8 100755 --- a/mk/run-test.sh +++ b/mk/run-test.sh @@ -28,7 +28,7 @@ run_test if [[ "$status" = 0 ]]; then echo "$post_run_msg [${green}PASS$normal]" -elif [[ "$status" = 99 ]]; then +elif [[ "$status" = 77 ]]; then echo "$post_run_msg [${yellow}SKIP$normal]" else echo "$post_run_msg [${red}FAIL$normal]" diff --git a/package.nix b/package.nix index 99ffd5e40..4f18eb8bb 100644 --- a/package.nix +++ b/package.nix @@ -218,7 +218,8 @@ in { ] ++ lib.optional stdenv.hostPlatform.isStatic unixtools.hexdump ; - buildInputs = lib.optionals doBuild [ + buildInputs = lib.optionals doBuild ( + [ brotli bzip2 curl @@ -239,16 +240,14 @@ in { ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid # There have been issues building these dependencies ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin)) - (aws-sdk-cpp.override { - apis = ["s3" "transfer"]; - customMemoryManagement = false; - }) - ; + aws-sdk-cpp + ); - propagatedBuildInputs = [ + propagatedBuildInputs = lib.optionals doBuild ([ boost nlohmann_json - ] ++ lib.optional enableGC boehmgc; + ] ++ lib.optional enableGC boehmgc + ); dontBuild = !attrs.doBuild; doCheck = attrs.doCheck; diff --git a/packaging/components.nix b/packaging/components.nix index 0e369a055..870e9ae61 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -29,6 +29,7 @@ in nix-flake-tests = callPackage ../tests/unit/libflake/package.nix { }; nix-main = callPackage ../src/libmain/package.nix { }; + nix-main-c = callPackage ../src/libmain-c/package.nix { }; nix-cmd = callPackage ../src/libcmd/package.nix { }; diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 66e9f1ea6..5a0981bfb 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -61,11 +61,33 @@ let workDir = null; }; + # Work around weird `--as-needed` linker behavior with BSD, see + # https://github.com/mesonbuild/meson/issues/3593 + bsdNoLinkAsNeeded = finalAttrs: prevAttrs: + lib.optionalAttrs stdenv.hostPlatform.isBSD { + mesonFlags = [ (lib.mesonBool "b_asneeded" false) ] ++ prevAttrs.mesonFlags or []; + }; + + miscGoodPractice = finalAttrs: prevAttrs: + { + strictDeps = prevAttrs.strictDeps or true; + enableParallelBuilding = true; + }; + in scope: { inherit stdenv versionSuffix; version = lib.fileContents ../.version + versionSuffix; + aws-sdk-cpp = (pkgs.aws-sdk-cpp.override { + apis = [ "s3" "transfer" ]; + customMemoryManagement = false; + }).overrideAttrs { + # only a stripped down version is built, which takes a lot less resources + # to build, so we don't need a "big-parallel" machine. + requiredSystemFeatures = [ ]; + }; + libseccomp = pkgs.libseccomp.overrideAttrs (_: rec { version = "2.5.5"; src = pkgs.fetchurl { @@ -130,7 +152,16 @@ scope: { inherit resolvePath filesetToSource; - mkMesonDerivation = f: stdenv.mkDerivation (lib.extends localSourceLayer f); + mkMesonDerivation = f: let + exts = [ + miscGoodPractice + bsdNoLinkAsNeeded + localSourceLayer + ]; + in stdenv.mkDerivation + (lib.extends + (lib.foldr lib.composeExtensions (_: _: {}) exts) + f); inherit (inputs) flake-schemas; } diff --git a/packaging/hydra.nix b/packaging/hydra.nix index d56340231..24c614e67 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -54,6 +54,7 @@ let "nix-flake" "nix-flake-tests" "nix-main" + "nix-main-c" "nix-cmd" "nix-ng" ]; diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 600fc7ee2..82ad7d862 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -11,6 +11,7 @@ #include "machines.hh" #include "shared.hh" +#include "plugin.hh" #include "pathlocks.hh" #include "globals.hh" #include "serialise.hh" @@ -100,7 +101,7 @@ static int main_build_remote(int argc, char * * argv) } std::optional drvPath; - StoreReference storeUri; + std::string storeUri; while (true) { @@ -233,17 +234,16 @@ static int main_build_remote(int argc, char * * argv) lock = -1; try { + storeUri = bestMachine->storeUri.render(); - Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri.render())); + Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", storeUri)); sshStore = bestMachine->openStore(); sshStore->connect(); - storeUri = bestMachine->storeUri; - } catch (std::exception & e) { auto msg = chomp(drainFD(5, false)); printError("cannot build on '%s': %s%s", - bestMachine->storeUri.render(), e.what(), + storeUri, e.what(), msg.empty() ? "" : ": " + msg); bestMachine->enabled = false; continue; @@ -258,15 +258,28 @@ connected: assert(sshStore); - std::cerr << "# accept\n" << storeUri.render() << "\n"; + std::cerr << "# accept\n" << storeUri << "\n"; auto inputs = readStrings(source); auto wantedOutputs = readStrings(source); - AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri.render()) + ".upload-lock", true); + AutoCloseFD uploadLock; + { + auto setUpdateLock = [&](auto && fileName){ + uploadLock = openLockFile(currentLoad + "/" + escapeUri(fileName) + ".upload-lock", true); + }; + try { + setUpdateLock(storeUri); + } catch (SysError & e) { + if (e.errNo != ENAMETOOLONG) throw; + // Try again hashing the store URL so we have a shorter path + auto h = hashString(HashAlgorithm::MD5, storeUri); + setUpdateLock(h.to_string(HashFormat::Base64, false)); + } + } { - Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri.render())); + Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri)); auto old = signal(SIGALRM, handleAlarm); alarm(15 * 60); @@ -279,7 +292,7 @@ connected: auto substitute = settings.buildersUseSubstitutes ? Substitute : NoSubstitute; { - Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri.render())); + Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri)); copyPaths(*store, *sshStore, store->parseStorePathSet(inputs), NoRepair, NoCheckSigs, substitute); } @@ -317,7 +330,7 @@ connected: optResult = sshStore->buildDerivation(*drvPath, (const BasicDerivation &) drv); auto & result = *optResult; if (!result.success()) - throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri.render(), result.errorMsg); + throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg); } else { copyClosure(*store, *sshStore, StorePathSet {*drvPath}, NoRepair, NoCheckSigs, substitute); auto res = sshStore->buildPathsWithResults({ @@ -360,7 +373,7 @@ connected: } if (!missingPaths.empty()) { - Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri.render())); + Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri)); if (auto localStore = store.dynamic_pointer_cast()) for (auto & path : missingPaths) localStore->locksHeld.insert(store->printStorePath(path)); /* FIXME: ugly */ diff --git a/src/external-api-docs/package.nix b/src/external-api-docs/package.nix index da136bbe1..743b3e9b7 100644 --- a/src/external-api-docs/package.nix +++ b/src/external-api-docs/package.nix @@ -53,10 +53,6 @@ mkMesonDerivation (finalAttrs: { echo "doc external-api-docs $out/share/doc/nix/external-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products ''; - enableParallelBuilding = true; - - strictDeps = true; - meta = { platforms = lib.platforms.all; }; diff --git a/src/internal-api-docs/package.nix b/src/internal-api-docs/package.nix index f2077dcaf..07ca6d4d9 100644 --- a/src/internal-api-docs/package.nix +++ b/src/internal-api-docs/package.nix @@ -48,10 +48,6 @@ mkMesonDerivation (finalAttrs: { echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products ''; - enableParallelBuilding = true; - - strictDeps = true; - meta = { platforms = lib.platforms.all; }; diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 9d7f8dbb8..39c25bcc1 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -90,75 +90,11 @@ MixEvalArgs::MixEvalArgs() .longName = "include", .shortName = 'I', .description = R"( - Add *path* to the Nix search path. The Nix search path is - initialized from the colon-separated [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) environment - variable, and is used to look up the location of Nix expressions using [paths](@docroot@/language/types.md#type-path) enclosed in angle - brackets (i.e., ``). + Add *path* to search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md) - For instance, passing + This option may be given multiple times. - ``` - -I /home/eelco/Dev - -I /etc/nixos - ``` - - will cause Nix to look for paths relative to `/home/eelco/Dev` and - `/etc/nixos`, in that order. This is equivalent to setting the - `NIX_PATH` environment variable to - - ``` - /home/eelco/Dev:/etc/nixos - ``` - - It is also possible to match paths against a prefix. For example, - passing - - ``` - -I nixpkgs=/home/eelco/Dev/nixpkgs-branch - -I /etc/nixos - ``` - - will cause Nix to search for `` in - `/home/eelco/Dev/nixpkgs-branch/path` and `/etc/nixos/nixpkgs/path`. - - If a path in the Nix search path starts with `http://` or `https://`, - it is interpreted as the URL of a tarball that will be downloaded and - unpacked to a temporary location. The tarball must consist of a single - top-level directory. For example, passing - - ``` - -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/master.tar.gz - ``` - - tells Nix to download and use the current contents of the `master` - branch in the `nixpkgs` repository. - - The URLs of the tarballs from the official `nixos.org` channels - (see [the manual page for `nix-channel`](../nix-channel.md)) can be - abbreviated as `channel:`. For instance, the - following two flags are equivalent: - - ``` - -I nixpkgs=channel:nixos-21.05 - -I nixpkgs=https://nixos.org/channels/nixos-21.05/nixexprs.tar.xz - ``` - - You can also fetch source trees using [flake URLs](./nix3-flake.md#url-like-syntax) and add them to the - search path. For instance, - - ``` - -I nixpkgs=flake:nixpkgs - ``` - - specifies that the prefix `nixpkgs` shall refer to the source tree - downloaded from the `nixpkgs` entry in the flake registry. Similarly, - - ``` - -I nixpkgs=flake:github:NixOS/nixpkgs/nixos-22.05 - ``` - - makes `` refer to a particular branch of the - `NixOS/nixpkgs` repository on GitHub. + Paths added through `-I` take precedence over the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path) and the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH). )", .category = category, .labels = {"path"}, @@ -213,7 +149,7 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state) auto v = state.allocValue(); std::visit(overloaded { [&](const AutoArgExpr & arg) { - state.mkThunk_(*v, state.parseExprFromString(arg.expr, state.rootPath("."))); + state.mkThunk_(*v, state.parseExprFromString(arg.expr, compatibilitySettings.nixShellShebangArgumentsRelativeToScript ? state.rootPath(absPath(getCommandBaseDir())) : state.rootPath("."))); }, [&](const AutoArgString & arg) { v->mkString(arg.s); diff --git a/src/libcmd/compatibility-settings.hh b/src/libcmd/compatibility-settings.hh index 5dc0eaf2b..a129a957a 100644 --- a/src/libcmd/compatibility-settings.hh +++ b/src/libcmd/compatibility-settings.hh @@ -7,12 +7,29 @@ struct CompatibilitySettings : public Config CompatibilitySettings() = default; + // Added in Nix 2.24, July 2024. Setting nixShellAlwaysLooksForShellNix{this, true, "nix-shell-always-looks-for-shell-nix", R"( Before Nix 2.24, [`nix-shell`](@docroot@/command-ref/nix-shell.md) would only look at `shell.nix` if it was in the working directory - when no file was specified. Since Nix 2.24, `nix-shell` always looks for a `shell.nix`, whether that's in the working directory, or in a directory that was passed as an argument. - You may set this to `false` to revert to the Nix 2.3 behavior. + You may set this to `false` to temporarily revert to the behavior of Nix 2.23 and older. + + Using this setting is not recommended. + It will be deprecated and removed. + )"}; + + // Added in Nix 2.24, July 2024. + Setting nixShellShebangArgumentsRelativeToScript{ + this, true, "nix-shell-shebang-arguments-relative-to-script", R"( + Before Nix 2.24, relative file path expressions in arguments in a `nix-shell` shebang were resolved relative to the working directory. + + Since Nix 2.24, `nix-shell` resolves these paths in a manner that is relative to the [base directory](@docroot@/glossary.md#gloss-base-directory), defined as the script's directory. + + You may set this to `false` to temporarily revert to the behavior of Nix 2.23 and older. + + Using this setting is not recommended. + It will be deprecated and removed. )"}; }; diff --git a/src/libcmd/installable-value.hh b/src/libcmd/installable-value.hh index f300d392b..798cb5e1a 100644 --- a/src/libcmd/installable-value.hh +++ b/src/libcmd/installable-value.hh @@ -66,7 +66,7 @@ struct ExtraPathInfoValue : ExtraPathInfo }; /** - * An Installable which corresponds a Nix langauge value, in addition to + * An Installable which corresponds a Nix language value, in addition to * a collection of \ref DerivedPath "derived paths". */ struct InstallableValue : Installable diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc index 88c3f640b..6a0d05d9f 100644 --- a/src/libcmd/markdown.cc +++ b/src/libcmd/markdown.cc @@ -1,21 +1,23 @@ #include "markdown.hh" -#include "util.hh" +#include "environment-variables.hh" +#include "error.hh" #include "finally.hh" #include "terminal.hh" #if HAVE_LOWDOWN -# include -# include +# include +# include #endif namespace nix { -std::string renderMarkdownToTerminal(std::string_view markdown) -{ #if HAVE_LOWDOWN +static std::string doRenderMarkdownToTerminal(std::string_view markdown) +{ int windowWidth = getWindowSize().second; - struct lowdown_opts opts { + struct lowdown_opts opts + { .type = LOWDOWN_TERM, .maxdepth = 20, .cols = (size_t) std::max(windowWidth - 5, 60), @@ -51,9 +53,21 @@ std::string renderMarkdownToTerminal(std::string_view markdown) throw Error("allocation error while rendering Markdown"); return filterANSIEscapes(std::string(buf->data, buf->size), !isTTY()); -#else - return std::string(markdown); -#endif } +std::string renderMarkdownToTerminal(std::string_view markdown) +{ + if (auto e = getEnv("_NIX_TEST_RAW_MARKDOWN"); e && *e == "1") + return std::string(markdown); + else + return doRenderMarkdownToTerminal(markdown); } + +#else +std::string renderMarkdownToTerminal(std::string_view markdown) +{ + return std::string(markdown); +} +#endif + +} // namespace nix diff --git a/src/libcmd/markdown.hh b/src/libcmd/markdown.hh index a04d32a4f..66db1736c 100644 --- a/src/libcmd/markdown.hh +++ b/src/libcmd/markdown.hh @@ -1,10 +1,17 @@ #pragma once ///@file -#include "types.hh" +#include namespace nix { +/** + * Render the given Markdown text to the terminal. + * + * If Nix is compiled without Markdown support, this function will return the input text as-is. + * + * The renderer takes into account the terminal width, and wraps text accordingly. + */ std::string renderMarkdownToTerminal(std::string_view markdown); } diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build index 687b37aac..c484cf998 100644 --- a/src/libcmd/meson.build +++ b/src/libcmd/meson.build @@ -30,6 +30,8 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') +subdir('build-utils-meson/threads') + nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') deps_public += nlohmann_json diff --git a/src/libcmd/package.nix b/src/libcmd/package.nix index ec3aa4660..cde494901 100644 --- a/src/libcmd/package.nix +++ b/src/libcmd/package.nix @@ -93,14 +93,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - # TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated - # to work with `strictDeps`. - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libcmd/repl-interacter.cc b/src/libcmd/repl-interacter.cc index b285c8a9a..187af46ea 100644 --- a/src/libcmd/repl-interacter.cc +++ b/src/libcmd/repl-interacter.cc @@ -19,6 +19,7 @@ extern "C" { #include "repl-interacter.hh" #include "file-system.hh" #include "repl.hh" +#include "environment-variables.hh" namespace nix { @@ -34,6 +35,7 @@ void sigintHandler(int signo) static detail::ReplCompleterMixin * curRepl; // ugly +#ifndef USE_READLINE static char * completionCallback(char * s, int * match) { auto possible = curRepl->completePrefix(s); @@ -100,6 +102,7 @@ static int listPossibleCallback(char * s, char *** avp) return ac; } +#endif ReadlineLikeInteracter::Guard ReadlineLikeInteracter::init(detail::ReplCompleterMixin * repl) { @@ -175,10 +178,23 @@ bool ReadlineLikeInteracter::getLine(std::string & input, ReplPromptType promptT return true; } + // editline doesn't echo the input to the output when non-interactive, unlike readline + // this results in a different behavior when running tests. The echoing is + // quite useful for reading the test output, so we add it here. + if (auto e = getEnv("_NIX_TEST_REPL_ECHO"); s && e && *e == "1") + { +#ifndef USE_READLINE + // This is probably not right for multi-line input, but we don't use that + // in the characterisation tests, so it's fine. + std::cout << promptForType(promptType) << s << std::endl; +#endif + } + if (!s) return false; input += s; input += '\n'; + return true; } diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index b5d0816dd..efc04b029 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -217,7 +217,7 @@ ReplExitStatus NixRepl::mainLoop() case ProcessLineResult::PromptAgain: break; default: - abort(); + unreachable(); } } catch (ParseError & e) { if (e.msg().find("unexpected end of file") != std::string::npos) { @@ -644,9 +644,6 @@ ProcessLineResult NixRepl::processLine(std::string line) fallbackPos = attr->pos; fallbackDoc = state->getDocCommentForPos(fallbackPos); } - - } else { - evalString(arg, v); } evalString(arg, v); diff --git a/src/libexpr-c/meson.build b/src/libexpr-c/meson.build index 2a2669b3e..6db5b83b8 100644 --- a/src/libexpr-c/meson.build +++ b/src/libexpr-c/meson.build @@ -29,6 +29,8 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') +subdir('build-utils-meson/threads') + # TODO rename, because it will conflict with downstream projects configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc index 547453f8f..8f21d7022 100644 --- a/src/libexpr-c/nix_api_expr.cc +++ b/src/libexpr-c/nix_api_expr.cc @@ -14,10 +14,10 @@ #include "nix_api_util.h" #include "nix_api_util_internal.h" -#ifdef HAVE_BOEHMGC -#include -#define GC_INCLUDE_NEW 1 -#include "gc_cpp.h" +#if HAVE_BOEHMGC +# include +# define GC_INCLUDE_NEW 1 +# include "gc_cpp.h" #endif nix_err nix_libexpr_init(nix_c_context * context) @@ -131,7 +131,7 @@ void nix_state_free(EvalState * state) delete state; } -#ifdef HAVE_BOEHMGC +#if HAVE_BOEHMGC std::unordered_map< const void *, unsigned int, @@ -207,7 +207,7 @@ nix_err nix_value_decref(nix_c_context * context, nix_value *x) void nix_gc_register_finalizer(void * obj, void * cd, void (*finalizer)(void * obj, void * cd)) { -#ifdef HAVE_BOEHMGC +#if HAVE_BOEHMGC GC_REGISTER_FINALIZER(obj, finalizer, cd, 0, 0); #endif } diff --git a/src/libexpr-c/nix_api_expr.h b/src/libexpr-c/nix_api_expr.h index adf8b65b1..1764b49f3 100644 --- a/src/libexpr-c/nix_api_expr.h +++ b/src/libexpr-c/nix_api_expr.h @@ -14,6 +14,16 @@ #include "nix_api_util.h" #include +#ifndef __has_c_attribute +# define __has_c_attribute(x) 0 +#endif + +#if __has_c_attribute(deprecated) +# define NIX_DEPRECATED(msg) [[deprecated(msg)]] +#else +# define NIX_DEPRECATED(msg) +#endif + #ifdef __cplusplus extern "C" { #endif @@ -45,7 +55,7 @@ typedef struct EvalState EvalState; // nix::EvalState * @see nix_value_incref, nix_value_decref */ typedef struct nix_value nix_value; -[[deprecated("use nix_value instead")]] typedef nix_value Value; +NIX_DEPRECATED("use nix_value instead") typedef nix_value Value; // Function prototypes /** diff --git a/src/libexpr-c/nix_api_external.cc b/src/libexpr-c/nix_api_external.cc index 3565092a4..fa78eb5df 100644 --- a/src/libexpr-c/nix_api_external.cc +++ b/src/libexpr-c/nix_api_external.cc @@ -14,7 +14,7 @@ #include -#ifdef HAVE_BOEHMGC +#if HAVE_BOEHMGC # include "gc/gc.h" # define GC_INCLUDE_NEW 1 # include "gc_cpp.h" @@ -174,7 +174,7 @@ ExternalValue * nix_create_external_value(nix_c_context * context, NixCExternalV context->last_err_code = NIX_OK; try { auto ret = new -#ifdef HAVE_BOEHMGC +#if HAVE_BOEHMGC (GC) #endif NixCExternalValue(*desc, v); diff --git a/src/libexpr-c/nix_api_value.cc b/src/libexpr-c/nix_api_value.cc index cb5d9ee89..845e87935 100644 --- a/src/libexpr-c/nix_api_value.cc +++ b/src/libexpr-c/nix_api_value.cc @@ -14,7 +14,7 @@ #include "nix_api_value.h" #include "value/context.hh" -#ifdef HAVE_BOEHMGC +#if HAVE_BOEHMGC # include "gc/gc.h" # define GC_INCLUDE_NEW 1 # include "gc_cpp.h" @@ -131,7 +131,7 @@ PrimOp * nix_alloc_primop( try { using namespace std::placeholders; auto p = new -#ifdef HAVE_BOEHMGC +#if HAVE_BOEHMGC (GC) #endif nix::PrimOp{ diff --git a/src/libexpr-c/package.nix b/src/libexpr-c/package.nix index 0b895437b..eb42195a4 100644 --- a/src/libexpr-c/package.nix +++ b/src/libexpr-c/package.nix @@ -63,12 +63,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index d61d93630..2f67260c5 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -134,7 +134,7 @@ std::pair findPackageFilename(EvalState & state, Value & v return {SourcePath{path.accessor, CanonPath(fn.substr(0, colon))}, lineno}; } catch (std::invalid_argument & e) { fail(); - abort(); + unreachable(); } } diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 0307236a4..043ec23d3 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -95,7 +95,7 @@ struct AttrDb { try { auto state(_state->lock()); - if (!failed) + if (!failed && state->txn->active) state->txn->commit(); state->txn.reset(); } catch (...) { diff --git a/src/libexpr/eval-error.cc b/src/libexpr/eval-error.cc index bd84e0428..cdb0b4772 100644 --- a/src/libexpr/eval-error.cc +++ b/src/libexpr/eval-error.cc @@ -92,6 +92,14 @@ void EvalErrorBuilder::debugThrow() throw error; } +template +void EvalErrorBuilder::panic() +{ + logError(error.info()); + printError("This is a bug! An unexpected condition occurred, causing the Nix evaluator to have to stop. If you could share a reproducible example or a core dump, please open an issue at https://github.com/NixOS/nix/issues"); + abort(); +} + template class EvalErrorBuilder; template class EvalErrorBuilder; template class EvalErrorBuilder; diff --git a/src/libexpr/eval-error.hh b/src/libexpr/eval-error.hh index 2f6046473..ed004eb53 100644 --- a/src/libexpr/eval-error.hh +++ b/src/libexpr/eval-error.hh @@ -110,6 +110,12 @@ public: * Delete the `EvalErrorBuilder` and throw the underlying exception. */ [[gnu::noinline, gnu::noreturn]] void debugThrow(); + + /** + * A programming error or fatal condition occurred. Abort the process for core dump and debugging. + * This does not print a proper backtrace, because unwinding the stack is destructive. + */ + [[gnu::noinline, gnu::noreturn]] void panic(); }; } diff --git a/src/libexpr/eval-gc.cc b/src/libexpr/eval-gc.cc index baf9df332..2f0e8c0c9 100644 --- a/src/libexpr/eval-gc.cc +++ b/src/libexpr/eval-gc.cc @@ -1,5 +1,7 @@ #include "error.hh" #include "environment-variables.hh" +#include "eval-settings.hh" +#include "config-global.hh" #include "serialise.hh" #include "eval-gc.hh" @@ -84,14 +86,17 @@ void fixupBoehmStackPointer(void ** sp_ptr, void * _pthread_id) { void *& sp = *sp_ptr; auto pthread_id = reinterpret_cast(_pthread_id); +# ifndef __APPLE__ pthread_attr_t pattr; +# endif size_t osStackSize; - void * osStackLow; + // The low address of the stack, which grows down. + void * osStackLimit; void * osStackBase; # ifdef __APPLE__ osStackSize = pthread_get_stacksize_np(pthread_id); - osStackLow = pthread_get_stackaddr_np(pthread_id); + osStackLimit = pthread_get_stackaddr_np(pthread_id); # else if (pthread_attr_init(&pattr)) { throw Error("fixupBoehmStackPointer: pthread_attr_init failed"); @@ -110,18 +115,18 @@ void fixupBoehmStackPointer(void ** sp_ptr, void * _pthread_id) # else # error "Need one of `pthread_attr_get_np` or `pthread_getattr_np`" # endif - if (pthread_attr_getstack(&pattr, &osStackLow, &osStackSize)) { + if (pthread_attr_getstack(&pattr, &osStackLimit, &osStackSize)) { throw Error("fixupBoehmStackPointer: pthread_attr_getstack failed"); } if (pthread_attr_destroy(&pattr)) { throw Error("fixupBoehmStackPointer: pthread_attr_destroy failed"); } # endif - osStackBase = (char *) osStackLow + osStackSize; + osStackBase = (char *) osStackLimit + osStackSize; // NOTE: We assume the stack grows down, as it does on all architectures we support. // Architectures that grow the stack up are rare. - if (sp >= osStackBase || sp < osStackLow) { // lo is outside the os stack - sp = osStackBase; + if (sp >= osStackBase || sp < osStackLimit) { // sp is outside the os stack + sp = osStackLimit; } } @@ -155,6 +160,10 @@ static inline void initGCReal() there. */ GC_set_no_dls(1); + /* Enable perf measurements. This is just a setting; not much of a + start of something. */ + GC_start_performance_measurement(); + GC_INIT(); GC_set_oom_fn(oomHandler); @@ -202,6 +211,14 @@ static inline void initGCReal() } } +static size_t gcCyclesAfterInit = 0; + +size_t getGCCycles() +{ + assertGCInitialized(); + return static_cast(GC_get_gc_no()) - gcCyclesAfterInit; +} + #endif static bool gcInitialised = false; @@ -213,8 +230,16 @@ void initGC() #if HAVE_BOEHMGC initGCReal(); + + gcCyclesAfterInit = GC_get_gc_no(); #endif + // NIX_PATH must override the regular setting + // See the comment in applyConfig + if (auto nixPathEnv = getEnv("NIX_PATH")) { + globalConfig.set("nix-path", concatStringsSep(" ", EvalSettings::parseNixPath(nixPathEnv.value()))); + } + gcInitialised = true; } @@ -223,4 +248,4 @@ void assertGCInitialized() assert(gcInitialised); } -} +} // namespace nix diff --git a/src/libexpr/eval-gc.hh b/src/libexpr/eval-gc.hh index cd4ea914d..005175eb7 100644 --- a/src/libexpr/eval-gc.hh +++ b/src/libexpr/eval-gc.hh @@ -1,6 +1,8 @@ #pragma once ///@file +#include + namespace nix { /** @@ -13,4 +15,11 @@ void initGC(); */ void assertGCInitialized(); -} +#ifdef HAVE_BOEHMGC +/** + * The number of GC cycles since initGC(). + */ +size_t getGCCycles(); +#endif + +} // namespace nix diff --git a/src/libexpr/eval-settings.cc b/src/libexpr/eval-settings.cc index e2151aa7f..2846eccbc 100644 --- a/src/libexpr/eval-settings.cc +++ b/src/libexpr/eval-settings.cc @@ -8,7 +8,7 @@ namespace nix { /* Very hacky way to parse $NIX_PATH, which is colon-separated, but can contain URLs (e.g. "nixpkgs=https://bla...:foo=https://"). */ -static Strings parseNixPath(const std::string & s) +Strings EvalSettings::parseNixPath(const std::string & s) { Strings res; @@ -48,15 +48,12 @@ EvalSettings::EvalSettings(bool & readOnlyMode, EvalSettings::LookupPathHooks lo : readOnlyMode{readOnlyMode} , lookupPathHooks{lookupPathHooks} { - auto var = getEnv("NIX_PATH"); - if (var) nixPath = parseNixPath(*var); - - var = getEnv("NIX_ABORT_ON_WARN"); + auto var = getEnv("NIX_ABORT_ON_WARN"); if (var && (var == "1" || var == "yes" || var == "true")) builtinsAbortOnWarn = true; } -Strings EvalSettings::getDefaultNixPath() const +Strings EvalSettings::getDefaultNixPath() { Strings res; auto add = [&](const Path & p, const std::string & s = std::string()) { @@ -69,11 +66,9 @@ Strings EvalSettings::getDefaultNixPath() const } }; - if (!restrictEval && !pureEval) { - add(getNixDefExpr() + "/channels"); - add(rootChannelsDir() + "/nixpkgs", "nixpkgs"); - add(rootChannelsDir()); - } + add(getNixDefExpr() + "/channels"); + add(rootChannelsDir() + "/nixpkgs", "nixpkgs"); + add(rootChannelsDir()); return res; } diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index b915ba530..0cfc14c1b 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -43,10 +43,12 @@ struct EvalSettings : Config bool & readOnlyMode; - Strings getDefaultNixPath() const; + static Strings getDefaultNixPath(); static bool isPseudoUrl(std::string_view s); + static Strings parseNixPath(const std::string & s); + static std::string resolvePseudoUrl(std::string_view url); LookupPathHooks lookupPathHooks; @@ -63,7 +65,7 @@ struct EvalSettings : Config extern "C" typedef void (*ValueInitialiser) (EvalState & state, Value & v); ``` - The [Nix C++ API documentation](@docroot@/contributing/documentation.md#api-documentation) has more details on evaluator internals. + The [Nix C++ API documentation](@docroot@/development/documentation.md#api-documentation) has more details on evaluator internals. - `builtins.exec` *arguments* @@ -71,25 +73,30 @@ struct EvalSettings : Config )"}; Setting nixPath{ - this, getDefaultNixPath(), "nix-path", + this, {}, "nix-path", R"( List of search paths to use for [lookup path](@docroot@/language/constructs/lookup-path.md) resolution. This setting determines the value of [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath) and can be used with [`builtins.findFile`](@docroot@/language/builtins.md#builtins-findFile). - The default value is + - The configuration setting is overridden by the [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) + environment variable. + - `NIX_PATH` is overridden by [specifying the setting as the command line flag](@docroot@/command-ref/conf-file.md#command-line-flags) `--nix-path`. + - Any current value is extended by the [`-I` option](@docroot@/command-ref/opt-common.md#opt-I) or `--extra-nix-path`. - ``` - $HOME/.nix-defexpr/channels - nixpkgs=$NIX_STATE_DIR/profiles/per-user/root/channels/nixpkgs - $NIX_STATE_DIR/profiles/per-user/root/channels - ``` + If the respective paths are accessible, the default values are: - It can be overridden with the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH) or the [`-I` command line option](@docroot@/command-ref/opt-common.md#opt-I). + - `$HOME/.nix-defexpr/channels` + - `nixpkgs=$NIX_STATE_DIR/profiles/per-user/root/channels/nixpkgs` + - `$NIX_STATE_DIR/profiles/per-user/root/channels` + + See [`NIX_STATE_DIR`](@docroot@/command-ref/env-common.md#env-NIX_STATE_DIR) for details. > **Note** > - > If [pure evaluation](#conf-pure-eval) is enabled, `nixPath` evaluates to the empty list `[ ]`. + > If [restricted evaluation](@docroot@/command-ref/conf-file.md#conf-restrict-eval) is enabled, the default value is empty. + > + > If [pure evaluation](#conf-pure-eval) is enabled, `builtins.nixPath` *always* evaluates to the empty list `[ ]`. )", {}, false}; Setting currentSystem{ diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 4f01d0a62..de5d85821 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -149,7 +149,7 @@ std::string_view showType(ValueType type, bool withArticle) case nFloat: return WA("a", "float"); case nThunk: return WA("a", "thunk"); } - abort(); + unreachable(); } @@ -215,7 +215,7 @@ static Symbol getName(const AttrName & name, EvalState & state, Env & env) static constexpr size_t BASE_ENV_SIZE = 128; EvalState::EvalState( - const LookupPath & _lookupPath, + const LookupPath & lookupPathFromArguments, ref store, const fetchers::Settings & fetchSettings, const EvalSettings & settings, @@ -331,12 +331,21 @@ EvalState::EvalState( vStringSymlink.mkString("symlink"); vStringUnknown.mkString("unknown"); - /* Initialise the Nix expression search path. */ + /* Construct the Nix expression search path. */ + assert(lookupPath.elements.empty()); if (!settings.pureEval) { - for (auto & i : _lookupPath.elements) + for (auto & i : lookupPathFromArguments.elements) { lookupPath.elements.emplace_back(LookupPath::Elem {i}); - for (auto & i : settings.nixPath.get()) + } + /* $NIX_PATH overriding regular settings is implemented as a hack in `initGC()` */ + for (auto & i : settings.nixPath.get()) { lookupPath.elements.emplace_back(LookupPath::Elem::parse(i)); + } + if (!settings.restrictEval) { + for (auto & i : EvalSettings::getDefaultNixPath()) { + lookupPath.elements.emplace_back(LookupPath::Elem::parse(i)); + } + } } /* Allow access to all paths in the search path. */ @@ -771,7 +780,7 @@ void EvalState::runDebugRepl(const Error * error, const Env & env, const Expr & case ReplExitStatus::Continue: break; default: - abort(); + unreachable(); } } } @@ -1140,7 +1149,7 @@ inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v, const PosIdx po void Expr::eval(EvalState & state, Env & env, Value & v) { - abort(); + unreachable(); } @@ -1573,7 +1582,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & .withFrame(*fun.payload.lambda.env, lambda) .debugThrow(); } - abort(); // can't happen + unreachable(); } } @@ -1825,9 +1834,24 @@ void ExprIf::eval(EvalState & state, Env & env, Value & v) void ExprAssert::eval(EvalState & state, Env & env, Value & v) { if (!state.evalBool(env, cond, pos, "in the condition of the assert statement")) { - std::ostringstream out; - cond->show(state.symbols, out); - state.error("assertion '%1%' failed", out.str()).atPos(pos).withFrame(env, *this).debugThrow(); + auto exprStr = ({ + std::ostringstream out; + cond->show(state.symbols, out); + out.str(); + }); + + if (auto eq = dynamic_cast(cond)) { + try { + Value v1; eq->e1->eval(state, env, v1); + Value v2; eq->e2->eval(state, env, v2); + state.assertEqValues(v1, v2, eq->pos, "in an equality assertion"); + } catch (AssertionError & e) { + e.addTrace(state.positions[pos], "while evaluating the condition of the assertion '%s'", exprStr); + throw; + } + } + + state.error("assertion '%1%' failed", exprStr).atPos(pos).withFrame(env, *this).debugThrow(); } body->eval(state, env, v); } @@ -2484,6 +2508,214 @@ SingleDerivedPath EvalState::coerceToSingleDerivedPath(const PosIdx pos, Value & } + +// NOTE: This implementation must match eqValues! +// We accept this burden because informative error messages for +// `assert a == b; x` are critical for our users' testing UX. +void EvalState::assertEqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx) +{ + // This implementation must match eqValues. + forceValue(v1, pos); + forceValue(v2, pos); + + if (&v1 == &v2) + return; + + // Special case type-compatibility between float and int + if ((v1.type() == nInt || v1.type() == nFloat) && (v2.type() == nInt || v2.type() == nFloat)) { + if (eqValues(v1, v2, pos, errorCtx)) { + return; + } else { + error( + "%s with value '%s' is not equal to %s with value '%s'", + showType(v1), + ValuePrinter(*this, v1, errorPrintOptions), + showType(v2), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + } + + if (v1.type() != v2.type()) { + error( + "%s of value '%s' is not equal to %s of value '%s'", + showType(v1), + ValuePrinter(*this, v1, errorPrintOptions), + showType(v2), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + + switch (v1.type()) { + case nInt: + if (v1.integer() != v2.integer()) { + error("integer '%d' is not equal to integer '%d'", v1.integer(), v2.integer()).debugThrow(); + } + return; + + case nBool: + if (v1.boolean() != v2.boolean()) { + error( + "boolean '%s' is not equal to boolean '%s'", + ValuePrinter(*this, v1, errorPrintOptions), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + return; + + case nString: + if (strcmp(v1.c_str(), v2.c_str()) != 0) { + error( + "string '%s' is not equal to string '%s'", + ValuePrinter(*this, v1, errorPrintOptions), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + return; + + case nPath: + if (v1.payload.path.accessor != v2.payload.path.accessor) { + error( + "path '%s' is not equal to path '%s' because their accessors are different", + ValuePrinter(*this, v1, errorPrintOptions), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + if (strcmp(v1.payload.path.path, v2.payload.path.path) != 0) { + error( + "path '%s' is not equal to path '%s'", + ValuePrinter(*this, v1, errorPrintOptions), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + return; + + case nNull: + return; + + case nList: + if (v1.listSize() != v2.listSize()) { + error( + "list of size '%d' is not equal to list of size '%d', left hand side is '%s', right hand side is '%s'", + v1.listSize(), + v2.listSize(), + ValuePrinter(*this, v1, errorPrintOptions), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + for (size_t n = 0; n < v1.listSize(); ++n) { + try { + assertEqValues(*v1.listElems()[n], *v2.listElems()[n], pos, errorCtx); + } catch (Error & e) { + e.addTrace(positions[pos], "while comparing list element %d", n); + throw; + } + } + return; + + case nAttrs: { + if (isDerivation(v1) && isDerivation(v2)) { + auto i = v1.attrs()->get(sOutPath); + auto j = v2.attrs()->get(sOutPath); + if (i && j) { + try { + assertEqValues(*i->value, *j->value, pos, errorCtx); + return; + } catch (Error & e) { + e.addTrace(positions[pos], "while comparing a derivation by its '%s' attribute", "outPath"); + throw; + } + assert(false); + } + } + + if (v1.attrs()->size() != v2.attrs()->size()) { + error( + "attribute names of attribute set '%s' differs from attribute set '%s'", + ValuePrinter(*this, v1, errorPrintOptions), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + + // Like normal comparison, we compare the attributes in non-deterministic Symbol index order. + // This function is called when eqValues has found a difference, so to reliably + // report about its result, we should follow in its literal footsteps and not + // try anything fancy that could lead to an error. + Bindings::const_iterator i, j; + for (i = v1.attrs()->begin(), j = v2.attrs()->begin(); i != v1.attrs()->end(); ++i, ++j) { + if (i->name != j->name) { + // A difference in a sorted list means that one attribute is not contained in the other, but we don't + // know which. Let's find out. Could use <, but this is more clear. + if (!v2.attrs()->get(i->name)) { + error( + "attribute name '%s' is contained in '%s', but not in '%s'", + symbols[i->name], + ValuePrinter(*this, v1, errorPrintOptions), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + if (!v1.attrs()->get(j->name)) { + error( + "attribute name '%s' is missing in '%s', but is contained in '%s'", + symbols[j->name], + ValuePrinter(*this, v1, errorPrintOptions), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + assert(false); + } + try { + assertEqValues(*i->value, *j->value, pos, errorCtx); + } catch (Error & e) { + // The order of traces is reversed, so this presents as + // where left hand side is + // at + // where right hand side is + // at + // while comparing attribute '' + if (j->pos != noPos) + e.addTrace(positions[j->pos], "where right hand side is"); + if (i->pos != noPos) + e.addTrace(positions[i->pos], "where left hand side is"); + e.addTrace(positions[pos], "while comparing attribute '%s'", symbols[i->name]); + throw; + } + } + return; + } + + case nFunction: + error("distinct functions and immediate comparisons of identical functions compare as unequal") + .debugThrow(); + + case nExternal: + if (!(*v1.external() == *v2.external())) { + error( + "external value '%s' is not equal to external value '%s'", + ValuePrinter(*this, v1, errorPrintOptions), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + return; + + case nFloat: + // !!! + if (!(v1.fpoint() == v2.fpoint())) { + error("float '%f' is not equal to float '%f'", v1.fpoint(), v2.fpoint()).debugThrow(); + } + return; + + case nThunk: // Must not be left by forceValue + assert(false); + default: // Note that we pass compiler flags that should make `default:` unreachable. + // Also note that this probably ran after `eqValues`, which implements + // the same logic more efficiently (without having to unwind stacks), + // so maybe `assertEqValues` and `eqValues` are out of sync. Check it for solutions. + error("assertEqValues: cannot compare %1% with %2%", showType(v1), showType(v2)).withTrace(pos, errorCtx).panic(); + } +} + +// This implementation must match assertEqValues bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx) { forceValue(v1, pos); @@ -2557,11 +2789,13 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v return *v1.external() == *v2.external(); case nFloat: + // !!! return v1.fpoint() == v2.fpoint(); case nThunk: // Must not be left by forceValue - default: - error("cannot compare %1% with %2%", showType(v1), showType(v2)).withTrace(pos, errorCtx).debugThrow(); + assert(false); + default: // Note that we pass compiler flags that should make `default:` unreachable. + error("eqValues: cannot compare %1% with %2%", showType(v1), showType(v2)).withTrace(pos, errorCtx).panic(); } } @@ -2610,6 +2844,11 @@ void EvalState::printStatistics() #if HAVE_BOEHMGC GC_word heapSize, totalBytes; GC_get_heap_usage_safe(&heapSize, 0, 0, 0, &totalBytes); + double gcFullOnlyTime = ({ + auto ms = GC_get_full_gc_total_time(); + ms * 0.001; + }); + auto gcCycles = getGCCycles(); #endif auto outPath = getEnv("NIX_SHOW_STATS_PATH").value_or("-"); @@ -2620,6 +2859,15 @@ void EvalState::printStatistics() #ifndef _WIN32 // TODO implement topObj["cpuTime"] = cpuTime; #endif + topObj["time"] = { +#ifndef _WIN32 // TODO implement + {"cpu", cpuTime}, +#endif +#if HAVE_BOEHMGC + {GC_is_incremental_mode() ? "gcNonIncremental" : "gc", gcFullOnlyTime}, + {GC_is_incremental_mode() ? "gcNonIncrementalFraction" : "gcFraction", gcFullOnlyTime / cpuTime}, +#endif + }; topObj["envs"] = { {"number", nrEnvs}, {"elements", nrValuesInEnvs}, @@ -2661,6 +2909,7 @@ void EvalState::printStatistics() topObj["gc"] = { {"heapSize", heapSize}, {"totalBytes", totalBytes}, + {"cycles", gcCycles}, }; #endif diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index d376046ae..ddf5dcf94 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -130,7 +130,7 @@ struct Constant typedef std::map ValMap; #endif -typedef std::map DocCommentMap; +typedef std::unordered_map DocCommentMap; struct Env { @@ -335,7 +335,7 @@ private: * Associate source positions of certain AST nodes with their preceding doc comment, if they have one. * Grouped by file. */ - std::map positionToDocComment; + std::unordered_map positionToDocComment; LookupPath lookupPath; @@ -655,6 +655,15 @@ public: */ bool eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx); + /** + * Like `eqValues`, but throws an `AssertionError` if not equal. + * + * WARNING: + * Callers should call `eqValues` first and report if `assertEqValues` behaves + * incorrectly. (e.g. if it doesn't throw if eqValues returns false or vice versa) + */ + void assertEqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx); + bool isFunctor(Value & fun); // FIXME: use std::span diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 20bee193f..21074bdd8 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -42,7 +42,7 @@ class JSONSax : nlohmann::json_sax { auto attrs2 = state.buildBindings(attrs.size()); for (auto & i : attrs) attrs2.insert(i.first, i.second); - parent->value(state).mkAttrs(attrs2.alreadySorted()); + parent->value(state).mkAttrs(attrs2); return std::move(parent); } void add() override { v = nullptr; } @@ -80,42 +80,42 @@ class JSONSax : nlohmann::json_sax { public: JSONSax(EvalState & state, Value & v) : state(state), rs(new JSONState(&v)) {}; - bool null() + bool null() override { rs->value(state).mkNull(); rs->add(); return true; } - bool boolean(bool val) + bool boolean(bool val) override { rs->value(state).mkBool(val); rs->add(); return true; } - bool number_integer(number_integer_t val) + bool number_integer(number_integer_t val) override { rs->value(state).mkInt(val); rs->add(); return true; } - bool number_unsigned(number_unsigned_t val) + bool number_unsigned(number_unsigned_t val) override { rs->value(state).mkInt(val); rs->add(); return true; } - bool number_float(number_float_t val, const string_t & s) + bool number_float(number_float_t val, const string_t & s) override { rs->value(state).mkFloat(val); rs->add(); return true; } - bool string(string_t & val) + bool string(string_t & val) override { rs->value(state).mkString(val); rs->add(); @@ -123,7 +123,7 @@ public: } #if NLOHMANN_JSON_VERSION_MAJOR >= 3 && NLOHMANN_JSON_VERSION_MINOR >= 8 - bool binary(binary_t&) + bool binary(binary_t&) override { // This function ought to be unreachable assert(false); @@ -131,35 +131,35 @@ public: } #endif - bool start_object(std::size_t len) + bool start_object(std::size_t len) override { rs = std::make_unique(std::move(rs)); return true; } - bool key(string_t & name) + bool key(string_t & name) override { dynamic_cast(rs.get())->key(name, state); return true; } - bool end_object() { + bool end_object() override { rs = rs->resolve(state); rs->add(); return true; } - bool end_array() { + bool end_array() override { return end_object(); } - bool start_array(size_t len) { + bool start_array(size_t len) override { rs = std::make_unique(std::move(rs), len != std::numeric_limits::max() ? len : 128); return true; } - bool parse_error(std::size_t, const std::string&, const nlohmann::detail::exception& ex) { + bool parse_error(std::size_t, const std::string&, const nlohmann::detail::exception& ex) override { throw JSONParseError("%s", ex.what()); } }; diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 58401be8e..eb1825b7c 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -67,6 +67,14 @@ static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length) return {result, size_t(t - result)}; } +static void requireExperimentalFeature(const ExperimentalFeature & feature, const Pos & pos) +{ + if (!experimentalFeatureSettings.isEnabled(feature)) + throw ParseError(ErrorInfo{ + .msg = HintFmt("experimental Nix feature '%1%' is disabled; add '--extra-experimental-features %1%' to enable it", showExperimentalFeature(feature)), + .pos = pos, + }); +} } @@ -119,6 +127,12 @@ or { return OR_KW; } \-\> { return IMPL; } \/\/ { return UPDATE; } \+\+ { return CONCAT; } +\<\| { requireExperimentalFeature(Xp::PipeOperators, state->positions[CUR_POS]); + return PIPE_FROM; + } +\|\> { requireExperimentalFeature(Xp::PipeOperators, state->positions[CUR_POS]); + return PIPE_INTO; + } {ID} { yylval->id = {yytext, (size_t) yyleng}; return ID; } {INT} { errno = 0; diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index fa90e7b41..4d8a38b43 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -32,6 +32,7 @@ subdir('build-utils-meson/threads') boost = dependency( 'boost', modules : ['container', 'context'], + include_type: 'system', ) # boost is a public dependency, but not a pkg-config dependency unfortunately, so we # put in `deps_other`. @@ -55,7 +56,12 @@ if bdw_gc.found() endif configdata.set('HAVE_BOEHMGC', bdw_gc.found().to_int()) -toml11 = dependency('toml11', version : '>=3.7.0', method : 'cmake') +toml11 = dependency( + 'toml11', + version : '>=3.7.0', + method : 'cmake', + include_type: 'system', +) deps_other += toml11 config_h = configure_file( diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 93c8bdef6..dbc74faf9 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -25,7 +25,7 @@ std::ostream & operator <<(std::ostream & str, const SymbolStr & symbol) void Expr::show(const SymbolTable & symbols, std::ostream & str) const { - abort(); + unreachable(); } void ExprInt::show(const SymbolTable & symbols, std::ostream & str) const @@ -82,7 +82,9 @@ void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) co return sa < sb; }); std::vector inherits; - std::map> inheritsFrom; + // We can use the displacement as a proxy for the order in which the symbols were parsed. + // The assignment of displacements should be deterministic, so that showBindings is deterministic. + std::map> inheritsFrom; for (auto & i : sorted) { switch (i->second.kind) { case AttrDef::Kind::Plain: @@ -93,7 +95,7 @@ void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) co case AttrDef::Kind::InheritedFrom: { auto & select = dynamic_cast(*i->second.e); auto & from = dynamic_cast(*select.e); - inheritsFrom[&from].push_back(i->first); + inheritsFrom[from.displ].push_back(i->first); break; } } @@ -105,7 +107,7 @@ void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) co } for (const auto & [from, syms] : inheritsFrom) { str << "inherit ("; - (*inheritFromExprs)[from->displ]->show(symbols, str); + (*inheritFromExprs)[from]->show(symbols, str); str << ")"; for (auto sym : syms) str << " " << symbols[sym]; str << "; "; @@ -269,7 +271,7 @@ std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath) void Expr::bindVars(EvalState & es, const std::shared_ptr & env) { - abort(); + unreachable(); } void ExprInt::bindVars(EvalState & es, const std::shared_ptr & env) diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 1bcc962c5..3279e3d48 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -186,7 +186,7 @@ struct ExprInheritFrom : ExprVar this->fromWith = nullptr; } - void bindVars(EvalState & es, const std::shared_ptr & env); + void bindVars(EvalState & es, const std::shared_ptr & env) override; }; struct ExprSelect : Expr @@ -203,7 +203,7 @@ struct ExprSelect : Expr * * @param[out] v The attribute set that should contain the last attribute name (if it exists). * @return The last attribute name in `attrPath` - * + * * @note This does *not* evaluate the final attribute, and does not fail if that's the only attribute that does not exist. */ Symbol evalExceptFinalSelect(EvalState & state, Env & env, Value & attrs); diff --git a/src/libexpr/package.nix b/src/libexpr/package.nix index 704456c96..4d10079ff 100644 --- a/src/libexpr/package.nix +++ b/src/libexpr/package.nix @@ -102,12 +102,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libexpr/parser-state.hh b/src/libexpr/parser-state.hh index 983a17a2e..4bb5c9204 100644 --- a/src/libexpr/parser-state.hh +++ b/src/libexpr/parser-state.hh @@ -64,7 +64,7 @@ struct LexerState /** * @brief Maps some positions to a DocComment, where the comment is relevant to the location. */ - std::map & positionToDocComment; + std::unordered_map & positionToDocComment; PosTable & positions; PosTable::Origin origin; @@ -291,12 +291,23 @@ inline Expr * ParserState::stripIndentation(const PosIdx pos, s2 = std::string(s2, 0, p + 1); } - es2->emplace_back(i->first, new ExprString(std::move(s2))); + // Ignore empty strings for a minor optimisation and AST simplification + if (s2 != "") { + es2->emplace_back(i->first, new ExprString(std::move(s2))); + } }; for (; i != es.end(); ++i, --n) { std::visit(overloaded { trimExpr, trimString }, i->second); } + // If there is nothing at all, return the empty string directly. + // This also ensures that equivalent empty strings result in the same ast, which is helpful when testing formatters. + if (es2->size() == 0) { + auto *const result = new ExprString(""); + delete es2; + return result; + } + /* If this is a single string, then don't do a concatenation. */ if (es2->size() == 1 && dynamic_cast((*es2)[0].second)) { auto *const result = (*es2)[0].second; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 452d265bc..9ad41c148 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -48,7 +48,7 @@ namespace nix { -typedef std::map DocCommentMap; +typedef std::unordered_map DocCommentMap; Expr * parseExprFromBuf( char * text, @@ -99,6 +99,14 @@ static void setDocPosition(const LexerState & lexerState, ExprLambda * lambda, P } } +static Expr * makeCall(PosIdx pos, Expr * fn, Expr * arg) { + if (auto e2 = dynamic_cast(fn)) { + e2->args.push_back(arg); + return fn; + } + return new ExprCall(pos, fn, {arg}); +} + %} @@ -123,6 +131,7 @@ static void setDocPosition(const LexerState & lexerState, ExprLambda * lambda, P %type start expr expr_function expr_if expr_op %type expr_select expr_simple expr_app +%type expr_pipe_from expr_pipe_into %type expr_list %type binds %type formals @@ -140,6 +149,7 @@ static void setDocPosition(const LexerState & lexerState, ExprLambda * lambda, P %token PATH HPATH SPATH PATH_END %token URI %token IF THEN ELSE ASSERT WITH LET IN_KW REC INHERIT EQ NEQ AND OR IMPL OR_KW +%token PIPE_FROM PIPE_INTO /* <| and |> */ %token DOLLAR_CURLY /* == ${ */ %token IND_STRING_OPEN IND_STRING_CLOSE %token ELLIPSIS @@ -206,9 +216,21 @@ expr_function expr_if : IF expr THEN expr ELSE expr { $$ = new ExprIf(CUR_POS, $2, $4, $6); } + | expr_pipe_from + | expr_pipe_into | expr_op ; +expr_pipe_from + : expr_op PIPE_FROM expr_pipe_from { $$ = makeCall(state->at(@2), $1, $3); } + | expr_op PIPE_FROM expr_op { $$ = makeCall(state->at(@2), $1, $3); } + ; + +expr_pipe_into + : expr_pipe_into PIPE_INTO expr_op { $$ = makeCall(state->at(@2), $3, $1); } + | expr_op PIPE_INTO expr_op { $$ = makeCall(state->at(@2), $3, $1); } + ; + expr_op : '!' expr_op %prec NOT { $$ = new ExprOpNot($2); } | '-' expr_op %prec NEGATE { $$ = new ExprCall(CUR_POS, new ExprVar(state->s.sub), {new ExprInt(0), $2}); } @@ -233,13 +255,7 @@ expr_op ; expr_app - : expr_app expr_select { - if (auto e2 = dynamic_cast($1)) { - e2->args.push_back($2); - $$ = $1; - } else - $$ = new ExprCall(CUR_POS, $1, {$2}); - } + : expr_app expr_select { $$ = makeCall(CUR_POS, $1, $2); } | expr_select ; diff --git a/src/libexpr/pos-idx.hh b/src/libexpr/pos-idx.hh index e13491560..2faa6b7fe 100644 --- a/src/libexpr/pos-idx.hh +++ b/src/libexpr/pos-idx.hh @@ -1,6 +1,7 @@ #pragma once #include +#include namespace nix { @@ -8,6 +9,7 @@ class PosIdx { friend struct LazyPosAcessors; friend class PosTable; + friend class std::hash; private: uint32_t id; @@ -37,8 +39,26 @@ public: { return id == other.id; } + + size_t hash() const noexcept + { + return std::hash{}(id); + } }; inline PosIdx noPos = {}; } + +namespace std { + +template<> +struct hash +{ + std::size_t operator()(nix::PosIdx pos) const noexcept + { + return pos.hash(); + } +}; + +} // namespace std diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 127823d49..7ceb84f0e 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -426,7 +426,7 @@ static void prim_typeOf(EvalState & state, const PosIdx pos, Value * * args, Val t = args[0]->external()->typeOf(); break; case nFloat: t = "float"; break; - case nThunk: abort(); + case nThunk: unreachable(); } v.mkString(t); } @@ -719,7 +719,7 @@ static RegisterPrimOp primop_genericClosure(PrimOp { .doc = R"( `builtins.genericClosure` iteratively computes the transitive closure over an arbitrary relation defined by a function. - It takes *attrset* with two attributes named `startSet` and `operator`, and returns a list of attrbute sets: + It takes *attrset* with two attributes named `startSet` and `operator`, and returns a list of attribute sets: - `startSet`: The initial list of attribute sets. @@ -1843,34 +1843,6 @@ static RegisterPrimOp primop_findFile(PrimOp { .doc = R"( Find *lookup-path* in *search-path*. - A search path is represented list of [attribute sets](./types.md#attribute-set) with two attributes: - - `prefix` is a relative path. - - `path` denotes a file system location - The exact syntax depends on the command line interface. - - Examples of search path attribute sets: - - - ``` - { - prefix = "nixos-config"; - path = "/etc/nixos/configuration.nix"; - } - ``` - - - ``` - { - prefix = ""; - path = "/nix/var/nix/profiles/per-user/root/channels"; - } - ``` - - The lookup algorithm checks each entry until a match is found, returning a [path value](@docroot@/language/types.md#type-path) of the match: - - - If *lookup-path* matches `prefix`, then the remainder of *lookup-path* (the "suffix") is searched for within the directory denoted by `path`. - Note that the `path` may need to be downloaded at this point to look inside. - - If the suffix is found inside that directory, then the entry is a match. - The combined absolute path of the directory (now downloaded if need be) and the suffix is returned. - [Lookup path](@docroot@/language/constructs/lookup-path.md) expressions are [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and [`builtins.nixPath`](#builtins-nixPath): ```nix @@ -1882,6 +1854,119 @@ static RegisterPrimOp primop_findFile(PrimOp { ```nix builtins.findFile builtins.nixPath "nixpkgs" ``` + + A search path is represented as a list of [attribute sets](./types.md#attribute-set) with two attributes: + - `prefix` is a relative path. + - `path` denotes a file system location + + Examples of search path attribute sets: + + - ``` + { + prefix = ""; + path = "/nix/var/nix/profiles/per-user/root/channels"; + } + ``` + - ``` + { + prefix = "nixos-config"; + path = "/etc/nixos/configuration.nix"; + } + ``` + - ``` + { + prefix = "nixpkgs"; + path = "https://github.com/NixOS/nixpkgs/tarballs/master"; + } + ``` + - ``` + { + prefix = "nixpkgs"; + path = "channel:nixpkgs-unstable"; + } + ``` + - ``` + { + prefix = "flake-compat"; + path = "flake:github:edolstra/flake-compat"; + } + ``` + + The lookup algorithm checks each entry until a match is found, returning a [path value](@docroot@/language/types.md#type-path) of the match: + + - If a prefix of `lookup-path` matches `prefix`, then the remainder of *lookup-path* (the "suffix") is searched for within the directory denoted by `path`. + The contents of `path` may need to be downloaded at this point to look inside. + + - If the suffix is found inside that directory, then the entry is a match. + The combined absolute path of the directory (now downloaded if need be) and the suffix is returned. + + > **Example** + > + > A *search-path* value + > + > ``` + > [ + > { + > prefix = ""; + > path = "/home/eelco/Dev"; + > } + > { + > prefix = "nixos-config"; + > path = "/etc/nixos"; + > } + > ] + > ``` + > + > and a *lookup-path* value `"nixos-config"` will cause Nix to try `/home/eelco/Dev/nixos-config` and `/etc/nixos` in that order and return the first path that exists. + + If `path` starts with `http://` or `https://`, it is interpreted as the URL of a tarball that will be downloaded and unpacked to a temporary location. + The tarball must consist of a single top-level directory. + + The URLs of the tarballs from the official `nixos.org` channels can be abbreviated as `channel:`. + See [documentation on `nix-channel`](@docroot@/command-ref/nix-channel.md) for details about channels. + + > **Example** + > + > These two search path entries are equivalent: + > + > - ``` + > { + > prefix = "nixpkgs"; + > path = "channel:nixpkgs-unstable"; + > } + > ``` + > - ``` + > { + > prefix = "nixpkgs"; + > path = "https://nixos.org/channels/nixos-unstable/nixexprs.tar.xz"; + > } + > ``` + + Search paths can also point to source trees using [flake URLs](@docroot@/command-ref/new-cli/nix3-flake.md#url-like-syntax). + + + > **Example** + > + > The search path entry + > + > ``` + > { + > prefix = "nixpkgs"; + > path = "flake:nixpkgs"; + > } + > ``` + > specifies that the prefix `nixpkgs` shall refer to the source tree downloaded from the `nixpkgs` entry in the flake registry. + > + > Similarly + > + > ``` + > { + > prefix = "nixpkgs"; + > path = "flake:github:nixos/nixpkgs/nixos-22.05"; + > } + > ``` + > + > makes `` refer to a particular branch of the `NixOS/nixpkgs` repository on GitHub. )", .fun = prim_findFile, }); @@ -4731,6 +4816,13 @@ void EvalState::createBaseEnv() .doc = R"( The value of the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path): a list of search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). + > **Example** + > + > ```bash + > $ NIX_PATH= nix-instantiate --eval --expr "builtins.nixPath" -I foo=bar --no-pure-eval + > [ { path = "bar"; prefix = "foo"; } ] + > ``` + Lookup path expressions are [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and [`builtins.findFile`](./builtins.html#builtins-findFile): diff --git a/src/libexpr/primops/derivation.nix b/src/libexpr/primops/derivation.nix index c0fbe8082..f329ff71e 100644 --- a/src/libexpr/primops/derivation.nix +++ b/src/libexpr/primops/derivation.nix @@ -1,6 +1,31 @@ -/* This is the implementation of the ‘derivation’ builtin function. - It's actually a wrapper around the ‘derivationStrict’ primop. */ +# This is the implementation of the ‘derivation’ builtin function. +# It's actually a wrapper around the ‘derivationStrict’ primop. +# Note that the following comment will be shown in :doc in the repl, but not in the manual. +/** + Create a derivation. + + # Inputs + + The single argument is an attribute set that describes what to build and how to build it. + See https://nix.dev/manual/nix/2.23/language/derivations + + # Output + + The result is an attribute set that describes the derivation. + Notably it contains the outputs, which in the context of the Nix language are special strings that refer to the output paths, which may not yet exist. + The realisation of these outputs only occurs when needed; for example + + * When `nix-build` or a similar command is run, it realises the outputs that were requested on its command line. + See https://nix.dev/manual/nix/2.23/command-ref/nix-build + + * When `import`, `readFile`, `readDir` or some other functions are called, they have to realise the outputs they depend on. + This is referred to as "import from derivation". + See https://nix.dev/manual/nix/2.23/language/import-from-derivation + + Note that `derivation` is very bare-bones, and provides almost no commands during the build. + Most likely, you'll want to use functions like `stdenv.mkDerivation` in Nixpkgs to set up a basic environment. +*/ drvAttrs @ { outputs ? [ "out" ], ... }: let diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index da6af394f..9b6cd2302 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -523,9 +523,19 @@ static void prim_fetchurl(EvalState & state, const PosIdx pos, Value * * args, V static RegisterPrimOp primop_fetchurl({ .name = "__fetchurl", - .args = {"url"}, + .args = {"arg"}, .doc = R"( Download the specified URL and return the path of the downloaded file. + `arg` can be either a string denoting the URL, or an attribute set with the following attributes: + + - `url` + + The URL of the file to download. + + - `name` (default: the last path component of the URL) + + A name for the file in the store. This can be useful if the URL has any + characters that are invalid for the store. Not available in [restricted evaluation mode](@docroot@/command-ref/conf-file.md#conf-restrict-eval). )", @@ -543,11 +553,11 @@ static RegisterPrimOp primop_fetchTarball({ .doc = R"( Download the specified URL, unpack it and return the path of the unpacked tree. The file must be a tape archive (`.tar`) compressed - with `gzip`, `bzip2` or `xz`. The top-level path component of the - files in the tarball is removed, so it is best if the tarball - contains a single directory at top level. The typical use of the - function is to obtain external Nix expression dependencies, such as - a particular version of Nixpkgs, e.g. + with `gzip`, `bzip2` or `xz`. If the tarball consists of a + single directory, then the top-level path component of the files + in the tarball is removed. The typical use of the function is to + obtain external Nix expression dependencies, such as a + particular version of Nixpkgs, e.g. ```nix with import (fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz) {}; @@ -654,12 +664,12 @@ static RegisterPrimOp primop_fetchGit({ Whether to check `rev` for a signature matching `publicKey` or `publicKeys`. If `verifyCommit` is enabled, then `fetchGit` cannot use a local repository with uncommitted changes. - Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + Requires the [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches). - `publicKey` The public key against which `rev` is verified if `verifyCommit` is enabled. - Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + Requires the [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches). - `keytype` (default: `"ssh-ed25519"`) @@ -671,7 +681,7 @@ static RegisterPrimOp primop_fetchGit({ - `"ssh-ed25519"` - `"ssh-ed25519-sk"` - `"ssh-rsa"` - Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + Requires the [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches). - `publicKeys` @@ -685,7 +695,7 @@ static RegisterPrimOp primop_fetchGit({ } ``` - Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + Requires the [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches). Here are some examples of how to use `fetchGit`. diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index 6c7d303e8..b4f1df7a8 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -2,6 +2,7 @@ #include "eval-inline.hh" #include + #include namespace nix { diff --git a/src/libexpr/print-ambiguous.cc b/src/libexpr/print-ambiguous.cc index 5d55b45da..a40c98643 100644 --- a/src/libexpr/print-ambiguous.cc +++ b/src/libexpr/print-ambiguous.cc @@ -94,7 +94,7 @@ void printAmbiguous( break; default: printError("Nix evaluator internal error: printAmbiguous: invalid value type"); - abort(); + unreachable(); } } diff --git a/src/libexpr/print.cc b/src/libexpr/print.cc index 2f377e588..4d1a6868c 100644 --- a/src/libexpr/print.cc +++ b/src/libexpr/print.cc @@ -299,6 +299,9 @@ private: output << ANSI_NORMAL; } + /** + * @note This may force items. + */ bool shouldPrettyPrintAttrs(AttrVec & v) { if (!options.shouldPrettyPrint() || v.empty()) { @@ -315,6 +318,9 @@ private: return true; } + // It is ok to force the item(s) here, because they will be printed anyway. + state.forceValue(*item, item->determinePos(noPos)); + // Pretty-print single-item attrsets only if they contain nested // structures. auto itemType = item->type(); @@ -371,6 +377,9 @@ private: } } + /** + * @note This may force items. + */ bool shouldPrettyPrintList(std::span list) { if (!options.shouldPrettyPrint() || list.empty()) { @@ -387,6 +396,9 @@ private: return true; } + // It is ok to force the item(s) here, because they will be printed anyway. + state.forceValue(*item, item->determinePos(noPos)); + // Pretty-print single-item lists only if they contain nested // structures. auto itemType = item->type(); @@ -463,7 +475,7 @@ private: else output << "primop"; } else { - abort(); + unreachable(); } output << "»"; @@ -492,7 +504,7 @@ private: if (options.ansiColors) output << ANSI_NORMAL; } else { - abort(); + unreachable(); } } diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh index c7a3563b0..be12f6248 100644 --- a/src/libexpr/symbol-table.hh +++ b/src/libexpr/symbol-table.hh @@ -7,6 +7,7 @@ #include "types.hh" #include "chunked-vector.hh" +#include "error.hh" namespace nix { @@ -69,6 +70,8 @@ public: auto operator<=>(const Symbol other) const { return id <=> other.id; } bool operator==(const Symbol other) const { return id == other.id; } + + friend class std::hash; }; /** @@ -113,7 +116,7 @@ public: SymbolStr operator[](Symbol s) const { if (s.id == 0 || s.id > store.size()) - abort(); + unreachable(); return SymbolStr(store[s.id - 1]); } @@ -132,3 +135,12 @@ public: }; } + +template<> +struct std::hash +{ + std::size_t operator()(const nix::Symbol & s) const noexcept + { + return std::hash{}(s.id); + } +}; diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 1f4d72d39..257da1d2d 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -285,7 +285,7 @@ public: if (invalidIsThunk) return nThunk; else - abort(); + unreachable(); } inline void finishValue(InternalType newType, Payload newPayload) @@ -494,11 +494,11 @@ void Value::mkBlackhole() #if HAVE_BOEHMGC typedef std::vector> ValueVector; -typedef std::map, traceable_allocator>> ValueMap; +typedef std::unordered_map, std::equal_to, traceable_allocator>> ValueMap; typedef std::map, traceable_allocator>> ValueVectorMap; #else typedef std::vector ValueVector; -typedef std::map ValueMap; +typedef std::unordered_map ValueMap; typedef std::map ValueVectorMap; #endif diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc index b788c5948..25d04cdc9 100644 --- a/src/libfetchers/attrs.cc +++ b/src/libfetchers/attrs.cc @@ -33,7 +33,7 @@ nlohmann::json attrsToJSON(const Attrs & attrs) json[attr.first] = *v; } else if (auto v = std::get_if>(&attr.second)) { json[attr.first] = v->t; - } else abort(); + } else unreachable(); } return json; } @@ -99,7 +99,7 @@ std::map attrsToQuery(const Attrs & attrs) query.insert_or_assign(attr.first, *v); } else if (auto v = std::get_if>(&attr.second)) { query.insert_or_assign(attr.first, v->t ? "1" : "0"); - } else abort(); + } else unreachable(); } return query; } diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index ecc71ae47..114aa4ec0 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -126,16 +126,39 @@ Object lookupObject(git_repository * repo, const git_oid & oid, git_object_t typ } template -T peelObject(git_repository * repo, git_object * obj, git_object_t type) +T peelObject(git_object * obj, git_object_t type) { T obj2; if (git_object_peel((git_object * *) (typename T::pointer *) Setter(obj2), obj, type)) { auto err = git_error_last(); - throw Error("peeling Git object '%s': %s", git_object_id(obj), err->message); + throw Error("peeling Git object '%s': %s", *git_object_id(obj), err->message); } return obj2; } +template +T dupObject(typename T::pointer obj) +{ + T obj2; + if (git_object_dup((git_object * *) (typename T::pointer *) Setter(obj2), (git_object *) obj)) + throw Error("duplicating object '%s': %s", *git_object_id((git_object *) obj), git_error_last()->message); + return obj2; +} + +/** + * Peel the specified object (i.e. follow tag and commit objects) to + * either a blob or a tree. + */ +static Object peelToTreeOrBlob(git_object * obj) +{ + /* git_object_peel() doesn't handle blob objects, so handle those + specially. */ + if (git_object_type(obj) == GIT_OBJECT_BLOB) + return dupObject(obj); + else + return peelObject(obj, GIT_OBJECT_TREE); +} + struct GitRepoImpl : GitRepo, std::enable_shared_from_this { /** Location of the repository on disk. */ @@ -166,7 +189,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this std::unordered_set done; std::queue todo; - todo.push(peelObject(*this, lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT)); + todo.push(peelObject(lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT)); while (auto commit = pop(todo)) { if (!done.insert(*git_commit_id(commit->get())).second) continue; @@ -184,7 +207,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this uint64_t getLastModified(const Hash & rev) override { - auto commit = peelObject(*this, lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT); + auto commit = peelObject(lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT); return git_commit_time(commit.get()); } @@ -463,6 +486,23 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this return narHash; } + + Hash dereferenceSingletonDirectory(const Hash & oid_) override + { + auto oid = hashToOID(oid_); + + auto _tree = lookupObject(*this, oid, GIT_OBJECT_TREE); + auto tree = (const git_tree *) &*_tree; + + if (git_tree_entrycount(tree) == 1) { + auto entry = git_tree_entry_byindex(tree, 0); + auto mode = git_tree_entry_filemode(entry); + if (mode == GIT_FILEMODE_TREE) + oid = *git_tree_entry_id(entry); + } + + return toHash(oid); + } }; ref GitRepo::openRepo(const std::filesystem::path & path, bool create, bool bare) @@ -476,11 +516,11 @@ ref GitRepo::openRepo(const std::filesystem::path & path, bool create, struct GitSourceAccessor : SourceAccessor { ref repo; - Tree root; + Object root; GitSourceAccessor(ref repo_, const Hash & rev) : repo(repo_) - , root(peelObject(*repo, lookupObject(*repo, hashToOID(rev)).get(), GIT_OBJECT_TREE)) + , root(peelToTreeOrBlob(lookupObject(*repo, hashToOID(rev)).get())) { } @@ -506,7 +546,7 @@ struct GitSourceAccessor : SourceAccessor std::optional maybeLstat(const CanonPath & path) override { if (path.isRoot()) - return Stat { .type = tDirectory }; + return Stat { .type = git_object_type(root.get()) == GIT_OBJECT_TREE ? tDirectory : tRegular }; auto entry = lookup(path); if (!entry) @@ -616,10 +656,10 @@ struct GitSourceAccessor : SourceAccessor std::optional lookupTree(const CanonPath & path) { if (path.isRoot()) { - Tree tree; - if (git_tree_dup(Setter(tree), root.get())) - throw Error("duplicating directory '%s': %s", showPath(path), git_error_last()->message); - return tree; + if (git_object_type(root.get()) == GIT_OBJECT_TREE) + return dupObject((git_tree *) &*root); + else + return std::nullopt; } auto entry = lookup(path); @@ -646,10 +686,10 @@ struct GitSourceAccessor : SourceAccessor std::variant getTree(const CanonPath & path) { if (path.isRoot()) { - Tree tree; - if (git_tree_dup(Setter(tree), root.get())) - throw Error("duplicating directory '%s': %s", showPath(path), git_error_last()->message); - return tree; + if (git_object_type(root.get()) == GIT_OBJECT_TREE) + return dupObject((git_tree *) &*root); + else + throw Error("Git root object '%s' is not a directory", *git_object_id(root.get())); } auto entry = need(path); @@ -669,6 +709,9 @@ struct GitSourceAccessor : SourceAccessor Blob getBlob(const CanonPath & path, bool expectSymlink) { + if (!expectSymlink && git_object_type(root.get()) == GIT_OBJECT_BLOB) + return dupObject((git_blob *) &*root); + auto notExpected = [&]() { throw Error( @@ -782,8 +825,6 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink std::vector pendingDirs; - size_t componentsToStrip = 1; - void pushBuilder(std::string name) { git_treebuilder * b; @@ -839,9 +880,6 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink { std::span pathComponents2{pathComponents}; - if (pathComponents2.size() <= componentsToStrip) return false; - pathComponents2 = pathComponents2.subspan(componentsToStrip); - updateBuilders( isDir ? pathComponents2 @@ -964,7 +1002,8 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink git_tree_entry_filemode(entry)); } - Hash sync() override { + Hash sync() override + { updateBuilders({}); auto [oid, _name] = popBuilder(); diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh index 495916f62..915252868 100644 --- a/src/libfetchers/git-utils.hh +++ b/src/libfetchers/git-utils.hh @@ -98,6 +98,13 @@ struct GitRepo * serialisation. This is memoised on-disk. */ virtual Hash treeHashToNarHash(const Hash & treeHash) = 0; + + /** + * If the specified Git object is a directory with a single entry + * that is a directory, return the ID of that object. + * Otherwise, return the passed ID unchanged. + */ + virtual Hash dereferenceSingletonDirectory(const Hash & oid) = 0; }; ref getTarballCache(); diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 8cb7e8b8a..b40f08c76 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -254,12 +254,18 @@ struct GitArchiveInputScheme : InputScheme getFileTransfer()->download(std::move(req), sink); }); + auto act = std::make_unique(*logger, lvlInfo, actUnknown, + fmt("unpacking '%s' into the Git cache", input.to_string())); + TarArchive archive { *source }; - auto parseSink = getTarballCache()->getFileSystemObjectSink(); + auto tarballCache = getTarballCache(); + auto parseSink = tarballCache->getFileSystemObjectSink(); auto lastModified = unpackTarfileToSink(archive, *parseSink); + act.reset(); + TarballInfo tarballInfo { - .treeHash = parseSink->sync(), + .treeHash = tarballCache->dereferenceSingletonDirectory(parseSink->sync()), .lastModified = lastModified }; diff --git a/src/libfetchers/meson.build b/src/libfetchers/meson.build index c39fe99f3..d4f202796 100644 --- a/src/libfetchers/meson.build +++ b/src/libfetchers/meson.build @@ -26,6 +26,8 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') +subdir('build-utils-meson/threads') + nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') deps_public += nlohmann_json diff --git a/src/libfetchers/package.nix b/src/libfetchers/package.nix index b4abb144b..9b5d8bff7 100644 --- a/src/libfetchers/package.nix +++ b/src/libfetchers/package.nix @@ -67,14 +67,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - # TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated - # to work with `strictDeps`. - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 55db3eafb..457210542 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -143,6 +143,9 @@ DownloadTarballResult downloadTarball( // TODO: fall back to cached value if download fails. + auto act = std::make_unique(*logger, lvlInfo, actUnknown, + fmt("unpacking '%s' into the Git cache", url)); + AutoDelete cleanupTemp; /* Note: if the download is cached, `importTarball()` will receive @@ -164,9 +167,12 @@ DownloadTarballResult downloadTarball( TarArchive{path}; }) : TarArchive{*source}; - auto parseSink = getTarballCache()->getFileSystemObjectSink(); + auto tarballCache = getTarballCache(); + auto parseSink = tarballCache->getFileSystemObjectSink(); auto lastModified = unpackTarfileToSink(archive, *parseSink); + act.reset(); + auto res(_res->lock()); Attrs infoAttrs; @@ -177,7 +183,8 @@ DownloadTarballResult downloadTarball( infoAttrs = cached->value; } else { infoAttrs.insert_or_assign("etag", res->etag); - infoAttrs.insert_or_assign("treeHash", parseSink->sync().gitRev()); + infoAttrs.insert_or_assign("treeHash", + tarballCache->dereferenceSingletonDirectory(parseSink->sync()).gitRev()); infoAttrs.insert_or_assign("lastModified", uint64_t(lastModified)); if (res->immutableUrl) infoAttrs.insert_or_assign("immutableUrl", *res->immutableUrl); diff --git a/src/libflake/flake/nix-flake.pc.in b/src/libflake/flake/nix-flake.pc.in new file mode 100644 index 000000000..10c52f5e9 --- /dev/null +++ b/src/libflake/flake/nix-flake.pc.in @@ -0,0 +1,10 @@ +prefix=@prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: Nix +Description: Nix Package Manager +Version: @PACKAGE_VERSION@ +Requires: nix-util nix-store nix-expr +Libs: -L${libdir} -lnixflake +Cflags: -I${includedir}/nix -std=c++2a diff --git a/src/libflake/local.mk b/src/libflake/local.mk index 2cceda2bf..5e604ef3a 100644 --- a/src/libflake/local.mk +++ b/src/libflake/local.mk @@ -15,3 +15,8 @@ libflake_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetcher libflake_LDFLAGS += $(THREAD_LDFLAGS) libflake_LIBS = libutil libstore libfetchers libexpr + +$(eval $(call install-file-in, $(buildprefix)$(d)/flake/nix-flake.pc, $(libdir)/pkgconfig, 0644)) + +$(foreach i, $(wildcard src/libflake/flake/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644))) diff --git a/src/libflake/meson.build b/src/libflake/meson.build index 38d70c678..d2bb179df 100644 --- a/src/libflake/meson.build +++ b/src/libflake/meson.build @@ -26,6 +26,8 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') +subdir('build-utils-meson/threads') + nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') deps_public += nlohmann_json diff --git a/src/libflake/package.nix b/src/libflake/package.nix index af6f5da94..851adf07e 100644 --- a/src/libflake/package.nix +++ b/src/libflake/package.nix @@ -67,14 +67,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - # TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated - # to work with `strictDeps`. - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libmain-c/.version b/src/libmain-c/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libmain-c/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libmain-c/build-utils-meson b/src/libmain-c/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libmain-c/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libmain-c/meson.build b/src/libmain-c/meson.build new file mode 100644 index 000000000..345382712 --- /dev/null +++ b/src/libmain-c/meson.build @@ -0,0 +1,86 @@ +project('nix-main-c', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), + dependency('nix-main'), +] +deps_public_maybe_subproject = [ + dependency('nix-util-c'), + dependency('nix-store-c'), +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +# TODO rename, because it will conflict with downstream projects +configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) + +config_h = configure_file( + configuration : configdata, + output : 'config-main.h', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + + # From C++ libraries, only for internals + '-include', 'config-util.hh', + '-include', 'config-store.hh', + '-include', 'config-main.hh', + + # From C libraries, for our public, installed headers too + '-include', 'config-util.h', + '-include', 'config-store.h', + '-include', 'config-main.h', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'nix_api_main.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'nix_api_main.h', +) + +subdir('build-utils-meson/export-all-symbols') + +this_library = library( + 'nixmainc', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + link_args: linker_export_flags, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libmain-c/nix_api_main.cc b/src/libmain-c/nix_api_main.cc new file mode 100644 index 000000000..692d53f47 --- /dev/null +++ b/src/libmain-c/nix_api_main.cc @@ -0,0 +1,16 @@ +#include "nix_api_store.h" +#include "nix_api_store_internal.h" +#include "nix_api_util.h" +#include "nix_api_util_internal.h" + +#include "plugin.hh" + +nix_err nix_init_plugins(nix_c_context * context) +{ + if (context) + context->last_err_code = NIX_OK; + try { + nix::initPlugins(); + } + NIXC_CATCH_ERRS +} diff --git a/src/libmain-c/nix_api_main.h b/src/libmain-c/nix_api_main.h new file mode 100644 index 000000000..3957b992f --- /dev/null +++ b/src/libmain-c/nix_api_main.h @@ -0,0 +1,40 @@ +#ifndef NIX_API_MAIN_H +#define NIX_API_MAIN_H +/** + * @defgroup libmain libmain + * @brief C bindings for nix libmain + * + * libmain has misc utilities for CLI commands + * @{ + */ +/** @file + * @brief Main entry for the libmain C bindings + */ + +#include "nix_api_util.h" +#include + +#ifdef __cplusplus +extern "C" { +#endif +// cffi start + +/** + * @brief Loads the plugins specified in Nix's plugin-files setting. + * + * Call this once, after calling your desired init functions and setting + * relevant settings. + * + * @param[out] context Optional, stores error information + * @return NIX_OK if the initialization was successful, an error code otherwise. + */ +nix_err nix_init_plugins(nix_c_context * context); + +// cffi end +#ifdef __cplusplus +} +#endif +/** + * @} + */ +#endif // NIX_API_MAIN_H diff --git a/src/libmain-c/package.nix b/src/libmain-c/package.nix new file mode 100644 index 000000000..ce6f67300 --- /dev/null +++ b/src/libmain-c/package.nix @@ -0,0 +1,79 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-util-c +, nix-store +, nix-store-c +, nix-main + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-main-c"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + (fileset.fileFilter (file: file.hasExt "h") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + propagatedBuildInputs = [ + nix-util-c + nix-store + nix-store-c + nix-main + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index a94845ab8..768b2177c 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -5,6 +5,7 @@ #include "logging.hh" #include "loggers.hh" #include "util.hh" +#include "plugin.hh" namespace nix { diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc index 9829859de..a4e0530c8 100644 --- a/src/libmain/loggers.cc +++ b/src/libmain/loggers.cc @@ -36,7 +36,7 @@ Logger * makeDefaultLogger() { return logger; } default: - abort(); + unreachable(); } } diff --git a/src/libmain/meson.build b/src/libmain/meson.build index 859ce22f8..7fcadf06d 100644 --- a/src/libmain/meson.build +++ b/src/libmain/meson.build @@ -26,6 +26,7 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') +subdir('build-utils-meson/threads') pubsetbuf_test = ''' #include @@ -64,6 +65,7 @@ subdir('build-utils-meson/diagnostics') sources = files( 'common-args.cc', 'loggers.cc', + 'plugin.cc', 'progress-bar.cc', 'shared.cc', ) @@ -79,6 +81,7 @@ include_dirs = [include_directories('.')] headers = [config_h] + files( 'common-args.hh', 'loggers.hh', + 'plugin.hh', 'progress-bar.hh', 'shared.hh', ) diff --git a/src/libmain/package.nix b/src/libmain/package.nix index bbd97ec3e..47513dbdc 100644 --- a/src/libmain/package.nix +++ b/src/libmain/package.nix @@ -62,14 +62,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - # TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated - # to work with `strictDeps`. - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libmain/plugin.cc b/src/libmain/plugin.cc new file mode 100644 index 000000000..ccfd7f900 --- /dev/null +++ b/src/libmain/plugin.cc @@ -0,0 +1,119 @@ +#ifndef _WIN32 +# include +#endif + +#include + +#include "config-global.hh" +#include "signals.hh" + +namespace nix { + +struct PluginFilesSetting : public BaseSetting +{ + bool pluginsLoaded = false; + + PluginFilesSetting( + Config * options, + const Paths & def, + const std::string & name, + const std::string & description, + const std::set & aliases = {}) + : BaseSetting(def, true, name, description, aliases) + { + options->addSetting(this); + } + + Paths parse(const std::string & str) const override; +}; + +Paths PluginFilesSetting::parse(const std::string & str) const +{ + if (pluginsLoaded) + throw UsageError( + "plugin-files set after plugins were loaded, you may need to move the flag before the subcommand"); + return BaseSetting::parse(str); +} + +struct PluginSettings : Config +{ + PluginFilesSetting pluginFiles{ + this, + {}, + "plugin-files", + R"( + A list of plugin files to be loaded by Nix. Each of these files will + be dlopened by Nix. If they contain the symbol `nix_plugin_entry()`, + this symbol will be called. Alternatively, they can affect execution + through static initialization. In particular, these plugins may construct + static instances of RegisterPrimOp to add new primops or constants to the + expression language, RegisterStoreImplementation to add new store + implementations, RegisterCommand to add new subcommands to the `nix` + command, and RegisterSetting to add new nix config settings. See the + constructors for those types for more details. + + Warning! These APIs are inherently unstable and may change from + release to release. + + Since these files are loaded into the same address space as Nix + itself, they must be DSOs compatible with the instance of Nix + running at the time (i.e. compiled against the same headers, not + linked to any incompatible libraries). They should not be linked to + any Nix libs directly, as those will be available already at load + time. + + If an entry in the list is a directory, all files in the directory + are loaded as plugins (non-recursively). + )"}; +}; + +static PluginSettings pluginSettings; + +static GlobalConfig::Register rPluginSettings(&pluginSettings); + +void initPlugins() +{ + assert(!pluginSettings.pluginFiles.pluginsLoaded); + for (const auto & pluginFile : pluginSettings.pluginFiles.get()) { + std::vector pluginFiles; + try { + auto ents = std::filesystem::directory_iterator{pluginFile}; + for (const auto & ent : ents) { + checkInterrupt(); + pluginFiles.emplace_back(ent.path()); + } + } catch (std::filesystem::filesystem_error & e) { + if (e.code() != std::errc::not_a_directory) + throw; + pluginFiles.emplace_back(pluginFile); + } + for (const auto & file : pluginFiles) { + checkInterrupt(); + /* handle is purposefully leaked as there may be state in the + DSO needed by the action of the plugin. */ +#ifndef _WIN32 // TODO implement via DLL loading on Windows + void * handle = dlopen(file.c_str(), RTLD_LAZY | RTLD_LOCAL); + if (!handle) + throw Error("could not dynamically open plugin file '%s': %s", file, dlerror()); + + /* Older plugins use a statically initialized object to run their code. + Newer plugins can also export nix_plugin_entry() */ + void (*nix_plugin_entry)() = (void (*)()) dlsym(handle, "nix_plugin_entry"); + if (nix_plugin_entry) + nix_plugin_entry(); +#else + throw Error("could not dynamically open plugin file '%s'", file); +#endif + } + } + + /* Since plugins can add settings, try to re-apply previously + unknown settings. */ + globalConfig.reapplyUnknownSettings(); + globalConfig.warnUnknownSettings(); + + /* Tell the user if they try to set plugin-files after we've already loaded */ + pluginSettings.pluginFiles.pluginsLoaded = true; +} + +} diff --git a/src/libmain/plugin.hh b/src/libmain/plugin.hh new file mode 100644 index 000000000..4221c1b17 --- /dev/null +++ b/src/libmain/plugin.hh @@ -0,0 +1,12 @@ +#pragma once +///@file + +namespace nix { + +/** + * This should be called after settings are initialized, but before + * anything else + */ +void initPlugins(); + +} diff --git a/src/libstore-c/meson.build b/src/libstore-c/meson.build index 917de4cda..4bfd944c6 100644 --- a/src/libstore-c/meson.build +++ b/src/libstore-c/meson.build @@ -27,6 +27,8 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') +subdir('build-utils-meson/threads') + # TODO rename, because it will conflict with downstream projects configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) diff --git a/src/libstore-c/nix_api_store.cc b/src/libstore-c/nix_api_store.cc index 4fe25c7d4..79841ca49 100644 --- a/src/libstore-c/nix_api_store.cc +++ b/src/libstore-c/nix_api_store.cc @@ -29,16 +29,6 @@ nix_err nix_libstore_init_no_load_config(nix_c_context * context) NIXC_CATCH_ERRS } -nix_err nix_init_plugins(nix_c_context * context) -{ - if (context) - context->last_err_code = NIX_OK; - try { - nix::initPlugins(); - } - NIXC_CATCH_ERRS -} - Store * nix_store_open(nix_c_context * context, const char * uri, const char *** params) { if (context) diff --git a/src/libstore-c/nix_api_store.h b/src/libstore-c/nix_api_store.h index d3cb8fab8..4b2134457 100644 --- a/src/libstore-c/nix_api_store.h +++ b/src/libstore-c/nix_api_store.h @@ -42,17 +42,6 @@ nix_err nix_libstore_init(nix_c_context * context); */ nix_err nix_libstore_init_no_load_config(nix_c_context * context); -/** - * @brief Loads the plugins specified in Nix's plugin-files setting. - * - * Call this once, after calling your desired init functions and setting - * relevant settings. - * - * @param[out] context Optional, stores error information - * @return NIX_OK if the initialization was successful, an error code otherwise. - */ -nix_err nix_init_plugins(nix_c_context * context); - /** * @brief Open a nix store. * diff --git a/src/libstore-c/package.nix b/src/libstore-c/package.nix index fc34c1bda..e4f372236 100644 --- a/src/libstore-c/package.nix +++ b/src/libstore-c/package.nix @@ -64,12 +64,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index ec1b99411..a1f88b79e 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1567,7 +1567,7 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result) { Goal::waiteeDone(waitee, result); - if (!useDerivation) return; + if (!useDerivation || !drv) return; auto & fullDrv = *dynamic_cast(drv.get()); auto * dg = dynamic_cast(&*waitee); diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index 02284d93c..dedcad2b1 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -62,7 +62,7 @@ Goal::Co DrvOutputSubstitutionGoal::init() #ifndef _WIN32 outPipe->readSide.get() #else - &outPipe + &*outPipe #endif }, true, false); diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh index 807054926..8c60d0198 100644 --- a/src/libstore/build/drv-output-substitution-goal.hh +++ b/src/libstore/build/drv-output-substitution-goal.hh @@ -36,7 +36,7 @@ public: Co init() override; Co realisationFetched(std::shared_ptr outputInfo, nix::ref sub); - void timedOut(Error && ex) override { abort(); }; + void timedOut(Error && ex) override { unreachable(); }; std::string key() override; diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 162c392d0..9c6a40c84 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -400,12 +400,12 @@ public: virtual void handleChildOutput(Descriptor fd, std::string_view data) { - abort(); + unreachable(); } virtual void handleEOF(Descriptor fd) { - abort(); + unreachable(); } void trace(std::string_view s); diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 7deeb4748..0152f1808 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -145,8 +145,10 @@ Goal::Co PathSubstitutionGoal::init() /* None left. Terminate this goal and let someone else deal with it. */ - worker.failedSubstitutions++; - worker.updateProgress(); + if (substituterFailed) { + worker.failedSubstitutions++; + worker.updateProgress(); + } /* Hack: don't indicate failure if there were no substituters. In that case the calling derivation should just do a @@ -158,7 +160,7 @@ Goal::Co PathSubstitutionGoal::init() } -Goal::Co PathSubstitutionGoal::tryToRun(StorePath subPath, nix::ref sub, std::shared_ptr info, bool& substituterFailed) +Goal::Co PathSubstitutionGoal::tryToRun(StorePath subPath, nix::ref sub, std::shared_ptr info, bool & substituterFailed) { trace("all references realised"); diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index 86e4f5423..f2cf797e5 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -50,7 +50,7 @@ public: PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); ~PathSubstitutionGoal(); - void timedOut(Error && ex) override { abort(); }; + void timedOut(Error && ex) override { unreachable(); }; /** * We prepend "a$" to the key name to ensure substitution goals @@ -66,7 +66,7 @@ public: */ Co init() override; Co gotInfo(); - Co tryToRun(StorePath subPath, nix::ref sub, std::shared_ptr info, bool& substituterFailed); + Co tryToRun(StorePath subPath, nix::ref sub, std::shared_ptr info, bool & substituterFailed); Co finished(); /** diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 7fc41b121..ab0ba67b5 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -216,7 +216,7 @@ void Worker::childStarted(GoalPtr goal, const std::set(value)) != experimentalFeatureSettings.experimentalFeatures.get()) debug("Ignoring the client-specified experimental features"); - } else if (name == settings.pluginFiles.name) { - if (tokenizeString(value) != settings.pluginFiles.get()) - warn("Ignoring the client-specified plugin-files.\n" - "The client specifying plugins to the daemon never made sense, and was removed in Nix >=2.14."); + } else if (name == "plugin-files") { + warn("Ignoring the client-specified plugin-files.\n" + "The client specifying plugins to the daemon never made sense, and was removed in Nix >=2.14."); } else if (trusted || name == settings.buildTimeout.name @@ -270,26 +269,21 @@ struct ClientSettings }; static void performOp(TunnelLogger * logger, ref store, - TrustedFlag trusted, RecursiveFlag recursive, WorkerProto::Version clientVersion, - Source & from, BufferedSink & to, WorkerProto::Op op) + TrustedFlag trusted, RecursiveFlag recursive, + WorkerProto::BasicServerConnection & conn, + WorkerProto::Op op) { - WorkerProto::ReadConn rconn { - .from = from, - .version = clientVersion, - }; - WorkerProto::WriteConn wconn { - .to = to, - .version = clientVersion, - }; + WorkerProto::ReadConn rconn(conn); + WorkerProto::WriteConn wconn(conn); switch (op) { case WorkerProto::Op::IsValidPath: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); bool result = store->isValidPath(path); logger->stopWork(); - to << result; + conn.to << result; break; } @@ -297,8 +291,8 @@ static void performOp(TunnelLogger * logger, ref store, auto paths = WorkerProto::Serialise::read(*store, rconn); SubstituteFlag substitute = NoSubstitute; - if (GET_PROTOCOL_MINOR(clientVersion) >= 27) { - substitute = readInt(from) ? Substitute : NoSubstitute; + if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 27) { + substitute = readInt(conn.from) ? Substitute : NoSubstitute; } logger->startWork(); @@ -312,13 +306,13 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::HasSubstitutes: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); StorePathSet paths; // FIXME paths.insert(path); auto res = store->querySubstitutablePaths(paths); logger->stopWork(); - to << (res.count(path) != 0); + conn.to << (res.count(path) != 0); break; } @@ -332,11 +326,11 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::QueryPathHash: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); auto hash = store->queryPathInfo(path)->narHash; logger->stopWork(); - to << hash.to_string(HashFormat::Base16, false); + conn.to << hash.to_string(HashFormat::Base16, false); break; } @@ -344,7 +338,7 @@ static void performOp(TunnelLogger * logger, ref store, case WorkerProto::Op::QueryReferrers: case WorkerProto::Op::QueryValidDerivers: case WorkerProto::Op::QueryDerivationOutputs: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); StorePathSet paths; if (op == WorkerProto::Op::QueryReferences) @@ -361,16 +355,16 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::QueryDerivationOutputNames: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); auto names = store->readDerivation(path).outputNames(); logger->stopWork(); - to << names; + conn.to << names; break; } case WorkerProto::Op::QueryDerivationOutputMap: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); auto outputs = store->queryPartialDerivationOutputMap(path); logger->stopWork(); @@ -379,37 +373,37 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::QueryDeriver: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); auto info = store->queryPathInfo(path); logger->stopWork(); - to << (info->deriver ? store->printStorePath(*info->deriver) : ""); + conn.to << (info->deriver ? store->printStorePath(*info->deriver) : ""); break; } case WorkerProto::Op::QueryPathFromHashPart: { - auto hashPart = readString(from); + auto hashPart = readString(conn.from); logger->startWork(); auto path = store->queryPathFromHashPart(hashPart); logger->stopWork(); - to << (path ? store->printStorePath(*path) : ""); + conn.to << (path ? store->printStorePath(*path) : ""); break; } case WorkerProto::Op::AddToStore: { - if (GET_PROTOCOL_MINOR(clientVersion) >= 25) { - auto name = readString(from); - auto camStr = readString(from); + if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 25) { + auto name = readString(conn.from); + auto camStr = readString(conn.from); auto refs = WorkerProto::Serialise::read(*store, rconn); bool repairBool; - from >> repairBool; + conn.from >> repairBool; auto repair = RepairFlag{repairBool}; logger->startWork(); auto pathInfo = [&]() { // NB: FramedSource must be out of scope before logger->stopWork(); auto [contentAddressMethod, hashAlgo] = ContentAddressMethod::parseWithAlgo(camStr); - FramedSource source(from); + FramedSource source(conn.from); FileSerialisationMethod dumpMethod; switch (contentAddressMethod.getFileIngestionMethod()) { case FileIngestionMethod::Flat: @@ -440,7 +434,7 @@ static void performOp(TunnelLogger * logger, ref store, bool fixed; uint8_t recursive; std::string hashAlgoRaw; - from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw; + conn.from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw; if (recursive > true) throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive); method = recursive @@ -460,11 +454,11 @@ static void performOp(TunnelLogger * logger, ref store, so why all this extra work? We still parse the NAR so that we aren't sending arbitrary data to `saved` unwittingly`, and we know when the NAR ends so we don't - consume the rest of `from` and can't parse another + consume the rest of `conn.from` and can't parse another command. (We don't trust `addToStoreFromDump` to not eagerly consume the entire stream it's given, past the length of the Nar. */ - TeeSource savedNARSource(from, saved); + TeeSource savedNARSource(conn.from, saved); NullFileSystemObjectSink sink; /* just parse the NAR */ parseDump(sink, savedNARSource); }); @@ -473,20 +467,20 @@ static void performOp(TunnelLogger * logger, ref store, *dumpSource, baseName, FileSerialisationMethod::NixArchive, method, hashAlgo); logger->stopWork(); - to << store->printStorePath(path); + conn.to << store->printStorePath(path); } break; } case WorkerProto::Op::AddMultipleToStore: { bool repair, dontCheckSigs; - from >> repair >> dontCheckSigs; + conn.from >> repair >> dontCheckSigs; if (!trusted && dontCheckSigs) dontCheckSigs = false; logger->startWork(); { - FramedSource source(from); + FramedSource source(conn.from); store->addMultipleToStore(source, RepairFlag{repair}, dontCheckSigs ? NoCheckSigs : CheckSigs); @@ -496,8 +490,8 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::AddTextToStore: { - std::string suffix = readString(from); - std::string s = readString(from); + std::string suffix = readString(conn.from); + std::string s = readString(conn.from); auto refs = WorkerProto::Serialise::read(*store, rconn); logger->startWork(); auto path = ({ @@ -505,37 +499,37 @@ static void performOp(TunnelLogger * logger, ref store, store->addToStoreFromDump(source, suffix, FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, refs, NoRepair); }); logger->stopWork(); - to << store->printStorePath(path); + conn.to << store->printStorePath(path); break; } case WorkerProto::Op::ExportPath: { - auto path = store->parseStorePath(readString(from)); - readInt(from); // obsolete + auto path = store->parseStorePath(readString(conn.from)); + readInt(conn.from); // obsolete logger->startWork(); - TunnelSink sink(to); + TunnelSink sink(conn.to); store->exportPath(path, sink); logger->stopWork(); - to << 1; + conn.to << 1; break; } case WorkerProto::Op::ImportPaths: { logger->startWork(); - TunnelSource source(from, to); + TunnelSource source(conn.from, conn.to); auto paths = store->importPaths(source, trusted ? NoCheckSigs : CheckSigs); logger->stopWork(); Strings paths2; for (auto & i : paths) paths2.push_back(store->printStorePath(i)); - to << paths2; + conn.to << paths2; break; } case WorkerProto::Op::BuildPaths: { auto drvs = WorkerProto::Serialise::read(*store, rconn); BuildMode mode = bmNormal; - if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { + if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 15) { mode = WorkerProto::Serialise::read(*store, rconn); /* Repairing is not atomic, so disallowed for "untrusted" @@ -553,7 +547,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); store->buildPaths(drvs, mode); logger->stopWork(); - to << 1; + conn.to << 1; break; } @@ -579,7 +573,7 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::BuildDerivation: { - auto drvPath = store->parseStorePath(readString(from)); + auto drvPath = store->parseStorePath(readString(conn.from)); BasicDerivation drv; /* * Note: unlike wopEnsurePath, this operation reads a @@ -590,7 +584,7 @@ static void performOp(TunnelLogger * logger, ref store, * it cannot be trusted that its outPath was calculated * correctly. */ - readDerivation(from, *store, drv, Derivation::nameFromPath(drvPath)); + readDerivation(conn.from, *store, drv, Derivation::nameFromPath(drvPath)); auto buildMode = WorkerProto::Serialise::read(*store, rconn); logger->startWork(); @@ -656,20 +650,20 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::EnsurePath: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); store->ensurePath(path); logger->stopWork(); - to << 1; + conn.to << 1; break; } case WorkerProto::Op::AddTempRoot: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); store->addTempRoot(path); logger->stopWork(); - to << 1; + conn.to << 1; break; } @@ -679,24 +673,24 @@ static void performOp(TunnelLogger * logger, ref store, "you are not privileged to create perm roots\n\n" "hint: you can just do this client-side without special privileges, and probably want to do that instead."); auto storePath = WorkerProto::Serialise::read(*store, rconn); - Path gcRoot = absPath(readString(from)); + Path gcRoot = absPath(readString(conn.from)); logger->startWork(); auto & localFSStore = require(*store); localFSStore.addPermRoot(storePath, gcRoot); logger->stopWork(); - to << gcRoot; + conn.to << gcRoot; break; } case WorkerProto::Op::AddIndirectRoot: { - Path path = absPath(readString(from)); + Path path = absPath(readString(conn.from)); logger->startWork(); auto & indirectRootStore = require(*store); indirectRootStore.addIndirectRoot(path); logger->stopWork(); - to << 1; + conn.to << 1; break; } @@ -704,7 +698,7 @@ static void performOp(TunnelLogger * logger, ref store, case WorkerProto::Op::SyncWithGC: { logger->startWork(); logger->stopWork(); - to << 1; + conn.to << 1; break; } @@ -718,24 +712,24 @@ static void performOp(TunnelLogger * logger, ref store, for (auto & i : roots) size += i.second.size(); - to << size; + conn.to << size; for (auto & [target, links] : roots) for (auto & link : links) - to << link << store->printStorePath(target); + conn.to << link << store->printStorePath(target); break; } case WorkerProto::Op::CollectGarbage: { GCOptions options; - options.action = (GCOptions::GCAction) readInt(from); + options.action = (GCOptions::GCAction) readInt(conn.from); options.pathsToDelete = WorkerProto::Serialise::read(*store, rconn); - from >> options.ignoreLiveness >> options.maxFreed; + conn.from >> options.ignoreLiveness >> options.maxFreed; // obsolete fields - readInt(from); - readInt(from); - readInt(from); + readInt(conn.from); + readInt(conn.from); + readInt(conn.from); GCResults results; @@ -746,7 +740,7 @@ static void performOp(TunnelLogger * logger, ref store, gcStore.collectGarbage(options, results); logger->stopWork(); - to << results.paths << results.bytesFreed << 0 /* obsolete */; + conn.to << results.paths << results.bytesFreed << 0 /* obsolete */; break; } @@ -755,24 +749,24 @@ static void performOp(TunnelLogger * logger, ref store, ClientSettings clientSettings; - clientSettings.keepFailed = readInt(from); - clientSettings.keepGoing = readInt(from); - clientSettings.tryFallback = readInt(from); - clientSettings.verbosity = (Verbosity) readInt(from); - clientSettings.maxBuildJobs = readInt(from); - clientSettings.maxSilentTime = readInt(from); - readInt(from); // obsolete useBuildHook - clientSettings.verboseBuild = lvlError == (Verbosity) readInt(from); - readInt(from); // obsolete logType - readInt(from); // obsolete printBuildTrace - clientSettings.buildCores = readInt(from); - clientSettings.useSubstitutes = readInt(from); + clientSettings.keepFailed = readInt(conn.from); + clientSettings.keepGoing = readInt(conn.from); + clientSettings.tryFallback = readInt(conn.from); + clientSettings.verbosity = (Verbosity) readInt(conn.from); + clientSettings.maxBuildJobs = readInt(conn.from); + clientSettings.maxSilentTime = readInt(conn.from); + readInt(conn.from); // obsolete useBuildHook + clientSettings.verboseBuild = lvlError == (Verbosity) readInt(conn.from); + readInt(conn.from); // obsolete logType + readInt(conn.from); // obsolete printBuildTrace + clientSettings.buildCores = readInt(conn.from); + clientSettings.useSubstitutes = readInt(conn.from); - if (GET_PROTOCOL_MINOR(clientVersion) >= 12) { - unsigned int n = readInt(from); + if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 12) { + unsigned int n = readInt(conn.from); for (unsigned int i = 0; i < n; i++) { - auto name = readString(from); - auto value = readString(from); + auto name = readString(conn.from); + auto value = readString(conn.from); clientSettings.overrides.emplace(name, value); } } @@ -789,20 +783,20 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::QuerySubstitutablePathInfo: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); SubstitutablePathInfos infos; store->querySubstitutablePathInfos({{path, std::nullopt}}, infos); logger->stopWork(); auto i = infos.find(path); if (i == infos.end()) - to << 0; + conn.to << 0; else { - to << 1 + conn.to << 1 << (i->second.deriver ? store->printStorePath(*i->second.deriver) : ""); WorkerProto::write(*store, wconn, i->second.references); - to << i->second.downloadSize - << i->second.narSize; + conn.to << i->second.downloadSize + << i->second.narSize; } break; } @@ -810,7 +804,7 @@ static void performOp(TunnelLogger * logger, ref store, case WorkerProto::Op::QuerySubstitutablePathInfos: { SubstitutablePathInfos infos; StorePathCAMap pathsMap = {}; - if (GET_PROTOCOL_MINOR(clientVersion) < 22) { + if (GET_PROTOCOL_MINOR(conn.protoVersion) < 22) { auto paths = WorkerProto::Serialise::read(*store, rconn); for (auto & path : paths) pathsMap.emplace(path, std::nullopt); @@ -819,12 +813,12 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); store->querySubstitutablePathInfos(pathsMap, infos); logger->stopWork(); - to << infos.size(); + conn.to << infos.size(); for (auto & i : infos) { - to << store->printStorePath(i.first) - << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); + conn.to << store->printStorePath(i.first) + << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); WorkerProto::write(*store, wconn, i.second.references); - to << i.second.downloadSize << i.second.narSize; + conn.to << i.second.downloadSize << i.second.narSize; } break; } @@ -838,22 +832,22 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::QueryPathInfo: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); std::shared_ptr info; logger->startWork(); try { info = store->queryPathInfo(path); } catch (InvalidPath &) { - if (GET_PROTOCOL_MINOR(clientVersion) < 17) throw; + if (GET_PROTOCOL_MINOR(conn.protoVersion) < 17) throw; } logger->stopWork(); if (info) { - if (GET_PROTOCOL_MINOR(clientVersion) >= 17) - to << 1; + if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 17) + conn.to << 1; WorkerProto::write(*store, wconn, static_cast(*info)); } else { - assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); - to << 0; + assert(GET_PROTOCOL_MINOR(conn.protoVersion) >= 17); + conn.to << 0; } break; } @@ -862,61 +856,61 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); store->optimiseStore(); logger->stopWork(); - to << 1; + conn.to << 1; break; case WorkerProto::Op::VerifyStore: { bool checkContents, repair; - from >> checkContents >> repair; + conn.from >> checkContents >> repair; logger->startWork(); if (repair && !trusted) throw Error("you are not privileged to repair paths"); bool errors = store->verifyStore(checkContents, (RepairFlag) repair); logger->stopWork(); - to << errors; + conn.to << errors; break; } case WorkerProto::Op::AddSignatures: { - auto path = store->parseStorePath(readString(from)); - StringSet sigs = readStrings(from); + auto path = store->parseStorePath(readString(conn.from)); + StringSet sigs = readStrings(conn.from); logger->startWork(); store->addSignatures(path, sigs); logger->stopWork(); - to << 1; + conn.to << 1; break; } case WorkerProto::Op::NarFromPath: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); logger->stopWork(); - dumpPath(store->toRealPath(path), to); + dumpPath(store->toRealPath(path), conn.to); break; } case WorkerProto::Op::AddToStoreNar: { bool repair, dontCheckSigs; - auto path = store->parseStorePath(readString(from)); - auto deriver = readString(from); - auto narHash = Hash::parseAny(readString(from), HashAlgorithm::SHA256); + auto path = store->parseStorePath(readString(conn.from)); + auto deriver = readString(conn.from); + auto narHash = Hash::parseAny(readString(conn.from), HashAlgorithm::SHA256); ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = store->parseStorePath(deriver); info.references = WorkerProto::Serialise::read(*store, rconn); - from >> info.registrationTime >> info.narSize >> info.ultimate; - info.sigs = readStrings(from); - info.ca = ContentAddress::parseOpt(readString(from)); - from >> repair >> dontCheckSigs; + conn.from >> info.registrationTime >> info.narSize >> info.ultimate; + info.sigs = readStrings(conn.from); + info.ca = ContentAddress::parseOpt(readString(conn.from)); + conn.from >> repair >> dontCheckSigs; if (!trusted && dontCheckSigs) dontCheckSigs = false; if (!trusted) info.ultimate = false; - if (GET_PROTOCOL_MINOR(clientVersion) >= 23) { + if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 23) { logger->startWork(); { - FramedSource source(from); + FramedSource source(conn.from); store->addToStore(info, source, (RepairFlag) repair, dontCheckSigs ? NoCheckSigs : CheckSigs); } @@ -926,10 +920,10 @@ static void performOp(TunnelLogger * logger, ref store, else { std::unique_ptr source; StringSink saved; - if (GET_PROTOCOL_MINOR(clientVersion) >= 21) - source = std::make_unique(from, to); + if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 21) + source = std::make_unique(conn.from, conn.to); else { - TeeSource tee { from, saved }; + TeeSource tee { conn.from, saved }; NullFileSystemObjectSink ether; parseDump(ether, tee); source = std::make_unique(saved.s); @@ -957,15 +951,15 @@ static void performOp(TunnelLogger * logger, ref store, WorkerProto::write(*store, wconn, willBuild); WorkerProto::write(*store, wconn, willSubstitute); WorkerProto::write(*store, wconn, unknown); - to << downloadSize << narSize; + conn.to << downloadSize << narSize; break; } case WorkerProto::Op::RegisterDrvOutput: { logger->startWork(); - if (GET_PROTOCOL_MINOR(clientVersion) < 31) { - auto outputId = DrvOutput::parse(readString(from)); - auto outputPath = StorePath(readString(from)); + if (GET_PROTOCOL_MINOR(conn.protoVersion) < 31) { + auto outputId = DrvOutput::parse(readString(conn.from)); + auto outputPath = StorePath(readString(conn.from)); store->registerDrvOutput(Realisation{ .id = outputId, .outPath = outputPath}); } else { @@ -978,10 +972,10 @@ static void performOp(TunnelLogger * logger, ref store, case WorkerProto::Op::QueryRealisation: { logger->startWork(); - auto outputId = DrvOutput::parse(readString(from)); + auto outputId = DrvOutput::parse(readString(conn.from)); auto info = store->queryRealisation(outputId); logger->stopWork(); - if (GET_PROTOCOL_MINOR(clientVersion) < 31) { + if (GET_PROTOCOL_MINOR(conn.protoVersion) < 31) { std::set outPaths; if (info) outPaths.insert(info->outPath); WorkerProto::write(*store, wconn, outPaths); @@ -994,19 +988,19 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::AddBuildLog: { - StorePath path{readString(from)}; + StorePath path{readString(conn.from)}; logger->startWork(); if (!trusted) throw Error("you are not privileged to add logs"); auto & logStore = require(*store); { - FramedSource source(from); + FramedSource source(conn.from); StringSink sink; source.drainInto(sink); logStore.addBuildLog(path, sink.s); } logger->stopWork(); - to << 1; + conn.to << 1; break; } @@ -1021,8 +1015,8 @@ static void performOp(TunnelLogger * logger, ref store, void processConnection( ref store, - FdSource & from, - FdSink & to, + FdSource && from, + FdSink && to, TrustedFlag trusted, RecursiveFlag recursive) { @@ -1031,14 +1025,20 @@ void processConnection( #endif /* Exchange the greeting. */ - WorkerProto::Version clientVersion = + auto [protoVersion, features] = WorkerProto::BasicServerConnection::handshake( - to, from, PROTOCOL_VERSION); + to, from, PROTOCOL_VERSION, WorkerProto::allFeatures); - if (clientVersion < 0x10a) + if (protoVersion < 0x10a) throw Error("the Nix client version is too old"); - auto tunnelLogger = new TunnelLogger(to, clientVersion); + WorkerProto::BasicServerConnection conn; + conn.to = std::move(to); + conn.from = std::move(from); + conn.protoVersion = protoVersion; + conn.features = features; + + auto tunnelLogger = new TunnelLogger(conn.to, protoVersion); auto prevLogger = nix::logger; // FIXME if (!recursive) @@ -1051,12 +1051,6 @@ void processConnection( printMsgUsing(prevLogger, lvlDebug, "%d operations", opCount); }); - WorkerProto::BasicServerConnection conn { - .to = to, - .from = from, - .clientVersion = clientVersion, - }; - conn.postHandshake(*store, { .daemonNixVersion = nixVersion, // We and the underlying store both need to trust the client for @@ -1072,13 +1066,13 @@ void processConnection( try { tunnelLogger->stopWork(); - to.flush(); + conn.to.flush(); /* Process client requests. */ while (true) { WorkerProto::Op op; try { - op = (enum WorkerProto::Op) readInt(from); + op = (enum WorkerProto::Op) readInt(conn.from); } catch (Interrupted & e) { break; } catch (EndOfFile & e) { @@ -1092,7 +1086,7 @@ void processConnection( debug("performing daemon worker op: %d", op); try { - performOp(tunnelLogger, store, trusted, recursive, clientVersion, from, to, op); + performOp(tunnelLogger, store, trusted, recursive, conn, op); } catch (Error & e) { /* If we're not in a state where we can send replies, then something went wrong processing the input of the @@ -1108,19 +1102,19 @@ void processConnection( throw; } - to.flush(); + conn.to.flush(); assert(!tunnelLogger->state_.lock()->canSendStderr); }; } catch (Error & e) { tunnelLogger->stopWork(&e); - to.flush(); + conn.to.flush(); return; } catch (std::exception & e) { auto ex = Error(e.what()); tunnelLogger->stopWork(&ex); - to.flush(); + conn.to.flush(); return; } } diff --git a/src/libstore/daemon.hh b/src/libstore/daemon.hh index 1964c0d99..a8ce32d8d 100644 --- a/src/libstore/daemon.hh +++ b/src/libstore/daemon.hh @@ -10,8 +10,8 @@ enum RecursiveFlag : bool { NotRecursive = false, Recursive = true }; void processConnection( ref store, - FdSource & from, - FdSink & to, + FdSource && from, + FdSink && to, TrustedFlag trusted, RecursiveFlag recursive); diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 17ebaace6..c1e871e93 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -6,6 +6,13 @@ namespace nix { struct DummyStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; + DummyStoreConfig(std::string_view scheme, std::string_view authority, const Params & params) + : StoreConfig(params) + { + if (!authority.empty()) + throw UsageError("`%s` store URIs must not contain an authority part %s", scheme, authority); + } + const std::string name() override { return "Dummy Store"; } std::string doc() override @@ -14,21 +21,22 @@ struct DummyStoreConfig : virtual StoreConfig { #include "dummy-store.md" ; } + + static std::set uriSchemes() { + return {"dummy"}; + } }; struct DummyStore : public virtual DummyStoreConfig, public virtual Store { DummyStore(std::string_view scheme, std::string_view authority, const Params & params) - : DummyStore(params) - { - if (!authority.empty()) - throw UsageError("`%s` store URIs must not contain an authority part %s", scheme, authority); - } + : StoreConfig(params) + , DummyStoreConfig(scheme, authority, params) + , Store(params) + { } DummyStore(const Params & params) - : StoreConfig(params) - , DummyStoreConfig(params) - , Store(params) + : DummyStore("dummy", "", params) { } std::string getUri() override @@ -50,10 +58,6 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store return Trusted; } - static std::set uriSchemes() { - return {"dummy"}; - } - std::optional queryPathFromHashPart(const std::string & hashPart) override { unsupported("queryPathFromHashPart"); } diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index cbbb0fe7a..5ea8b6f96 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -71,7 +71,10 @@ struct curlFileTransfer : public FileTransfer curl_off_t writtenToSink = 0; + std::chrono::steady_clock::time_point startTime = std::chrono::steady_clock::now(); + inline static const std::set successfulStatuses {200, 201, 204, 206, 304, 0 /* other protocol */}; + /* Get the HTTP status code, or 0 for other protocols. */ long getHTTPStatus() { @@ -373,10 +376,14 @@ struct curlFileTransfer : public FileTransfer void finish(CURLcode code) { + auto finishTime = std::chrono::steady_clock::now(); + auto httpStatus = getHTTPStatus(); - debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes", - request.verb(), request.uri, code, httpStatus, result.bodySize); + debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes, duration = %.2f s", + request.verb(), request.uri, code, httpStatus, result.bodySize, + std::chrono::duration_cast(finishTime - startTime).count() / 1000.0f + ); appendCurrentUrl(); @@ -851,8 +858,10 @@ void FileTransfer::download( buffer). We don't wait forever to prevent stalling the download thread. (Hopefully sleeping will throttle the sender.) */ - if (state->data.size() > 1024 * 1024) { + if (state->data.size() > fileTransferSettings.downloadBufferSize) { debug("download buffer is full; going to sleep"); + static bool haveWarned = false; + warnOnce(haveWarned, "download buffer is full; consider increasing the 'download-buffer-size' setting"); state.wait_for(state->request, std::chrono::seconds(10)); } diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index 1f5b4ab93..d836ab2c4 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -47,6 +47,12 @@ struct FileTransferSettings : Config Setting tries{this, 5, "download-attempts", "How often Nix will attempt to download a file before giving up."}; + + Setting downloadBufferSize{this, 64 * 1024 * 1024, "download-buffer-size", + R"( + The size of Nix's internal download buffer during `curl` transfers. If data is + not processed quickly enough to exceed the size of this buffer, downloads may stall. + )"}; }; extern FileTransferSettings fileTransferSettings; diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 54e09cb16..1494712da 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -559,7 +559,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) non-blocking flag from the server socket, so explicitly make it blocking. */ if (fcntl(fdClient.get(), F_SETFL, fcntl(fdClient.get(), F_GETFL) & ~O_NONBLOCK) == -1) - abort(); + panic("Could not set non-blocking flag on client socket"); while (true) { try { @@ -891,7 +891,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) void LocalStore::autoGC(bool sync) { -#ifdef HAVE_STATVFS +#if HAVE_STATVFS static auto fakeFreeSpaceFile = getEnv("_NIX_TEST_FREE_SPACE_FILE"); auto getAvail = [this]() -> uint64_t { diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 4eabf6054..439a6f97c 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -15,7 +15,6 @@ #include #ifndef _WIN32 -# include # include #endif @@ -298,25 +297,28 @@ template<> std::string BaseSetting::to_string() const if (value == smEnabled) return "true"; else if (value == smRelaxed) return "relaxed"; else if (value == smDisabled) return "false"; - else abort(); + else unreachable(); } template<> void BaseSetting::convertToArg(Args & args, const std::string & category) { args.addFlag({ .longName = name, + .aliases = aliases, .description = "Enable sandboxing.", .category = category, .handler = {[this]() { override(smEnabled); }} }); args.addFlag({ .longName = "no-" + name, + .aliases = aliases, .description = "Disable sandboxing.", .category = category, .handler = {[this]() { override(smDisabled); }} }); args.addFlag({ .longName = "relaxed-" + name, + .aliases = aliases, .description = "Enable sandboxing, but allow builds to disable it.", .category = category, .handler = {[this]() { override(smRelaxed); }} @@ -335,60 +337,6 @@ unsigned int MaxBuildJobsSetting::parse(const std::string & str) const } -Paths PluginFilesSetting::parse(const std::string & str) const -{ - if (pluginsLoaded) - throw UsageError("plugin-files set after plugins were loaded, you may need to move the flag before the subcommand"); - return BaseSetting::parse(str); -} - - -void initPlugins() -{ - assert(!settings.pluginFiles.pluginsLoaded); - for (const auto & pluginFile : settings.pluginFiles.get()) { - std::vector pluginFiles; - try { - auto ents = std::filesystem::directory_iterator{pluginFile}; - for (const auto & ent : ents) { - checkInterrupt(); - pluginFiles.emplace_back(ent.path()); - } - } catch (std::filesystem::filesystem_error & e) { - if (e.code() != std::errc::not_a_directory) - throw; - pluginFiles.emplace_back(pluginFile); - } - for (const auto & file : pluginFiles) { - checkInterrupt(); - /* handle is purposefully leaked as there may be state in the - DSO needed by the action of the plugin. */ -#ifndef _WIN32 // TODO implement via DLL loading on Windows - void *handle = - dlopen(file.c_str(), RTLD_LAZY | RTLD_LOCAL); - if (!handle) - throw Error("could not dynamically open plugin file '%s': %s", file, dlerror()); - - /* Older plugins use a statically initialized object to run their code. - Newer plugins can also export nix_plugin_entry() */ - void (*nix_plugin_entry)() = (void (*)())dlsym(handle, "nix_plugin_entry"); - if (nix_plugin_entry) - nix_plugin_entry(); -#else - throw Error("could not dynamically open plugin file '%s'", file); -#endif - } - } - - /* Since plugins can add settings, try to re-apply previously - unknown settings. */ - globalConfig.reapplyUnknownSettings(); - globalConfig.warnUnknownSettings(); - - /* Tell the user if they try to set plugin-files after we've already loaded */ - settings.pluginFiles.pluginsLoaded = true; -} - static void preloadNSS() { /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index dfe25f317..8760c9d14 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -31,23 +31,6 @@ struct MaxBuildJobsSetting : public BaseSetting unsigned int parse(const std::string & str) const override; }; -struct PluginFilesSetting : public BaseSetting -{ - bool pluginsLoaded = false; - - PluginFilesSetting(Config * options, - const Paths & def, - const std::string & name, - const std::string & description, - const std::set & aliases = {}) - : BaseSetting(def, true, name, description, aliases) - { - options->addSetting(this); - } - - Paths parse(const std::string & str) const override; -}; - const uint32_t maxIdsPerBuild = #if __linux__ 1 << 16 @@ -303,7 +286,7 @@ public: For backward compatibility, `ssh://` may be omitted. The hostname may be an alias defined in `~/.ssh/config`. - 2. A comma-separated list of [Nix system types](@docroot@/contributing/hacking.md#system-type). + 2. A comma-separated list of [Nix system types](@docroot@/development/building.md#system-type). If omitted, this defaults to the local platform type. > **Example** @@ -883,13 +866,13 @@ public: - `ca-derivations` - Included by default if the [`ca-derivations` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-ca-derivations) is enabled. + Included by default if the [`ca-derivations` experimental feature](@docroot@/development/experimental-features.md#xp-feature-ca-derivations) is enabled. This system feature is implicitly required by derivations with the [`__contentAddressed` attribute](@docroot@/language/advanced-attributes.md#adv-attr-__contentAddressed). - `recursive-nix` - Included by default if the [`recursive-nix` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-recursive-nix) is enabled. + Included by default if the [`recursive-nix` experimental feature](@docroot@/development/experimental-features.md#xp-feature-recursive-nix) is enabled. - `uid-range` @@ -1158,33 +1141,6 @@ public: Setting minFreeCheckInterval{this, 5, "min-free-check-interval", "Number of seconds between checking free disk space."}; - PluginFilesSetting pluginFiles{ - this, {}, "plugin-files", - R"( - A list of plugin files to be loaded by Nix. Each of these files will - be dlopened by Nix. If they contain the symbol `nix_plugin_entry()`, - this symbol will be called. Alternatively, they can affect execution - through static initialization. In particular, these plugins may construct - static instances of RegisterPrimOp to add new primops or constants to the - expression language, RegisterStoreImplementation to add new store - implementations, RegisterCommand to add new subcommands to the `nix` - command, and RegisterSetting to add new nix config settings. See the - constructors for those types for more details. - - Warning! These APIs are inherently unstable and may change from - release to release. - - Since these files are loaded into the same address space as Nix - itself, they must be DSOs compatible with the instance of Nix - running at the time (i.e. compiled against the same headers, not - linked to any incompatible libraries). They should not be linked to - any Nix libs directly, as those will be available already at load - time. - - If an entry in the list is a directory, all files in the directory - are loaded as plugins (non-recursively). - )"}; - Setting narBufferSize{this, 32 * 1024 * 1024, "nar-buffer-size", "Maximum size of NARs before spilling them to disk."}; @@ -1278,12 +1234,6 @@ public: // FIXME: don't use a global variable. extern Settings settings; -/** - * This should be called after settings are initialized, but before - * anything else - */ -void initPlugins(); - /** * Load the configuration (from `nix.conf`, `NIX_CONFIG`, etc.) into the * given configuration object. diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 3328caef9..b15ef4e4c 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -1,4 +1,4 @@ -#include "binary-cache-store.hh" +#include "http-binary-cache-store.hh" #include "filetransfer.hh" #include "globals.hh" #include "nar-info-disk-cache.hh" @@ -8,26 +8,37 @@ namespace nix { MakeError(UploadToHTTP, Error); -struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig + +HttpBinaryCacheStoreConfig::HttpBinaryCacheStoreConfig( + std::string_view scheme, + std::string_view _cacheUri, + const Params & params) + : StoreConfig(params) + , BinaryCacheStoreConfig(params) + , cacheUri( + std::string { scheme } + + "://" + + (!_cacheUri.empty() + ? _cacheUri + : throw UsageError("`%s` Store requires a non-empty authority in Store URL", scheme))) { - using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + while (!cacheUri.empty() && cacheUri.back() == '/') + cacheUri.pop_back(); +} - const std::string name() override { return "HTTP Binary Cache Store"; } - std::string doc() override - { - return - #include "http-binary-cache-store.md" - ; - } -}; +std::string HttpBinaryCacheStoreConfig::doc() +{ + return + #include "http-binary-cache-store.md" + ; +} + class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public virtual BinaryCacheStore { private: - Path cacheUri; - struct State { bool enabled = true; @@ -40,23 +51,14 @@ public: HttpBinaryCacheStore( std::string_view scheme, - PathView _cacheUri, + PathView cacheUri, const Params & params) : StoreConfig(params) , BinaryCacheStoreConfig(params) - , HttpBinaryCacheStoreConfig(params) + , HttpBinaryCacheStoreConfig(scheme, cacheUri, params) , Store(params) , BinaryCacheStore(params) - , cacheUri( - std::string { scheme } - + "://" - + (!_cacheUri.empty() - ? _cacheUri - : throw UsageError("`%s` Store requires a non-empty authority in Store URL", scheme))) { - while (!cacheUri.empty() && cacheUri.back() == '/') - cacheUri.pop_back(); - diskCache = getNarInfoDiskCache(); } @@ -81,14 +83,6 @@ public: } } - static std::set uriSchemes() - { - static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; - auto ret = std::set({"http", "https"}); - if (forceHttp) ret.insert("file"); - return ret; - } - protected: void maybeDisable() diff --git a/src/libstore/http-binary-cache-store.hh b/src/libstore/http-binary-cache-store.hh new file mode 100644 index 000000000..d2fc43210 --- /dev/null +++ b/src/libstore/http-binary-cache-store.hh @@ -0,0 +1,30 @@ +#include "binary-cache-store.hh" + +namespace nix { + +struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +{ + using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + + HttpBinaryCacheStoreConfig(std::string_view scheme, std::string_view _cacheUri, const Params & params); + + Path cacheUri; + + const std::string name() override + { + return "HTTP Binary Cache Store"; + } + + static std::set uriSchemes() + { + static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; + auto ret = std::set({"http", "https"}); + if (forceHttp) + ret.insert("file"); + return ret; + } + + std::string doc() override; +}; + +} diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh index f26651898..b541455b4 100644 --- a/src/libstore/legacy-ssh-store.hh +++ b/src/libstore/legacy-ssh-store.hh @@ -26,6 +26,8 @@ struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig const std::string name() override { return "SSH Store"; } + static std::set uriSchemes() { return {"ssh"}; } + std::string doc() override; }; @@ -46,8 +48,6 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor SSHMaster master; - static std::set uriSchemes() { return {"ssh"}; } - LegacySSHStore( std::string_view scheme, std::string_view host, diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index aa2efdb8f..dcc6affe4 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -1,4 +1,4 @@ -#include "binary-cache-store.hh" +#include "local-binary-cache-store.hh" #include "globals.hh" #include "nar-info-disk-cache.hh" #include "signals.hh" @@ -7,28 +7,27 @@ namespace nix { -struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +LocalBinaryCacheStoreConfig::LocalBinaryCacheStoreConfig( + std::string_view scheme, + PathView binaryCacheDir, + const Params & params) + : StoreConfig(params) + , BinaryCacheStoreConfig(params) + , binaryCacheDir(binaryCacheDir) { - using BinaryCacheStoreConfig::BinaryCacheStoreConfig; +} - const std::string name() override { return "Local Binary Cache Store"; } - std::string doc() override - { - return - #include "local-binary-cache-store.md" - ; - } -}; - -class LocalBinaryCacheStore : public virtual LocalBinaryCacheStoreConfig, public virtual BinaryCacheStore +std::string LocalBinaryCacheStoreConfig::doc() { -private: + return + #include "local-binary-cache-store.md" + ; +} - Path binaryCacheDir; - -public: +struct LocalBinaryCacheStore : virtual LocalBinaryCacheStoreConfig, virtual BinaryCacheStore +{ /** * @param binaryCacheDir `file://` is a short-hand for `file:///` * for now. @@ -39,10 +38,9 @@ public: const Params & params) : StoreConfig(params) , BinaryCacheStoreConfig(params) - , LocalBinaryCacheStoreConfig(params) + , LocalBinaryCacheStoreConfig(scheme, binaryCacheDir, params) , Store(params) , BinaryCacheStore(params) - , binaryCacheDir(binaryCacheDir) { } @@ -53,8 +51,6 @@ public: return "file://" + binaryCacheDir; } - static std::set uriSchemes(); - protected: bool fileExists(const std::string & path) override; @@ -123,7 +119,7 @@ bool LocalBinaryCacheStore::fileExists(const std::string & path) return pathExists(binaryCacheDir + "/" + path); } -std::set LocalBinaryCacheStore::uriSchemes() +std::set LocalBinaryCacheStoreConfig::uriSchemes() { if (getEnv("_NIX_FORCE_HTTP") == "1") return {}; diff --git a/src/libstore/local-binary-cache-store.hh b/src/libstore/local-binary-cache-store.hh new file mode 100644 index 000000000..997e8ecbb --- /dev/null +++ b/src/libstore/local-binary-cache-store.hh @@ -0,0 +1,23 @@ +#include "binary-cache-store.hh" + +namespace nix { + +struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +{ + using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + + LocalBinaryCacheStoreConfig(std::string_view scheme, PathView binaryCacheDir, const Params & params); + + Path binaryCacheDir; + + const std::string name() override + { + return "Local Binary Cache Store"; + } + + static std::set uriSchemes(); + + std::string doc() override; +}; + +} diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 843c0d288..5449b20eb 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -8,6 +8,20 @@ namespace nix { +LocalFSStoreConfig::LocalFSStoreConfig(PathView rootDir, const Params & params) + : StoreConfig(params) + // Default `?root` from `rootDir` if non set + // FIXME don't duplicate description once we don't have root setting + , rootDir{ + this, + !rootDir.empty() && params.count("root") == 0 + ? (std::optional{rootDir}) + : std::nullopt, + "root", + "Directory prefixed to all other paths."} +{ +} + LocalFSStore::LocalFSStore(const Params & params) : Store(params) { diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 8fb081200..9bb569f0b 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -11,6 +11,15 @@ struct LocalFSStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; + /** + * Used to override the `root` settings. Can't be done via modifying + * `params` reliably because this parameter is unused except for + * passing to base class constructors. + * + * @todo Make this less error-prone with new store settings system. + */ + LocalFSStoreConfig(PathView path, const Params & params); + const OptionalPathSetting rootDir{this, std::nullopt, "root", "Directory prefixed to all other paths."}; diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 598415db8..ec2c5f4e9 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -18,11 +18,11 @@ Path LocalOverlayStoreConfig::toUpperPath(const StorePath & path) { return upperLayer + "/" + path.to_string(); } -LocalOverlayStore::LocalOverlayStore(const Params & params) +LocalOverlayStore::LocalOverlayStore(std::string_view scheme, PathView path, const Params & params) : StoreConfig(params) - , LocalFSStoreConfig(params) + , LocalFSStoreConfig(path, params) , LocalStoreConfig(params) - , LocalOverlayStoreConfig(params) + , LocalOverlayStoreConfig(scheme, path, params) , Store(params) , LocalFSStore(params) , LocalStore(params) diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 35a301013..63628abed 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -8,11 +8,16 @@ namespace nix { struct LocalOverlayStoreConfig : virtual LocalStoreConfig { LocalOverlayStoreConfig(const StringMap & params) - : StoreConfig(params) - , LocalFSStoreConfig(params) - , LocalStoreConfig(params) + : LocalOverlayStoreConfig("local-overlay", "", params) { } + LocalOverlayStoreConfig(std::string_view scheme, PathView path, const Params & params) + : StoreConfig(params) + , LocalFSStoreConfig(path, params) + , LocalStoreConfig(scheme, path, params) + { + } + const Setting lowerStoreUri{(StoreConfig*) this, "", "lower-store", R"( [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) @@ -58,6 +63,11 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig return ExperimentalFeature::LocalOverlayStore; } + static std::set uriSchemes() + { + return { "local-overlay" }; + } + std::string doc() override; protected: @@ -90,19 +100,12 @@ class LocalOverlayStore : public virtual LocalOverlayStoreConfig, public virtual ref lowerStore; public: - LocalOverlayStore(const Params & params); - - LocalOverlayStore(std::string_view scheme, PathView path, const Params & params) - : LocalOverlayStore(params) + LocalOverlayStore(const Params & params) + : LocalOverlayStore("local-overlay", "", params) { - if (!path.empty()) - throw UsageError("local-overlay:// store url doesn't support path part, only scheme and query params"); } - static std::set uriSchemes() - { - return { "local-overlay" }; - } + LocalOverlayStore(std::string_view scheme, PathView path, const Params & params); std::string getUri() override { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 82b70ff21..819cee345 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -56,6 +56,15 @@ namespace nix { +LocalStoreConfig::LocalStoreConfig( + std::string_view scheme, + std::string_view authority, + const Params & params) + : StoreConfig(params) + , LocalFSStoreConfig(authority, params) +{ +} + std::string LocalStoreConfig::doc() { return @@ -183,10 +192,13 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) } } -LocalStore::LocalStore(const Params & params) +LocalStore::LocalStore( + std::string_view scheme, + PathView path, + const Params & params) : StoreConfig(params) - , LocalFSStoreConfig(params) - , LocalStoreConfig(params) + , LocalFSStoreConfig(path, params) + , LocalStoreConfig(scheme, path, params) , Store(params) , LocalFSStore(params) , dbDir(stateDir + "/db") @@ -465,19 +477,8 @@ LocalStore::LocalStore(const Params & params) } -LocalStore::LocalStore( - std::string_view scheme, - PathView path, - const Params & _params) - : LocalStore([&]{ - // Default `?root` from `path` if non set - if (!path.empty() && _params.count("root") == 0) { - auto params = _params; - params.insert_or_assign("root", std::string { path }); - return params; - } - return _params; - }()) +LocalStore::LocalStore(const Params & params) + : LocalStore("local", "", params) { } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index b0a0def9a..a03cfc03b 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -38,6 +38,11 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig { using LocalFSStoreConfig::LocalFSStoreConfig; + LocalStoreConfig( + std::string_view scheme, + std::string_view authority, + const Params & params); + Setting requireSigs{this, settings.requireSigs, "require-sigs", @@ -62,6 +67,9 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig const std::string name() override { return "Local Store"; } + static std::set uriSchemes() + { return {"local"}; } + std::string doc() override; }; @@ -144,9 +152,6 @@ public: ~LocalStore(); - static std::set uriSchemes() - { return {"local"}; } - /** * Implementations of abstract store API methods. */ diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 297bf7187..50b15e15d 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -55,6 +55,7 @@ configdata.set('CAN_LINK_SYMLINK', can_link_symlink.to_int()) check_funcs = [ # Optionally used for canonicalising files from the build 'lchown', + 'statvfs', ] foreach funcspec : check_funcs define_name = 'HAVE_' + funcspec.underscorify().to_upper() @@ -72,6 +73,7 @@ subdir('build-utils-meson/threads') boost = dependency( 'boost', modules : ['container'], + include_type: 'system', ) # boost is a public dependency, but not a pkg-config dependency unfortunately, so we # put in `deps_other`. @@ -99,6 +101,23 @@ deps_public += nlohmann_json sqlite = dependency('sqlite3', 'sqlite', version : '>=3.6.19') deps_private += sqlite +# AWS C++ SDK has bad pkg-config +aws_s3 = dependency('aws-cpp-sdk-s3', required : false) +configdata.set('ENABLE_S3', aws_s3.found().to_int()) +if aws_s3.found() + aws_s3 = declare_dependency( + include_directories: include_directories(aws_s3.get_variable('includedir')), + link_args: [ + '-L' + aws_s3.get_variable('libdir'), + '-laws-cpp-sdk-transfer', + '-laws-cpp-sdk-s3', + '-laws-cpp-sdk-core', + '-laws-crt-cpp', + ], + ).as_system('system') +endif +deps_other += aws_s3 + subdir('build-utils-meson/generate-header') generated_headers = [] @@ -243,10 +262,12 @@ headers = [config_h] + files( 'filetransfer.hh', 'gc-store.hh', 'globals.hh', + 'http-binary-cache-store.hh', 'indirect-root-store.hh', 'keys.hh', 'legacy-ssh-store.hh', 'length-prefixed-protocol-helper.hh', + 'local-binary-cache-store.hh', 'local-fs-store.hh', 'local-overlay-store.hh', 'local-store.hh', diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 288f618d5..83e63794e 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -164,7 +164,7 @@ public: Cache & getCache(State & state, const std::string & uri) { auto i = state.caches.find(uri); - if (i == state.caches.end()) abort(); + if (i == state.caches.end()) unreachable(); return i->second; } @@ -211,7 +211,7 @@ public: { auto r(state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority)); - if (!r.next()) { abort(); } + if (!r.next()) { unreachable(); } ret.id = (int) r.getInt(0); } diff --git a/src/libstore/package.nix b/src/libstore/package.nix index 0a2ace91e..4582ba0d2 100644 --- a/src/libstore/package.nix +++ b/src/libstore/package.nix @@ -66,10 +66,7 @@ mkMesonDerivation (finalAttrs: { ] ++ lib.optional stdenv.hostPlatform.isLinux libseccomp # There have been issues building these dependencies ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin)) - (aws-sdk-cpp.override { - apis = ["s3" "transfer"]; - customMemoryManagement = false; - }) + aws-sdk-cpp ; propagatedBuildInputs = [ @@ -101,12 +98,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index d749ccd0a..555936c18 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -73,8 +73,11 @@ void RemoteStore::initConnection(Connection & conn) StringSink saved; TeeSource tee(conn.from, saved); try { - conn.daemonVersion = WorkerProto::BasicClientConnection::handshake( - conn.to, tee, PROTOCOL_VERSION); + auto [protoVersion, features] = WorkerProto::BasicClientConnection::handshake( + conn.to, tee, PROTOCOL_VERSION, + WorkerProto::allFeatures); + conn.protoVersion = protoVersion; + conn.features = features; } catch (SerialisationError & e) { /* In case the other side is waiting for our input, close it. */ @@ -88,6 +91,9 @@ void RemoteStore::initConnection(Connection & conn) static_cast(conn) = conn.postHandshake(*this); + for (auto & feature : conn.features) + debug("negotiated feature '%s'", feature); + auto ex = conn.processStderrReturn(); if (ex) std::rethrow_exception(ex); } @@ -115,7 +121,7 @@ void RemoteStore::setOptions(Connection & conn) << settings.buildCores << settings.useSubstitutes; - if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) { + if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 12) { std::map overrides; settings.getSettings(overrides, true); // libstore settings fileTransferSettings.getSettings(overrides, true); @@ -128,7 +134,7 @@ void RemoteStore::setOptions(Connection & conn) overrides.erase(settings.useSubstitutes.name); overrides.erase(loggerSettings.showTrace.name); overrides.erase(experimentalFeatureSettings.experimentalFeatures.name); - overrides.erase(settings.pluginFiles.name); + overrides.erase("plugin-files"); conn.to << overrides.size(); for (auto & i : overrides) conn.to << i.first << i.second.value; @@ -175,7 +181,7 @@ bool RemoteStore::isValidPathUncached(const StorePath & path) StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) { auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 12) { StorePathSet res; for (auto & i : paths) if (isValidPath(i)) res.insert(i); @@ -198,7 +204,7 @@ StorePathSet RemoteStore::queryAllValidPaths() StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) { auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 12) { StorePathSet res; for (auto & i : paths) { conn->to << WorkerProto::Op::HasSubstitutes << printStorePath(i); @@ -221,7 +227,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 12) { for (auto & i : pathsMap) { SubstitutablePathInfo info; @@ -241,7 +247,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S } else { conn->to << WorkerProto::Op::QuerySubstitutablePathInfos; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 22) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 22) { StorePathSet paths; for (auto & path : pathsMap) paths.insert(path.first); @@ -368,7 +374,7 @@ ref RemoteStore::addCAToStore( std::optional conn_(getConnection()); auto & conn = *conn_; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 25) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) >= 25) { conn->to << WorkerProto::Op::AddToStore @@ -485,7 +491,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, { auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 18) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 18) { auto source2 = sinkToSource([&](Sink & sink) { sink << 1 // == path follows ; @@ -513,11 +519,11 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 23) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) >= 23) { conn.withFramedSink([&](Sink & sink) { copyNAR(source, sink); }); - } else if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21) { + } else if (GET_PROTOCOL_MINOR(conn->protoVersion) >= 21) { conn.processStderr(0, &source); } else { copyNAR(source, conn->to); @@ -554,7 +560,7 @@ void RemoteStore::addMultipleToStore( RepairFlag repair, CheckSigsFlag checkSigs) { - if (GET_PROTOCOL_MINOR(getConnection()->daemonVersion) >= 32) { + if (GET_PROTOCOL_MINOR(getConnection()->protoVersion) >= 32) { auto conn(getConnection()); conn->to << WorkerProto::Op::AddMultipleToStore @@ -572,7 +578,7 @@ void RemoteStore::registerDrvOutput(const Realisation & info) { auto conn(getConnection()); conn->to << WorkerProto::Op::RegisterDrvOutput; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 31) { conn->to << info.id.to_string(); conn->to << std::string(info.outPath.to_string()); } else { @@ -587,7 +593,7 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id, try { auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 27) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 27) { warn("the daemon is too old to support content-addressed derivations, please upgrade it to 2.4"); return callback(nullptr); } @@ -597,7 +603,7 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id, conn.processStderr(); auto real = [&]() -> std::shared_ptr { - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 31) { auto outPaths = WorkerProto::Serialise>::read( *this, *conn); if (outPaths.empty()) @@ -644,9 +650,9 @@ void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMod auto conn(getConnection()); conn->to << WorkerProto::Op::BuildPaths; - assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13); + assert(GET_PROTOCOL_MINOR(conn->protoVersion) >= 13); WorkerProto::write(*this, *conn, drvPaths); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15) + if (GET_PROTOCOL_MINOR(conn->protoVersion) >= 15) conn->to << buildMode; else /* Old daemons did not take a 'buildMode' parameter, so we @@ -667,7 +673,7 @@ std::vector RemoteStore::buildPathsWithResults( std::optional conn_(getConnection()); auto & conn = *conn_; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 34) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) >= 34) { conn->to << WorkerProto::Op::BuildPathsWithResults; WorkerProto::write(*this, *conn, paths); conn->to << buildMode; @@ -841,7 +847,7 @@ void RemoteStore::queryMissing(const std::vector & targets, { { auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 19) + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 19) // Don't hold the connection handle in the fallback case // to prevent a deadlock. goto fallback; @@ -889,7 +895,7 @@ void RemoteStore::connect() unsigned int RemoteStore::getProtocol() { auto conn(connections->get()); - return conn->daemonVersion; + return conn->protoVersion; } std::optional RemoteStore::isTrustedClient() diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 27013c0f1..21175b1eb 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -1,5 +1,7 @@ #if ENABLE_S3 +#include + #include "s3.hh" #include "s3-binary-cache-store.hh" #include "nar-info.hh" @@ -58,7 +60,7 @@ class AwsLogger : public Aws::Utils::Logging::FormattedLogSystem debug("AWS: %s", chomp(statement)); } -#if !(AWS_VERSION_MAJOR <= 1 && AWS_VERSION_MINOR <= 7 && AWS_VERSION_PATCH <= 115) +#if !(AWS_SDK_VERSION_MAJOR <= 1 && AWS_SDK_VERSION_MINOR <= 7 && AWS_SDK_VERSION_PATCH <= 115) void Flush() override {} #endif }; @@ -101,7 +103,7 @@ S3Helper::S3Helper( std::make_shared(profile.c_str())), *config, // FIXME: https://github.com/aws/aws-sdk-cpp/issues/759 -#if AWS_VERSION_MAJOR == 1 && AWS_VERSION_MINOR < 3 +#if AWS_SDK_VERSION_MAJOR == 1 && AWS_SDK_VERSION_MINOR < 3 false, #else Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, @@ -190,81 +192,34 @@ S3BinaryCacheStore::S3BinaryCacheStore(const Params & params) , BinaryCacheStore(params) { } -struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig + +S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig( + std::string_view uriScheme, + std::string_view bucketName, + const Params & params) + : StoreConfig(params) + , BinaryCacheStoreConfig(params) + , bucketName(bucketName) { - using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + // Don't want to use use AWS SDK in header, so we check the default + // here. TODO do this better after we overhaul the store settings + // system. + assert(std::string{defaultRegion} == std::string{Aws::Region::US_EAST_1}); - const Setting profile{this, "", "profile", - R"( - The name of the AWS configuration profile to use. By default - Nix will use the `default` profile. - )"}; + if (bucketName.empty()) + throw UsageError("`%s` store requires a bucket name in its Store URI", uriScheme); +} - const Setting region{this, Aws::Region::US_EAST_1, "region", - R"( - The region of the S3 bucket. If your bucket is not in - `us–east-1`, you should always explicitly specify the region - parameter. - )"}; +std::string S3BinaryCacheStoreConfig::doc() +{ + return + #include "s3-binary-cache-store.md" + ; +} - const Setting scheme{this, "", "scheme", - R"( - The scheme used for S3 requests, `https` (default) or `http`. This - option allows you to disable HTTPS for binary caches which don't - support it. - - > **Note** - > - > HTTPS should be used if the cache might contain sensitive - > information. - )"}; - - const Setting endpoint{this, "", "endpoint", - R"( - The URL of the endpoint of an S3-compatible service such as MinIO. - Do not specify this setting if you're using Amazon S3. - - > **Note** - > - > This endpoint must support HTTPS and will use path-based - > addressing instead of virtual host based addressing. - )"}; - - const Setting narinfoCompression{this, "", "narinfo-compression", - "Compression method for `.narinfo` files."}; - - const Setting lsCompression{this, "", "ls-compression", - "Compression method for `.ls` files."}; - - const Setting logCompression{this, "", "log-compression", - R"( - Compression method for `log/*` files. It is recommended to - use a compression method supported by most web browsers - (e.g. `brotli`). - )"}; - - const Setting multipartUpload{ - this, false, "multipart-upload", - "Whether to use multi-part uploads."}; - - const Setting bufferSize{ - this, 5 * 1024 * 1024, "buffer-size", - "Size (in bytes) of each part in multi-part uploads."}; - - const std::string name() override { return "S3 Binary Cache Store"; } - - std::string doc() override - { - return - #include "s3-binary-cache-store.md" - ; - } -}; struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual S3BinaryCacheStore { - std::string bucketName; - Stats stats; S3Helper s3Helper; @@ -275,15 +230,12 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual const Params & params) : StoreConfig(params) , BinaryCacheStoreConfig(params) - , S3BinaryCacheStoreConfig(params) + , S3BinaryCacheStoreConfig(uriScheme, bucketName, params) , Store(params) , BinaryCacheStore(params) , S3BinaryCacheStore(params) - , bucketName(bucketName) , s3Helper(profile, region, scheme, endpoint) { - if (bucketName.empty()) - throw UsageError("`%s` store requires a bucket name in its Store URI", uriScheme); diskCache = getNarInfoDiskCache(); } @@ -521,9 +473,6 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual { return std::nullopt; } - - static std::set uriSchemes() { return {"s3"}; } - }; static RegisterStoreImplementation regS3BinaryCacheStore; diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh index c62ea5147..7d303a115 100644 --- a/src/libstore/s3-binary-cache-store.hh +++ b/src/libstore/s3-binary-cache-store.hh @@ -7,6 +7,101 @@ namespace nix { +struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +{ + std::string bucketName; + + using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + + S3BinaryCacheStoreConfig(std::string_view uriScheme, std::string_view bucketName, const Params & params); + + const Setting profile{ + this, + "", + "profile", + R"( + The name of the AWS configuration profile to use. By default + Nix will use the `default` profile. + )"}; + +protected: + + constexpr static const char * defaultRegion = "us-east-1"; + +public: + + const Setting region{ + this, + defaultRegion, + "region", + R"( + The region of the S3 bucket. If your bucket is not in + `us–east-1`, you should always explicitly specify the region + parameter. + )"}; + + const Setting scheme{ + this, + "", + "scheme", + R"( + The scheme used for S3 requests, `https` (default) or `http`. This + option allows you to disable HTTPS for binary caches which don't + support it. + + > **Note** + > + > HTTPS should be used if the cache might contain sensitive + > information. + )"}; + + const Setting endpoint{ + this, + "", + "endpoint", + R"( + The URL of the endpoint of an S3-compatible service such as MinIO. + Do not specify this setting if you're using Amazon S3. + + > **Note** + > + > This endpoint must support HTTPS and will use path-based + > addressing instead of virtual host based addressing. + )"}; + + const Setting narinfoCompression{ + this, "", "narinfo-compression", "Compression method for `.narinfo` files."}; + + const Setting lsCompression{this, "", "ls-compression", "Compression method for `.ls` files."}; + + const Setting logCompression{ + this, + "", + "log-compression", + R"( + Compression method for `log/*` files. It is recommended to + use a compression method supported by most web browsers + (e.g. `brotli`). + )"}; + + const Setting multipartUpload{this, false, "multipart-upload", "Whether to use multi-part uploads."}; + + const Setting bufferSize{ + this, 5 * 1024 * 1024, "buffer-size", "Size (in bytes) of each part in multi-part uploads."}; + + const std::string name() override + { + return "S3 Binary Cache Store"; + } + + static std::set uriSchemes() + { + return {"s3"}; + } + + std::string doc() override; +}; + class S3BinaryCacheStore : public virtual BinaryCacheStore { protected: diff --git a/src/libstore/s3.hh b/src/libstore/s3.hh index f0aeb3bed..18de115ae 100644 --- a/src/libstore/s3.hh +++ b/src/libstore/s3.hh @@ -8,7 +8,7 @@ #include #include -namespace Aws { namespace Client { class ClientConfiguration; } } +namespace Aws { namespace Client { struct ClientConfiguration; } } namespace Aws { namespace S3 { class S3Client; } } namespace nix { diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index b21c22d7e..954a97467 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -47,8 +47,6 @@ public: { } - static std::set uriSchemes() { return {"ssh-ng"}; } - std::string getUri() override { return *uriSchemes().begin() + "://" + host; @@ -89,43 +87,32 @@ protected: }; }; -struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfig + +MountedSSHStoreConfig::MountedSSHStoreConfig(StringMap params) + : StoreConfig(params) + , RemoteStoreConfig(params) + , CommonSSHStoreConfig(params) + , SSHStoreConfig(params) + , LocalFSStoreConfig(params) { - using SSHStoreConfig::SSHStoreConfig; - using LocalFSStoreConfig::LocalFSStoreConfig; +} - MountedSSHStoreConfig(StringMap params) - : StoreConfig(params) - , RemoteStoreConfig(params) - , CommonSSHStoreConfig(params) - , SSHStoreConfig(params) - , LocalFSStoreConfig(params) - { - } +MountedSSHStoreConfig::MountedSSHStoreConfig(std::string_view scheme, std::string_view host, StringMap params) + : StoreConfig(params) + , RemoteStoreConfig(params) + , CommonSSHStoreConfig(scheme, host, params) + , SSHStoreConfig(params) + , LocalFSStoreConfig(params) +{ +} - MountedSSHStoreConfig(std::string_view scheme, std::string_view host, StringMap params) - : StoreConfig(params) - , RemoteStoreConfig(params) - , CommonSSHStoreConfig(scheme, host, params) - , SSHStoreConfig(params) - , LocalFSStoreConfig(params) - { - } +std::string MountedSSHStoreConfig::doc() +{ + return + #include "mounted-ssh-store.md" + ; +} - const std::string name() override { return "Experimental SSH Store with filesystem mounted"; } - - std::string doc() override - { - return - #include "mounted-ssh-store.md" - ; - } - - std::optional experimentalFeature() const override - { - return ExperimentalFeature::MountedSSHStore; - } -}; /** * The mounted ssh store assumes that filesystems on the remote host are @@ -165,11 +152,6 @@ public: }; } - static std::set uriSchemes() - { - return {"mounted-ssh-ng"}; - } - std::string getUri() override { return *uriSchemes().begin() + "://" + host; diff --git a/src/libstore/ssh-store.hh b/src/libstore/ssh-store.hh index 6ef2219a2..29a2a8b2c 100644 --- a/src/libstore/ssh-store.hh +++ b/src/libstore/ssh-store.hh @@ -3,6 +3,7 @@ #include "common-ssh-store-config.hh" #include "store-api.hh" +#include "local-fs-store.hh" #include "remote-store.hh" namespace nix { @@ -22,7 +23,39 @@ struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig return "Experimental SSH Store"; } + static std::set uriSchemes() + { + return {"ssh-ng"}; + } + std::string doc() override; }; +struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfig +{ + using LocalFSStoreConfig::LocalFSStoreConfig; + using SSHStoreConfig::SSHStoreConfig; + + MountedSSHStoreConfig(StringMap params); + + MountedSSHStoreConfig(std::string_view scheme, std::string_view host, StringMap params); + + const std::string name() override + { + return "Experimental SSH Store with filesystem mounted"; + } + + static std::set uriSchemes() + { + return {"mounted-ssh-ng"}; + } + + std::string doc() override; + + std::optional experimentalFeature() const override + { + return ExperimentalFeature::MountedSSHStore; + } +}; + } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 2c4dee518..b3e5ad014 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -920,7 +920,7 @@ StorePathSet Store::exportReferences(const StorePathSet & storePaths, const Stor const Store::Stats & Store::getStats() { { - auto state_(state.lock()); + auto state_(state.readLock()); stats.pathInfoCacheSize = state_->pathInfoCache.size(); } return stats; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 749d7ea09..8288cfdf0 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -201,7 +201,7 @@ protected: LRUCache pathInfoCache; }; - Sync state; + SharedSync state; std::shared_ptr diskCache; @@ -216,6 +216,10 @@ public: virtual ~Store() { } + /** + * @todo move to `StoreConfig` one we store enough information in + * those to recover the scheme and authority in all cases. + */ virtual std::string getUri() = 0; /** @@ -897,7 +901,7 @@ struct Implementations { if (!registered) registered = new std::vector(); StoreFactory factory{ - .uriSchemes = T::uriSchemes(), + .uriSchemes = TConfig::uriSchemes(), .create = ([](auto scheme, auto uri, auto & params) -> std::shared_ptr diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 499f76967..3c445eb13 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -2,10 +2,8 @@ #include "unix-domain-socket.hh" #include "worker-protocol.hh" -#include #include #include -#include #include #include @@ -19,6 +17,21 @@ namespace nix { +UDSRemoteStoreConfig::UDSRemoteStoreConfig( + std::string_view scheme, + std::string_view authority, + const Params & params) + : StoreConfig(params) + , LocalFSStoreConfig(params) + , RemoteStoreConfig(params) + , path{authority.empty() ? settings.nixDaemonSocketFile : authority} +{ + if (scheme != UDSRemoteStoreConfig::scheme) { + throw UsageError("Scheme must be 'unix'"); + } +} + + std::string UDSRemoteStoreConfig::doc() { return @@ -27,11 +40,20 @@ std::string UDSRemoteStoreConfig::doc() } +// A bit gross that we now pass empty string but this is knowing that +// empty string will later default to the same nixDaemonSocketFile. Why +// don't we just wire it all through? I believe there are cases where it +// will live reload so we want to continue to account for that. UDSRemoteStore::UDSRemoteStore(const Params & params) + : UDSRemoteStore(scheme, "", params) +{} + + +UDSRemoteStore::UDSRemoteStore(std::string_view scheme, std::string_view authority, const Params & params) : StoreConfig(params) , LocalFSStoreConfig(params) , RemoteStoreConfig(params) - , UDSRemoteStoreConfig(params) + , UDSRemoteStoreConfig(scheme, authority, params) , Store(params) , LocalFSStore(params) , RemoteStore(params) @@ -39,25 +61,15 @@ UDSRemoteStore::UDSRemoteStore(const Params & params) } -UDSRemoteStore::UDSRemoteStore( - std::string_view scheme, - PathView socket_path, - const Params & params) - : UDSRemoteStore(params) -{ - if (!socket_path.empty()) - path.emplace(socket_path); -} - - std::string UDSRemoteStore::getUri() { - if (path) { - return std::string("unix://") + *path; - } else { - // unix:// with no path also works. Change what we return? - return "daemon"; - } + return path == settings.nixDaemonSocketFile + ? // FIXME: Not clear why we return daemon here and not default + // to settings.nixDaemonSocketFile + // + // unix:// with no path also works. Change what we return? + "daemon" + : std::string(scheme) + "://" + path; } @@ -74,7 +86,7 @@ ref UDSRemoteStore::openConnection() /* Connect to a daemon that does the privileged work for us. */ conn->fd = createUnixDomainSocket(); - nix::connect(toSocket(conn->fd.get()), path ? *path : settings.nixDaemonSocketFile); + nix::connect(toSocket(conn->fd.get()), path); conn->from.fd = conn->fd.get(); conn->to.fd = conn->fd.get(); diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index 6f0494bb6..a8e571664 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -9,16 +9,37 @@ namespace nix { struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig { - UDSRemoteStoreConfig(const Params & params) - : StoreConfig(params) - , LocalFSStoreConfig(params) - , RemoteStoreConfig(params) - { - } + // TODO(fzakaria): Delete this constructor once moved over to the factory pattern + // outlined in https://github.com/NixOS/nix/issues/10766 + using LocalFSStoreConfig::LocalFSStoreConfig; + using RemoteStoreConfig::RemoteStoreConfig; + + /** + * @param authority is the socket path. + */ + UDSRemoteStoreConfig( + std::string_view scheme, + std::string_view authority, + const Params & params); const std::string name() override { return "Local Daemon Store"; } std::string doc() override; + + /** + * The path to the unix domain socket. + * + * The default is `settings.nixDaemonSocketFile`, but we don't write + * that below, instead putting in the constructor. + */ + Path path; + +protected: + static constexpr char const * scheme = "unix"; + +public: + static std::set uriSchemes() + { return {scheme}; } }; class UDSRemoteStore : public virtual UDSRemoteStoreConfig @@ -27,17 +48,21 @@ class UDSRemoteStore : public virtual UDSRemoteStoreConfig { public: + /** + * @deprecated This is the old API to construct the store. + */ UDSRemoteStore(const Params & params); + + /** + * @param authority is the socket path. + */ UDSRemoteStore( std::string_view scheme, - PathView path, + std::string_view authority, const Params & params); std::string getUri() override; - static std::set uriSchemes() - { return {"unix"}; } - ref getFSAccessor(bool requireValidPath = true) override { return LocalFSStore::getFSAccessor(requireValidPath); } @@ -63,7 +88,6 @@ private: }; ref openConnection() override; - std::optional path; }; } diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index f968bbc5b..d3482df17 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -165,7 +165,7 @@ void LocalDerivationGoal::killSandbox(bool getStats) buildResult.cpuSystem = stats.cpuSystem; } #else - abort(); + unreachable(); #endif } @@ -1258,7 +1258,7 @@ bool LocalDerivationGoal::isAllowed(const DerivedPath & req) struct RestrictedStoreConfig : virtual LocalFSStoreConfig { using LocalFSStoreConfig::LocalFSStoreConfig; - const std::string name() { return "Restricted Store"; } + const std::string name() override { return "Restricted Store"; } }; /* A wrapper around LocalStore that only allows building/querying of @@ -1526,10 +1526,11 @@ void LocalDerivationGoal::startDaemon() debug("received daemon connection"); auto workerThread = std::thread([store, remote{std::move(remote)}]() { - FdSource from(remote.get()); - FdSink to(remote.get()); try { - daemon::processConnection(store, from, to, + daemon::processConnection( + store, + FdSource(remote.get()), + FdSink(remote.get()), NotTrusted, daemon::Recursive); debug("terminated daemon connection"); } catch (SystemError &) { @@ -1701,10 +1702,13 @@ void setupSeccomp() throw SysError("unable to add seccomp rule"); } - /* Prevent builders from creating EAs or ACLs. Not all filesystems + /* Prevent builders from using EAs or ACLs. Not all filesystems support these, and they're not allowed in the Nix store because they're not representable in the NAR serialisation. */ - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 || + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(getxattr), 0) != 0 || + seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lgetxattr), 0) != 0 || + seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fgetxattr), 0) != 0 || + seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 || seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lsetxattr), 0) != 0 || seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0) throw SysError("unable to add seccomp rule"); diff --git a/src/libstore/unix/pathlocks.cc b/src/libstore/unix/pathlocks.cc index af21319a7..1ec4579ec 100644 --- a/src/libstore/unix/pathlocks.cc +++ b/src/libstore/unix/pathlocks.cc @@ -45,7 +45,7 @@ bool lockFile(Descriptor desc, LockType lockType, bool wait) if (lockType == ltRead) type = LOCK_SH; else if (lockType == ltWrite) type = LOCK_EX; else if (lockType == ltNone) type = LOCK_UN; - else abort(); + else unreachable(); if (wait) { while (flock(desc, type) != 0) { diff --git a/src/libstore/worker-protocol-connection.cc b/src/libstore/worker-protocol-connection.cc index 072bae8da..a47dbb689 100644 --- a/src/libstore/worker-protocol-connection.cc +++ b/src/libstore/worker-protocol-connection.cc @@ -5,6 +5,8 @@ namespace nix { +const std::set WorkerProto::allFeatures{}; + WorkerProto::BasicClientConnection::~BasicClientConnection() { try { @@ -58,7 +60,7 @@ std::exception_ptr WorkerProto::BasicClientConnection::processStderrReturn(Sink } else if (msg == STDERR_ERROR) { - if (GET_PROTOCOL_MINOR(daemonVersion) >= 26) { + if (GET_PROTOCOL_MINOR(protoVersion) >= 26) { ex = std::make_exception_ptr(readError(from)); } else { auto error = readString(from); @@ -114,7 +116,7 @@ std::exception_ptr WorkerProto::BasicClientConnection::processStderrReturn(Sink // explain to users what's going on when their daemon is // older than #4628 (2023). if (experimentalFeatureSettings.isEnabled(Xp::DynamicDerivations) - && GET_PROTOCOL_MINOR(daemonVersion) <= 35) { + && GET_PROTOCOL_MINOR(protoVersion) <= 35) { auto m = e.msg(); if (m.find("parsing derivation") != std::string::npos && m.find("expected string") != std::string::npos && m.find("Derive([") != std::string::npos) @@ -137,8 +139,21 @@ void WorkerProto::BasicClientConnection::processStderr(bool * daemonException, S } } -WorkerProto::Version -WorkerProto::BasicClientConnection::handshake(BufferedSink & to, Source & from, WorkerProto::Version localVersion) +static std::set +intersectFeatures(const std::set & a, const std::set & b) +{ + std::set res; + for (auto & x : a) + if (b.contains(x)) + res.insert(x); + return res; +} + +std::tuple> WorkerProto::BasicClientConnection::handshake( + BufferedSink & to, + Source & from, + WorkerProto::Version localVersion, + const std::set & supportedFeatures) { to << WORKER_MAGIC_1 << localVersion; to.flush(); @@ -152,13 +167,25 @@ WorkerProto::BasicClientConnection::handshake(BufferedSink & to, Source & from, throw Error("Nix daemon protocol version not supported"); if (GET_PROTOCOL_MINOR(daemonVersion) < 10) throw Error("the Nix daemon version is too old"); - to << localVersion; - return std::min(daemonVersion, localVersion); + auto protoVersion = std::min(daemonVersion, localVersion); + + /* Exchange features. */ + std::set daemonFeatures; + if (GET_PROTOCOL_MINOR(protoVersion) >= 38) { + to << supportedFeatures; + to.flush(); + daemonFeatures = readStrings>(from); + } + + return {protoVersion, intersectFeatures(daemonFeatures, supportedFeatures)}; } -WorkerProto::Version -WorkerProto::BasicServerConnection::handshake(BufferedSink & to, Source & from, WorkerProto::Version localVersion) +std::tuple> WorkerProto::BasicServerConnection::handshake( + BufferedSink & to, + Source & from, + WorkerProto::Version localVersion, + const std::set & supportedFeatures) { unsigned int magic = readInt(from); if (magic != WORKER_MAGIC_1) @@ -166,22 +193,33 @@ WorkerProto::BasicServerConnection::handshake(BufferedSink & to, Source & from, to << WORKER_MAGIC_2 << localVersion; to.flush(); auto clientVersion = readInt(from); - return std::min(clientVersion, localVersion); + + auto protoVersion = std::min(clientVersion, localVersion); + + /* Exchange features. */ + std::set clientFeatures; + if (GET_PROTOCOL_MINOR(protoVersion) >= 38) { + clientFeatures = readStrings>(from); + to << supportedFeatures; + to.flush(); + } + + return {protoVersion, intersectFeatures(clientFeatures, supportedFeatures)}; } WorkerProto::ClientHandshakeInfo WorkerProto::BasicClientConnection::postHandshake(const StoreDirConfig & store) { WorkerProto::ClientHandshakeInfo res; - if (GET_PROTOCOL_MINOR(daemonVersion) >= 14) { + if (GET_PROTOCOL_MINOR(protoVersion) >= 14) { // Obsolete CPU affinity. to << 0; } - if (GET_PROTOCOL_MINOR(daemonVersion) >= 11) + if (GET_PROTOCOL_MINOR(protoVersion) >= 11) to << false; // obsolete reserveSpace - if (GET_PROTOCOL_MINOR(daemonVersion) >= 33) + if (GET_PROTOCOL_MINOR(protoVersion) >= 33) to.flush(); return WorkerProto::Serialise::read(store, *this); @@ -189,12 +227,12 @@ WorkerProto::ClientHandshakeInfo WorkerProto::BasicClientConnection::postHandsha void WorkerProto::BasicServerConnection::postHandshake(const StoreDirConfig & store, const ClientHandshakeInfo & info) { - if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) { + if (GET_PROTOCOL_MINOR(protoVersion) >= 14 && readInt(from)) { // Obsolete CPU affinity. readInt(from); } - if (GET_PROTOCOL_MINOR(clientVersion) >= 11) + if (GET_PROTOCOL_MINOR(protoVersion) >= 11) readInt(from); // obsolete reserveSpace WorkerProto::write(store, *this, info); @@ -212,7 +250,7 @@ UnkeyedValidPathInfo WorkerProto::BasicClientConnection::queryPathInfo( throw InvalidPath(std::move(e.info())); throw; } - if (GET_PROTOCOL_MINOR(daemonVersion) >= 17) { + if (GET_PROTOCOL_MINOR(protoVersion) >= 17) { bool valid; from >> valid; if (!valid) @@ -224,10 +262,10 @@ UnkeyedValidPathInfo WorkerProto::BasicClientConnection::queryPathInfo( StorePathSet WorkerProto::BasicClientConnection::queryValidPaths( const StoreDirConfig & store, bool * daemonException, const StorePathSet & paths, SubstituteFlag maybeSubstitute) { - assert(GET_PROTOCOL_MINOR(daemonVersion) >= 12); + assert(GET_PROTOCOL_MINOR(protoVersion) >= 12); to << WorkerProto::Op::QueryValidPaths; WorkerProto::write(store, *this, paths); - if (GET_PROTOCOL_MINOR(daemonVersion) >= 27) { + if (GET_PROTOCOL_MINOR(protoVersion) >= 27) { to << maybeSubstitute; } processStderr(daemonException); diff --git a/src/libstore/worker-protocol-connection.hh b/src/libstore/worker-protocol-connection.hh index 9dd723fd0..9c96195b5 100644 --- a/src/libstore/worker-protocol-connection.hh +++ b/src/libstore/worker-protocol-connection.hh @@ -6,7 +6,7 @@ namespace nix { -struct WorkerProto::BasicClientConnection +struct WorkerProto::BasicConnection { /** * Send with this. @@ -19,46 +19,14 @@ struct WorkerProto::BasicClientConnection FdSource from; /** - * Worker protocol version used for the connection. - * - * Despite its name, it is actually the maximum version both - * sides support. (If the maximum doesn't exist, we would fail to - * establish a connection and produce a value of this type.) + * The protocol version agreed by both sides. */ - WorkerProto::Version daemonVersion; + WorkerProto::Version protoVersion; /** - * Flush to direction + * The set of features that both sides support. */ - virtual ~BasicClientConnection(); - - virtual void closeWrite() = 0; - - std::exception_ptr processStderrReturn(Sink * sink = 0, Source * source = 0, bool flush = true); - - void processStderr(bool * daemonException, Sink * sink = 0, Source * source = 0, bool flush = true); - - /** - * Establishes connection, negotiating version. - * - * @return the version provided by the other side of the - * connection. - * - * @param to Taken by reference to allow for various error handling - * mechanisms. - * - * @param from Taken by reference to allow for various error - * handling mechanisms. - * - * @param localVersion Our version which is sent over - */ - static Version handshake(BufferedSink & to, Source & from, WorkerProto::Version localVersion); - - /** - * After calling handshake, must call this to exchange some basic - * information abou the connection. - */ - ClientHandshakeInfo postHandshake(const StoreDirConfig & store); + std::set features; /** * Coercion to `WorkerProto::ReadConn`. This makes it easy to use the @@ -72,7 +40,7 @@ struct WorkerProto::BasicClientConnection { return WorkerProto::ReadConn{ .from = from, - .version = daemonVersion, + .version = protoVersion, }; } @@ -88,9 +56,52 @@ struct WorkerProto::BasicClientConnection { return WorkerProto::WriteConn{ .to = to, - .version = daemonVersion, + .version = protoVersion, }; } +}; + +struct WorkerProto::BasicClientConnection : WorkerProto::BasicConnection +{ + /** + * Flush to direction + */ + virtual ~BasicClientConnection(); + + virtual void closeWrite() = 0; + + std::exception_ptr processStderrReturn(Sink * sink = 0, Source * source = 0, bool flush = true); + + void processStderr(bool * daemonException, Sink * sink = 0, Source * source = 0, bool flush = true); + + /** + * Establishes connection, negotiating version. + * + * @return the minimum version supported by both sides and the set + * of protocol features supported by both sides. + * + * @param to Taken by reference to allow for various error handling + * mechanisms. + * + * @param from Taken by reference to allow for various error + * handling mechanisms. + * + * @param localVersion Our version which is sent over + * + * @param features The protocol features that we support + */ + // FIXME: this should probably be a constructor. + static std::tuple> handshake( + BufferedSink & to, + Source & from, + WorkerProto::Version localVersion, + const std::set & supportedFeatures); + + /** + * After calling handshake, must call this to exchange some basic + * information abou the connection. + */ + ClientHandshakeInfo postHandshake(const StoreDirConfig & store); void addTempRoot(const StoreDirConfig & remoteStore, bool * daemonException, const StorePath & path); @@ -124,43 +135,8 @@ struct WorkerProto::BasicClientConnection void importPaths(const StoreDirConfig & store, bool * daemonException, Source & source); }; -struct WorkerProto::BasicServerConnection +struct WorkerProto::BasicServerConnection : WorkerProto::BasicConnection { - /** - * Send with this. - */ - FdSink & to; - - /** - * Receive with this. - */ - FdSource & from; - - /** - * Worker protocol version used for the connection. - * - * Despite its name, it is actually the maximum version both - * sides support. (If the maximum doesn't exist, we would fail to - * establish a connection and produce a value of this type.) - */ - WorkerProto::Version clientVersion; - - operator WorkerProto::ReadConn() - { - return WorkerProto::ReadConn{ - .from = from, - .version = clientVersion, - }; - } - - operator WorkerProto::WriteConn() - { - return WorkerProto::WriteConn{ - .to = to, - .version = clientVersion, - }; - } - /** * Establishes connection, negotiating version. * @@ -174,8 +150,15 @@ struct WorkerProto::BasicServerConnection * handling mechanisms. * * @param localVersion Our version which is sent over + * + * @param features The protocol features that we support */ - static WorkerProto::Version handshake(BufferedSink & to, Source & from, WorkerProto::Version localVersion); + // FIXME: this should probably be a constructor. + static std::tuple> handshake( + BufferedSink & to, + Source & from, + WorkerProto::Version localVersion, + const std::set & supportedFeatures); /** * After calling handshake, must call this to exchange some basic diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 62a12d182..c356fa1bf 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -11,7 +11,9 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION (1 << 8 | 37) +/* Note: you generally shouldn't change the protocol version. Define a + new `WorkerProto::Feature` instead. */ +#define PROTOCOL_VERSION (1 << 8 | 38) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -82,6 +84,7 @@ struct WorkerProto * * @todo remove once Hydra uses Store abstraction consistently. */ + struct BasicConnection; struct BasicClientConnection; struct BasicServerConnection; @@ -130,6 +133,10 @@ struct WorkerProto { WorkerProto::Serialise::write(store, conn, t); } + + using Feature = std::string; + + static const std::set allFeatures; }; enum struct WorkerProto::Op : uint64_t @@ -183,8 +190,7 @@ enum struct WorkerProto::Op : uint64_t struct WorkerProto::ClientHandshakeInfo { /** - * The version of the Nix daemon that is processing our requests -. + * The version of the Nix daemon that is processing our requests. * * Do note, it may or may not communicating with another daemon, * rather than being an "end" `LocalStore` or similar. diff --git a/src/libutil-c/meson.build b/src/libutil-c/meson.build index 3f0d96282..b5ed19631 100644 --- a/src/libutil-c/meson.build +++ b/src/libutil-c/meson.build @@ -25,6 +25,8 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') +subdir('build-utils-meson/threads') + # TODO rename, because it will conflict with downstream projects configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) diff --git a/src/libutil-c/nix_api_util.h b/src/libutil-c/nix_api_util.h index e0ca04e69..ad6f32859 100644 --- a/src/libutil-c/nix_api_util.h +++ b/src/libutil-c/nix_api_util.h @@ -56,47 +56,51 @@ extern "C" { * - NIX_ERR_KEY: A key error occurred (-3) * - NIX_ERR_NIX_ERROR: A generic Nix error occurred (-4) */ -typedef int nix_err; +enum nix_err { -/** - * @brief No error occurred. - * - * This error code is returned when no error has occurred during the function - * execution. - */ -#define NIX_OK 0 + /** + * @brief No error occurred. + * + * This error code is returned when no error has occurred during the function + * execution. + */ + NIX_OK = 0, -/** - * @brief An unknown error occurred. - * - * This error code is returned when an unknown error occurred during the - * function execution. - */ -#define NIX_ERR_UNKNOWN -1 + /** + * @brief An unknown error occurred. + * + * This error code is returned when an unknown error occurred during the + * function execution. + */ + NIX_ERR_UNKNOWN = -1, -/** - * @brief An overflow error occurred. - * - * This error code is returned when an overflow error occurred during the - * function execution. - */ -#define NIX_ERR_OVERFLOW -2 + /** + * @brief An overflow error occurred. + * + * This error code is returned when an overflow error occurred during the + * function execution. + */ + NIX_ERR_OVERFLOW = -2, -/** - * @brief A key error occurred. - * - * This error code is returned when a key error occurred during the function - * execution. - */ -#define NIX_ERR_KEY -3 + /** + * @brief A key error occurred. + * + * This error code is returned when a key error occurred during the function + * execution. + */ + NIX_ERR_KEY = -3, -/** - * @brief A generic Nix error occurred. - * - * This error code is returned when a generic Nix error occurred during the - * function execution. - */ -#define NIX_ERR_NIX_ERROR -4 + /** + * @brief A generic Nix error occurred. + * + * This error code is returned when a generic Nix error occurred during the + * function execution. + */ + NIX_ERR_NIX_ERROR = -4, + +}; + +typedef enum nix_err nix_err; /** * @brief This object stores error state. diff --git a/src/libutil-c/package.nix b/src/libutil-c/package.nix index 53451998d..ccfafd4d3 100644 --- a/src/libutil-c/package.nix +++ b/src/libutil-c/package.nix @@ -62,12 +62,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libutil/args/root.hh b/src/libutil/args/root.hh index 5c55c37a5..34a43b538 100644 --- a/src/libutil/args/root.hh +++ b/src/libutil/args/root.hh @@ -29,6 +29,7 @@ struct Completions final : AddCompletions */ class RootArgs : virtual public Args { +protected: /** * @brief The command's "working directory", but only set when top level. * diff --git a/src/libutil/chunked-vector.hh b/src/libutil/chunked-vector.hh index d914e2542..4709679a6 100644 --- a/src/libutil/chunked-vector.hh +++ b/src/libutil/chunked-vector.hh @@ -6,6 +6,8 @@ #include #include +#include "error.hh" + namespace nix { /** @@ -30,7 +32,7 @@ private: auto & addChunk() { if (size_ >= std::numeric_limits::max() - ChunkSize) - abort(); + unreachable(); chunks.emplace_back(); chunks.back().reserve(ChunkSize); return chunks.back(); diff --git a/src/libutil/config-impl.hh b/src/libutil/config-impl.hh index 1d349fab5..c3aa61ddb 100644 --- a/src/libutil/config-impl.hh +++ b/src/libutil/config-impl.hh @@ -81,6 +81,7 @@ void BaseSetting::convertToArg(Args & args, const std::string & category) { args.addFlag({ .longName = name, + .aliases = aliases, .description = fmt("Set the `%s` setting.", name), .category = category, .labels = {"value"}, @@ -91,6 +92,7 @@ void BaseSetting::convertToArg(Args & args, const std::string & category) if (isAppendable()) args.addFlag({ .longName = "extra-" + name, + .aliases = aliases, .description = fmt("Append to the `%s` setting.", name), .category = category, .labels = {"value"}, diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 89ac46d85..c9ced6b2b 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -1,6 +1,7 @@ #include "config.hh" #include "args.hh" #include "abstract-setting-to-json.hh" +#include "environment-variables.hh" #include "experimental-features.hh" #include "util.hh" #include "file-system.hh" @@ -170,9 +171,18 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string set(name, value); // Then apply other settings - for (const auto & [name, value] : parsedContents) - if (name != "experimental-features" && name != "extra-experimental-features") + // XXX: NIX_PATH must override the regular setting! This is done in `initGC()` + // Environment variables overriding settings should probably be part of the Config mechanism, + // but at the time of writing it's not worth building that for just one thing + for (const auto & [name, value] : parsedContents) { + if (name != "experimental-features" && name != "extra-experimental-features") { + if ((name == "nix-path" || name == "extra-nix-path") + && getEnv("NIX_PATH").has_value()) { + continue; + } set(name, value); + } + } } void Config::resetOverridden() @@ -292,6 +302,7 @@ template<> void BaseSetting::convertToArg(Args & args, const std::string & { args.addFlag({ .longName = name, + .aliases = aliases, .description = fmt("Enable the `%s` setting.", name), .category = category, .handler = {[this] { override(true); }}, @@ -299,6 +310,7 @@ template<> void BaseSetting::convertToArg(Args & args, const std::string & }); args.addFlag({ .longName = "no-" + name, + .aliases = aliases, .description = fmt("Disable the `%s` setting.", name), .category = category, .handler = {[this] { override(false); }}, diff --git a/src/libutil/config.hh b/src/libutil/config.hh index f1c7233ba..bcca030ac 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -393,7 +393,7 @@ struct ExperimentalFeatureSettings : Config { {{#include experimental-features-shortlist.md}} - Experimental features are [further documented in the manual](@docroot@/contributing/experimental-features.md). + Experimental features are [further documented in the manual](@docroot@/development/experimental-features.md). )"}; /** diff --git a/src/libutil/current-process.cc b/src/libutil/current-process.cc index 6ca48220d..0bc46d746 100644 --- a/src/libutil/current-process.cc +++ b/src/libutil/current-process.cc @@ -15,13 +15,12 @@ #if __linux__ # include -# include # include "cgroup.hh" # include "namespaces.hh" #endif #ifndef _WIN32 -# include +# include #endif namespace nix { @@ -138,7 +137,7 @@ std::optional getSelfExe() { static auto cached = []() -> std::optional { - #if __linux__ + #if __linux__ || __GNU__ return readLink("/proc/self/exe"); #elif __APPLE__ char buf[1024]; diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 33c391963..ccd008c7c 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -1,3 +1,5 @@ +#include + #include "error.hh" #include "environment-variables.hh" #include "signals.hh" @@ -430,4 +432,36 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s return out; } +/** Write to stderr in a robust and minimal way, considering that the process + * may be in a bad state. + */ +static void writeErr(std::string_view buf) +{ + while (!buf.empty()) { + auto n = write(STDERR_FILENO, buf.data(), buf.size()); + if (n < 0) { + if (errno == EINTR) continue; + abort(); + } + buf = buf.substr(n); + } +} + +void panic(std::string_view msg) +{ + writeErr("\n\n" ANSI_RED "terminating due to unexpected unrecoverable internal error: " ANSI_NORMAL ); + writeErr(msg); + writeErr("\n"); + abort(); +} + +void panic(const char * file, int line, const char * func) +{ + char buf[512]; + int n = snprintf(buf, sizeof(buf), "Unexpected condition in %s at %s:%d", func, file, line); + if (n < 0) + panic("Unexpected condition and could not format error message"); + panic(std::string_view(buf, std::min(static_cast(sizeof(buf)), n))); +} + } diff --git a/src/libutil/error.hh b/src/libutil/error.hh index d7fe902d6..58d902622 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -273,4 +273,24 @@ using NativeSysError = */ void throwExceptionSelfCheck(); +/** + * Print a message and abort(). + */ +[[noreturn]] +void panic(std::string_view msg); + +/** + * Print a basic error message with source position and abort(). + * Use the unreachable() macro to call this. + */ +[[noreturn]] +void panic(const char * file, int line, const char * func); + +/** + * Print a basic error message with source position and abort(). + * + * @note: This assumes that the logger is operational + */ +#define unreachable() (::nix::panic(__FILE__, __LINE__, __func__)) + } diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index c69f84685..df70adaa6 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -24,7 +24,7 @@ struct ExperimentalFeatureDetails * feature, we either have no issue at all if few features are not added * at the end of the list, or a proper merge conflict if they are. */ -constexpr size_t numXpFeatures = 1 + static_cast(Xp::VerifiedFetches); +constexpr size_t numXpFeatures = 1 + static_cast(Xp::PipeOperators); constexpr std::array xpFeatureDetails = {{ { @@ -278,6 +278,14 @@ constexpr std::array xpFeatureDetails )", .trackingUrl = "https://github.com/NixOS/nix/milestone/48", }, + { + .tag = Xp::PipeOperators, + .name = "pipe-operators", + .description = R"( + Add `|>` and `<|` operators to the Nix language. + )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/55", + }, }}; static_assert( diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index 9e3fce56c..6dda2d6f3 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -33,6 +33,7 @@ enum struct ExperimentalFeature ConfigurableImpureEnv, MountedSSHStore, VerifiedFetches, + PipeOperators, }; extern std::set stabilizedFeatures; diff --git a/src/libutil/file-content-address.cc b/src/libutil/file-content-address.cc index 438dac7da..86378dd67 100644 --- a/src/libutil/file-content-address.cc +++ b/src/libutil/file-content-address.cc @@ -63,7 +63,7 @@ std::string_view renderFileIngestionMethod(FileIngestionMethod method) case FileIngestionMethod::Git: return "git"; default: - abort(); + unreachable(); } } diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 9042e3a5e..060a806fb 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -416,7 +416,11 @@ void deletePath(const fs::path & path) void createDir(const Path & path, mode_t mode) { - if (mkdir(path.c_str(), mode) == -1) + if (mkdir(path.c_str() +#ifndef _WIN32 + , mode +#endif + ) == -1) throw SysError("creating directory '%1%'", path); } diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc index 194e86fdd..f15324d0a 100644 --- a/src/libutil/fs-sink.cc +++ b/src/libutil/fs-sink.cc @@ -53,7 +53,7 @@ void copyRecursive( throw Error("file '%1%' has an unsupported type", from); default: - abort(); + unreachable(); } } @@ -97,7 +97,7 @@ void RestoreSink::createRegularFile(const CanonPath & path, std::function convertMode(SourceAccessor::Type type) case SourceAccessor::tRegular: return Mode::Regular; case SourceAccessor::tDirectory: return Mode::Directory; case SourceAccessor::tMisc: return std::nullopt; - default: abort(); + default: unreachable(); } } diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 35b913e42..ab2a8695d 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -25,7 +25,7 @@ static size_t regularHashSize(HashAlgorithm type) { case HashAlgorithm::SHA256: return sha256HashSize; case HashAlgorithm::SHA512: return sha512HashSize; } - abort(); + unreachable(); } diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 55751b4cf..29427f2f6 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -189,7 +189,7 @@ struct JSONLogger : Logger { else if (f.type == Logger::Field::tString) arr.push_back(f.s); else - abort(); + unreachable(); } void write(const nlohmann::json & json) diff --git a/src/libutil/lru-cache.hh b/src/libutil/lru-cache.hh index 0e19517ed..6e14cac35 100644 --- a/src/libutil/lru-cache.hh +++ b/src/libutil/lru-cache.hh @@ -89,7 +89,7 @@ public: return i->second.second; } - size_t size() + size_t size() const { return data.size(); } diff --git a/src/libutil/meson.build b/src/libutil/meson.build index fbfcbe67c..8552c4c9d 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -62,6 +62,7 @@ endif boost = dependency( 'boost', modules : ['context', 'coroutine'], + include_type: 'system', ) # boost is a public dependency, but not a pkg-config dependency unfortunately, so we # put in `deps_other`. @@ -216,6 +217,7 @@ headers = [config_h] + files( 'source-accessor.hh', 'source-path.hh', 'split.hh', + 'std-hash.hh', 'strings.hh', 'strings-inline.hh', 'suggestions.hh', diff --git a/src/libutil/package.nix b/src/libutil/package.nix index 28d7d8f0e..4ce1a75b0 100644 --- a/src/libutil/package.nix +++ b/src/libutil/package.nix @@ -88,12 +88,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libutil/position.cc b/src/libutil/position.cc index 3289dbe8b..5a2529262 100644 --- a/src/libutil/position.cc +++ b/src/libutil/position.cc @@ -119,12 +119,12 @@ std::optional Pos::getSnippetUpTo(const Pos & end) const { if (auto source = getSource()) { auto firstLine = LinesIterator(*source); - for (auto i = 1; i < this->line; ++i) { + for (uint32_t i = 1; i < this->line; ++i) { ++firstLine; } auto lastLine = LinesIterator(*source); - for (auto i = 1; i < end.line; ++i) { + for (uint32_t i = 1; i < end.line; ++i) { ++lastLine; } diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 35047f89e..2b1a485d5 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -97,7 +97,7 @@ std::optional PosixSourceAccessor::cachedLstat(const CanonPath & pa Path absPath = makeAbsPath(path).string(); { - auto cache(_cache.read()); + auto cache(_cache.readLock()); auto i = cache->find(absPath); if (i != cache->end()) return i->second; } diff --git a/src/libutil/ref.hh b/src/libutil/ref.hh index 016fdd74a..3d0e64ab4 100644 --- a/src/libutil/ref.hh +++ b/src/libutil/ref.hh @@ -23,14 +23,14 @@ public: : p(r.p) { } - explicit ref(const std::shared_ptr & p) + explicit ref(const std::shared_ptr & p) : p(p) { if (!p) throw std::invalid_argument("null pointer cast to ref"); } - explicit ref(T * p) + explicit ref(T * p) : p(p) { if (!p) diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 36b99905a..4899134d7 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -190,11 +190,11 @@ struct VirtualStackAllocator { class DefaultStackAllocator : public StackAllocator { boost::coroutines2::default_stack stack; - boost::context::stack_context allocate() { + boost::context::stack_context allocate() override { return stack.allocate(); } - void deallocate(boost::context::stack_context sctx) { + void deallocate(boost::context::stack_context sctx) override { stack.deallocate(sctx); } }; @@ -260,7 +260,7 @@ std::unique_ptr sourceToSink(std::function fun) }); } - if (!*coro) { abort(); } + if (!*coro) { unreachable(); } if (!cur.empty()) { CoroutineContext ctx; @@ -271,12 +271,12 @@ std::unique_ptr sourceToSink(std::function fun) void finish() override { if (!coro) return; - if (!*coro) abort(); + if (!*coro) unreachable(); { CoroutineContext ctx; (*coro)(true); } - if (*coro) abort(); + if (*coro) unreachable(); } }; @@ -316,7 +316,7 @@ std::unique_ptr sinkToSource( }); } - if (!*coro) { eof(); abort(); } + if (!*coro) { eof(); unreachable(); } if (pos == cur.size()) { if (!cur.empty()) { diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 18f4a79c3..c7290dcef 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -159,13 +159,7 @@ struct FdSource : BufferedSource FdSource(Descriptor fd) : fd(fd) { } FdSource(FdSource &&) = default; - FdSource & operator=(FdSource && s) - { - fd = s.fd; - s.fd = INVALID_DESCRIPTOR; - read = s.read; - return *this; - } + FdSource & operator=(FdSource && s) = default; bool good() override; protected: @@ -210,7 +204,7 @@ struct TeeSink : Sink { Sink & sink1, & sink2; TeeSink(Sink & sink1, Sink & sink2) : sink1(sink1), sink2(sink2) { } - virtual void operator () (std::string_view data) + virtual void operator () (std::string_view data) override { sink1(data); sink2(data); @@ -227,7 +221,7 @@ struct TeeSource : Source Sink & sink; TeeSource(Source & orig, Sink & sink) : orig(orig), sink(sink) { } - size_t read(char * data, size_t len) + size_t read(char * data, size_t len) override { size_t n = orig.read(data, len); sink({data, n}); @@ -244,7 +238,7 @@ struct SizedSource : Source size_t remain; SizedSource(Source & orig, size_t size) : orig(orig), remain(size) { } - size_t read(char * data, size_t len) + size_t read(char * data, size_t len) override { if (this->remain <= 0) { throw EndOfFile("sized: unexpected end-of-file"); @@ -489,13 +483,17 @@ struct FramedSource : Source ~FramedSource() { - if (!eof) { - while (true) { - auto n = readInt(from); - if (!n) break; - std::vector data(n); - from(data.data(), n); + try { + if (!eof) { + while (true) { + auto n = readInt(from); + if (!n) break; + std::vector data(n); + from(data.data(), n); + } } + } catch (...) { + ignoreException(); } } diff --git a/src/libutil/source-path.hh b/src/libutil/source-path.hh index 1e96b72e5..fc2288f74 100644 --- a/src/libutil/source-path.hh +++ b/src/libutil/source-path.hh @@ -8,8 +8,7 @@ #include "ref.hh" #include "canon-path.hh" #include "source-accessor.hh" - -#include // for boost::hash_combine +#include "std-hash.hh" namespace nix { diff --git a/src/libutil/std-hash.hh b/src/libutil/std-hash.hh new file mode 100644 index 000000000..c359d11ca --- /dev/null +++ b/src/libutil/std-hash.hh @@ -0,0 +1,24 @@ +#pragma once + +//!@file Hashing utilities for use with unordered_map, etc. (ie low level implementation logic, not domain logic like +//! Nix hashing) + +#include + +namespace nix { + +/** + * hash_combine() from Boost. Hash several hashable values together + * into a single hash. + */ +inline void hash_combine(std::size_t & seed) {} + +template +inline void hash_combine(std::size_t & seed, const T & v, Rest... rest) +{ + std::hash hasher; + seed ^= hasher(v) + 0x9e3779b9 + (seed << 6) + (seed >> 2); + hash_combine(seed, rest...); +} + +} // namespace nix diff --git a/src/libutil/sync.hh b/src/libutil/sync.hh index 20dd6ee52..d340f3d97 100644 --- a/src/libutil/sync.hh +++ b/src/libutil/sync.hh @@ -7,6 +7,8 @@ #include #include +#include "error.hh" + namespace nix { /** @@ -47,7 +49,7 @@ public: friend SyncBase; Lock(SyncBase * s) : s(s), lk(s->mutex) { } public: - Lock(Lock && l) : s(l.s) { abort(); } + Lock(Lock && l) : s(l.s) { unreachable(); } Lock(const Lock & l) = delete; ~Lock() { } @@ -104,7 +106,7 @@ public: * Acquire read access to the inner value. When using * `std::shared_mutex`, this will use a shared lock. */ - ReadLock read() const { return ReadLock(const_cast(this)); } + ReadLock readLock() const { return ReadLock(const_cast(this)); } }; template diff --git a/src/libutil/unix/monitor-fd.hh b/src/libutil/unix/monitor-fd.hh index 103894de9..b6610feff 100644 --- a/src/libutil/unix/monitor-fd.hh +++ b/src/libutil/unix/monitor-fd.hh @@ -40,7 +40,9 @@ public: #endif ; auto count = poll(fds, 1, -1); - if (count == -1) abort(); // can't happen + if (count == -1) + unreachable(); + /* This shouldn't happen, but can on macOS due to a bug. See rdar://37550628. diff --git a/src/libutil/unix/processes.cc b/src/libutil/unix/processes.cc index 1af559a21..c5ce74acc 100644 --- a/src/libutil/unix/processes.cc +++ b/src/libutil/unix/processes.cc @@ -182,7 +182,7 @@ static pid_t doFork(bool allowVfork, ChildWrapperFunction & fun) #endif if (pid != 0) return pid; fun(); - abort(); + unreachable(); } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 83b42a528..877d15279 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -375,18 +375,4 @@ inline std::string operator + (std::string_view s1, const char * s2) return s; } -/** - * hash_combine() from Boost. Hash several hashable values together - * into a single hash. - */ -inline void hash_combine(std::size_t & seed) { } - -template -inline void hash_combine(std::size_t & seed, const T & v, Rest... rest) -{ - std::hash hasher; - seed ^= hasher(v) + 0x9e3779b9 + (seed<<6) + (seed>>2); - hash_combine(seed, rest...); -} - } diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 4d373ef8d..0ce987d8a 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -183,6 +183,9 @@ static void main_nix_build(int argc, char * * argv) struct MyArgs : LegacyArgs, MixEvalArgs { using LegacyArgs::LegacyArgs; + void setBaseDir(Path baseDir) { + commandBaseDir = baseDir; + } }; MyArgs myArgs(myName, [&](Strings::iterator & arg, const Strings::iterator & end) { @@ -290,6 +293,9 @@ static void main_nix_build(int argc, char * * argv) state->repair = myArgs.repair; if (myArgs.repair) buildMode = bmRepair; + if (inShebang && compatibilitySettings.nixShellShebangArgumentsRelativeToScript) { + myArgs.setBaseDir(absPath(dirOf(script))); + } auto autoArgs = myArgs.getAutoArgs(*state); auto autoArgsWithInNixShell = autoArgs; @@ -334,8 +340,13 @@ static void main_nix_build(int argc, char * * argv) exprs = {state->parseStdin()}; else for (auto i : remainingArgs) { + auto baseDir = inShebang && !packages ? absPath(dirOf(script)) : i; + if (fromArgs) - exprs.push_back(state->parseExprFromString(std::move(i), state->rootPath("."))); + exprs.push_back(state->parseExprFromString( + std::move(i), + (inShebang && compatibilitySettings.nixShellShebangArgumentsRelativeToScript) ? lookupFileArg(*state, baseDir) : state->rootPath(".") + )); else { auto absolute = i; try { diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 5e170c99d..40c200542 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1159,7 +1159,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) case cvEqual: ch = '='; break; case cvGreater: ch = '<'; break; case cvUnavail: ch = '-'; break; - default: abort(); + default: unreachable(); } if (xmlOutput) { diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index f073074e8..b4de42ba1 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -480,7 +480,7 @@ static void opQuery(Strings opFlags, Strings opArgs) } default: - abort(); + unreachable(); } } diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 7cc0965a9..92ec3b78a 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -699,7 +699,11 @@ struct CmdDevelop : Common, MixEnvironment } } - runProgramInStore(store, UseLookupPath::Use, shell, args, buildEnvironment.getSystem()); + // Release our references to eval caches to ensure they are persisted to disk, because + // we are about to exec out of this process without running C++ destructors. + getEvalState()->evalCaches.clear(); + + execProgramInStore(store, UseLookupPath::Use, shell, args, buildEnvironment.getSystem()); #endif } }; diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index 46c94b211..2bc7fe82b 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -25,15 +25,17 @@ GroupedPaths getClosureInfo(ref store, const StorePath & toplevel) GroupedPaths groupedPaths; - for (auto & path : closure) { + for (auto const & path : closure) { /* Strip the output name. Unfortunately this is ambiguous (we can't distinguish between output names like "bin" and version suffixes like "unstable"). */ static std::regex regex("(.*)-([a-z]+|lib32|lib64)"); - std::smatch match; - std::string name(path.name()); + std::cmatch match; + std::string name{path.name()}; + std::string_view const origName = path.name(); std::string outputName; - if (std::regex_match(name, match, regex)) { + + if (std::regex_match(origName.begin(), origName.end(), match, regex)) { name = match[1]; outputName = match[2]; } diff --git a/src/nix/env.cc b/src/nix/env.cc index bc4e1b5e5..9db03ca37 100644 --- a/src/nix/env.cc +++ b/src/nix/env.cc @@ -2,6 +2,7 @@ #include #include "command.hh" +#include "eval.hh" #include "run.hh" #include "strings.hh" @@ -103,7 +104,11 @@ struct CmdShell : InstallablesCommand, MixEnvironment for (auto & arg : command) args.push_back(arg); - runProgramInStore(store, UseLookupPath::Use, *command.begin(), args); + // Release our references to eval caches to ensure they are persisted to disk, because + // we are about to exec out of this process without running C++ destructors. + getEvalState()->evalCaches.clear(); + + execProgramInStore(store, UseLookupPath::Use, *command.begin(), args); } }; diff --git a/src/nix/flake.md b/src/nix/flake.md index 2f43d0264..46d5a3867 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -259,6 +259,8 @@ Currently the `type` attribute can be one of the following: `.tgz`, `.tar.gz`, `.tar.xz`, `.tar.bz2` or `.tar.zst`), then the `tarball+` can be dropped. + This can also be used to set the location of gitea/forgejo branches. [See here](@docroot@/protocols/tarball-fetcher.md#gitea-and-forgejo-support) + * `file`: Plain files or directory tarballs, either over http(s) or from the local disk. diff --git a/src/nix/fmt.cc b/src/nix/fmt.cc index 4b0fbb89d..d65834495 100644 --- a/src/nix/fmt.cc +++ b/src/nix/fmt.cc @@ -1,5 +1,6 @@ #include "command.hh" #include "installable-value.hh" +#include "eval.hh" #include "run.hh" using namespace nix; @@ -49,7 +50,11 @@ struct CmdFmt : SourceExprCommand { } } - runProgramInStore(store, UseLookupPath::DontUse, app.program, programArgs); + // Release our references to eval caches to ensure they are persisted to disk, because + // we are about to exec out of this process without running C++ destructors. + evalState->evalCaches.clear(); + + execProgramInStore(store, UseLookupPath::DontUse, app.program, programArgs); }; }; diff --git a/src/nix/main.cc b/src/nix/main.cc index 023998bba..1c5209cec 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -161,7 +161,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs {"ls-store", { AliasStatus::Deprecated, {"store", "ls"}}}, {"make-content-addressable", { AliasStatus::Deprecated, {"store", "make-content-addressed"}}}, {"optimise-store", { AliasStatus::Deprecated, {"store", "optimise"}}}, - {"ping-store", { AliasStatus::Deprecated, {"store", "ping"}}}, + {"ping-store", { AliasStatus::Deprecated, {"store", "info"}}}, {"sign-paths", { AliasStatus::Deprecated, {"store", "sign"}}}, {"shell", { AliasStatus::AcceptedShorthand, {"env", "shell"}}}, {"show-derivation", { AliasStatus::Deprecated, {"derivation", "show"}}}, diff --git a/src/nix/meson.build b/src/nix/meson.build index dd21c4b1b..7405548de 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -20,6 +20,7 @@ deps_private_maybe_subproject = [ dependency('nix-util'), dependency('nix-store'), dependency('nix-expr'), + dependency('nix-flake'), dependency('nix-fetchers'), dependency('nix-main'), dependency('nix-cmd'), @@ -28,6 +29,8 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') +subdir('build-utils-meson/threads') + subdir('build-utils-meson/export-all-symbols') add_project_arguments( @@ -88,12 +91,17 @@ nix_sources = files( 'store-info.cc', 'store-repair.cc', 'store.cc', - 'unix/daemon.cc', 'upgrade-nix.cc', 'verify.cc', 'why-depends.cc', ) +if host_machine.system() != 'windows' + nix_sources += files( + 'unix/daemon.cc', + ) +endif + nix_sources += [ gen_header.process('doc/manual/generate-manpage.nix'), gen_header.process('doc/manual/generate-settings.nix'), @@ -104,12 +112,6 @@ nix_sources += [ gen_header.process('help-stores.md'), ] -if host_machine.system() != 'windows' - nix_sources += files( - 'unix/daemon.cc', - ) -endif - # The rest of the subdirectories aren't separate components, # just source files in another directory, so we process them here. @@ -146,9 +148,7 @@ nix_store_sources = files( # Hurray for Meson list flattening! sources = [ nix_sources, - build_remote_sources, nix_build_sources, - nix_channel_sources, unpack_channel_gen, nix_collect_garbage_sources, nix_copy_closure_sources, @@ -158,6 +158,13 @@ sources = [ nix_store_sources, ] +if host_machine.system() != 'windows' + sources += [ + build_remote_sources, + nix_channel_sources, + ] +endif + include_dirs = [include_directories('.')] this_exe = executable( diff --git a/src/nix/nix.md b/src/nix/nix.md index 5ac00b940..d37cb9cce 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -54,9 +54,13 @@ These are command line arguments that represent something that can be realised i The following types of installable are supported by most commands: - [Flake output attribute](#flake-output-attribute) (experimental) + - This is the default - [Store path](#store-path) + - This is assumed if the argument is a Nix store path or a symlink to a Nix store path - [Nix file](#nix-file), optionally qualified by an attribute path + - Specified with `--file`/`-f` - [Nix expression](#nix-expression), optionally qualified by an attribute path + - Specified with `--expr`/`-E` For most commands, if no installable is specified, `.` is assumed. That is, Nix will operate on the default flake output attribute of the flake in the current directory. @@ -166,9 +170,10 @@ that contains programs, and a `dev` output that provides development artifacts like C/C++ header files. The outputs on which `nix` commands operate are determined as follows: -* You can explicitly specify the desired outputs using the syntax - *installable*`^`*output1*`,`*...*`,`*outputN*. For example, you can - obtain the `dev` and `static` outputs of the `glibc` package: +* You can explicitly specify the desired outputs using the syntax *installable*`^`*output1*`,`*...*`,`*outputN* — that is, a caret followed immediately by a comma-separated list of derivation outputs to select. + For installables specified as [Flake output attributes](#flake-output-attribute) or [Store paths](#store-path), the output is specified in the same argument: + + For example, you can obtain the `dev` and `static` outputs of the `glibc` package: ```console # nix build 'nixpkgs#glibc^dev,static' @@ -183,6 +188,19 @@ operate are determined as follows: … ``` + For `-e`/`--expr` and `-f`/`--file`, the derivation output is specified as part of the attribute path: + + ```console + $ nix build -f '' 'glibc^dev,static' + $ nix build --impure -E 'import { }' 'glibc^dev,static' + ``` + + This syntax is the same even if the actual attribute path is empty: + + ```console + $ nix build -E 'let pkgs = import { }; in pkgs.glibc' '^dev,static' + ``` + * You can also specify that *all* outputs should be used using the syntax *installable*`^*`. For example, the following shows the size of all outputs of the `glibc` package in the binary cache: diff --git a/src/nix/package.nix b/src/nix/package.nix index fe83c6969..ef7265458 100644 --- a/src/nix/package.nix +++ b/src/nix/package.nix @@ -84,8 +84,6 @@ mkMesonDerivation (finalAttrs: { ] ); - outputs = [ "out" "dev" ]; - nativeBuildInputs = [ meson ninja @@ -114,12 +112,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/nix/run.cc b/src/nix/run.cc index c1aae1685..ec6a4d1e8 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -25,7 +25,7 @@ std::string chrootHelperName = "__run_in_chroot"; namespace nix { -void runProgramInStore(ref store, +void execProgramInStore(ref store, UseLookupPath useLookupPath, const std::string & program, const Strings & args, @@ -128,7 +128,11 @@ struct CmdRun : InstallableValueCommand Strings allArgs{app.program}; for (auto & i : args) allArgs.push_back(i); - runProgramInStore(store, UseLookupPath::DontUse, app.program, allArgs); + // Release our references to eval caches to ensure they are persisted to disk, because + // we are about to exec out of this process without running C++ destructors. + state->evalCaches.clear(); + + execProgramInStore(store, UseLookupPath::DontUse, app.program, allArgs); } }; diff --git a/src/nix/run.hh b/src/nix/run.hh index 2fe6ed86a..51517fdc9 100644 --- a/src/nix/run.hh +++ b/src/nix/run.hh @@ -10,7 +10,7 @@ enum struct UseLookupPath { DontUse }; -void runProgramInStore(ref store, +void execProgramInStore(ref store, UseLookupPath useLookupPath, const std::string & program, const Strings & args, diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index 4a7997b1f..66d8dbcf0 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -370,9 +370,12 @@ static void daemonLoop(std::optional forceTrustClientOpt) } // Handle the connection. - FdSource from(remote.get()); - FdSink to(remote.get()); - processConnection(openUncachedStore(), from, to, trusted, NotRecursive); + processConnection( + openUncachedStore(), + FdSource(remote.get()), + FdSink(remote.get()), + trusted, + NotRecursive); exit(0); }, options); @@ -437,9 +440,11 @@ static void forwardStdioConnection(RemoteStore & store) { */ static void processStdioConnection(ref store, TrustedFlag trustClient) { - FdSource from(STDIN_FILENO); - FdSink to(STDOUT_FILENO); - processConnection(store, from, to, trustClient, NotRecursive); + processConnection( + store, + FdSource(STDIN_FILENO), + FdSink(STDOUT_FILENO), + trustClient, NotRecursive); } /** diff --git a/src/perl/meson.build b/src/perl/meson.build index 5fe7e1e28..dcb6a68a4 100644 --- a/src/perl/meson.build +++ b/src/perl/meson.build @@ -23,21 +23,22 @@ nix_perl_conf.set('PACKAGE_VERSION', meson.project_version()) # set error arguments #------------------------------------------------- error_args = [ - '-Werror=unused-result', '-Wdeprecated-copy', '-Wdeprecated-declarations', + '-Werror=suggest-override', + '-Werror=unused-result', '-Wignored-qualifiers', - '-Wno-pedantic', - '-Wno-non-virtual-dtor', - '-Wno-unused-parameter', - '-Wno-variadic-macros', - '-Wno-missing-field-initializers', - '-Wno-unknown-warning-option', - '-Wno-unused-variable', - '-Wno-literal-suffix', - '-Wno-reserved-user-defined-literal', '-Wno-duplicate-decl-specifier', + '-Wno-literal-suffix', + '-Wno-missing-field-initializers', + '-Wno-non-virtual-dtor', + '-Wno-pedantic', '-Wno-pointer-bool-conversion', + '-Wno-reserved-user-defined-literal', + '-Wno-unknown-warning-option', + '-Wno-unused-parameter', + '-Wno-unused-variable', + '-Wno-variadic-macros', ] add_project_arguments( diff --git a/src/perl/package.nix b/src/perl/package.nix index 26856e631..0b9343fba 100644 --- a/src/perl/package.nix +++ b/src/perl/package.nix @@ -73,5 +73,5 @@ perl.pkgs.toPerlModule (mkMesonDerivation (finalAttrs: { "--print-errorlogs" ]; - enableParallelBuilding = true; + strictDeps = false; })) diff --git a/tests/functional/build.sh b/tests/functional/build.sh index 9de953d8c..5396a465f 100755 --- a/tests/functional/build.sh +++ b/tests/functional/build.sh @@ -140,6 +140,18 @@ nix build --impure -f multiple-outputs.nix --json e --no-link | jq --exit-status (.outputs | keys == ["a_a", "b"])) ' +# Make sure that the 3 types of aliases work +# BaseSettings, BaseSettings, and BaseSettings. +nix build --impure -f multiple-outputs.nix --json e --no-link \ + --build-max-jobs 3 \ + --gc-keep-outputs \ + --build-use-sandbox | \ + jq --exit-status ' + (.[0] | + (.drvPath | match(".*multiple-outputs-e.drv")) and + (.outputs | keys == ["a_a", "b"])) +' + # Make sure that `--stdin` works and does not apply any defaults printf "" | nix build --no-link --stdin --json | jq --exit-status '. == []' printf "%s\n" "$drv^*" | nix build --no-link --stdin --json | jq --exit-status '.[0]|has("drvPath")' diff --git a/tests/functional/common/vars-and-functions.sh b/tests/functional/common/vars-and-functions.sh index e21a7ff0a..9484d6446 100644 --- a/tests/functional/common/vars-and-functions.sh +++ b/tests/functional/common/vars-and-functions.sh @@ -191,7 +191,7 @@ isDaemonNewer () { skipTest () { echo "$1, skipping this test..." >&2 - exit 99 + exit 77 } TODO_NixOS() { @@ -237,7 +237,8 @@ expect() { expected="$1" shift "$@" && res=0 || res="$?" - if [[ $res -ne $expected ]]; then + # also match "negative" codes, which wrap around to >127 + if [[ $res -ne $expected && $res -ne $[256 + expected] ]]; then echo "Expected exit code '$expected' but got '$res' from command ${*@Q}" >&2 return 1 fi @@ -251,7 +252,8 @@ expectStderr() { expected="$1" shift "$@" 2>&1 && res=0 || res="$?" - if [[ $res -ne $expected ]]; then + # also match "negative" codes, which wrap around to >127 + if [[ $res -ne $expected && $res -ne $[256 + expected] ]]; then echo "Expected exit code '$expected' but got '$res' from command ${*@Q}" >&2 return 1 fi @@ -296,13 +298,67 @@ onError() { done } +# Prints an error message prefix referring to the last call into this file. +# Ignores `expect` and `expectStderr` calls. +# Set a special exit code when test suite functions are misused, so that +# functions like expectStderr won't mistake them for expected Nix CLI errors. +# Suggestion: -101 (negative to indicate very abnormal, and beyond the normal +# range of signals) +# Example (showns as string): 'repl.sh:123: in call to grepQuiet: ' +# This function is inefficient, so it should only be used in error messages. +callerPrefix() { + # Find the closest caller that's not from this file + # using the bash `caller` builtin. + local i file line fn savedFn + # Use `caller` + for i in $(seq 0 100); do + caller $i > /dev/null || { + if [[ -n "${file:-}" ]]; then + echo "$file:$line: ${savedFn+in call to $savedFn: }" + fi + break + } + line="$(caller $i | cut -d' ' -f1)" + fn="$(caller $i | cut -d' ' -f2)" + file="$(caller $i | cut -d' ' -f3)" + if [[ $file != "${BASH_SOURCE[0]}" ]]; then + echo "$file:$line: ${savedFn+in call to $savedFn: }" + return + fi + case "$fn" in + # Ignore higher order functions that don't report any misuse of themselves + # This way a misuse of a foo in `expectStderr 1 foo` will be reported as + # calling foo, not expectStderr. + expect|expectStderr|callerPrefix) + ;; + *) + savedFn="$fn" + ;; + esac + done +} + +checkGrepArgs() { + local arg + for arg in "$@"; do + if [[ "$arg" != "${arg//$'\n'/_}" ]]; then + echo "$(callerPrefix)newline not allowed in arguments; grep would try each line individually as if connected by an OR operator" >&2 + return -101 + fi + done +} + # `grep -v` doesn't work well for exit codes. We want `!(exist line l. l # matches)`. It gives us `exist line l. !(l matches)`. # # `!` normally doesn't work well with `set -e`, but when we wrap in a # function it *does*. +# +# `command grep` lets us avoid re-checking the args by going directly to the +# executable. grepInverse() { - ! grep "$@" + checkGrepArgs "$@" && \ + ! command grep "$@" } # A shorthand, `> /dev/null` is a bit noisy. @@ -316,13 +372,26 @@ grepInverse() { # the closing of the pipe, the buffering of the pipe, and the speed of # the producer into the pipe. But rest assured we've seen it happen in # CI reliably. +# +# `command grep` lets us avoid re-checking the args by going directly to the +# executable. grepQuiet() { - grep "$@" > /dev/null + checkGrepArgs "$@" && \ + command grep "$@" > /dev/null } # The previous two, combined grepQuietInverse() { - ! grep "$@" > /dev/null + checkGrepArgs "$@" && \ + ! command grep "$@" > /dev/null +} + +# Wrap grep to remove its newline footgun; see checkGrepArgs. +# Note that we keep the checkGrepArgs calls in the other helpers, because some +# of them are negated and that would defeat this check. +grep() { + checkGrepArgs "$@" && \ + command grep "$@" } # Return the number of arguments diff --git a/tests/functional/eval.sh b/tests/functional/eval.sh index 27cdce478..22d2d02a2 100755 --- a/tests/functional/eval.sh +++ b/tests/functional/eval.sh @@ -58,3 +58,7 @@ fi # Test that unknown settings are warned about out="$(expectStderr 0 nix eval --option foobar baz --expr '""' --raw)" [[ "$(echo "$out" | grep foobar | wc -l)" = 1 ]] + +# Test flag alias +out="$(nix eval --expr '{}' --build-cores 1)" +[[ "$(echo "$out" | wc -l)" = 1 ]] diff --git a/tests/functional/flakes/local.mk b/tests/functional/flakes/local.mk new file mode 100644 index 000000000..71e50ad07 --- /dev/null +++ b/tests/functional/flakes/local.mk @@ -0,0 +1,24 @@ +flake-tests := \ + $(d)/flakes.sh \ + $(d)/develop.sh \ + $(d)/edit.sh \ + $(d)/run.sh \ + $(d)/mercurial.sh \ + $(d)/circular.sh \ + $(d)/init.sh \ + $(d)/inputs.sh \ + $(d)/follow-paths.sh \ + $(d)/bundle.sh \ + $(d)/check.sh \ + $(d)/unlocked-override.sh \ + $(d)/absolute-paths.sh \ + $(d)/absolute-attr-paths.sh \ + $(d)/build-paths.sh \ + $(d)/flake-in-submodule.sh \ + $(d)/prefetch.sh \ + $(d)/eval-cache.sh \ + $(d)/search-root.sh \ + $(d)/config.sh \ + $(d)/show.sh + +install-tests-groups += flake diff --git a/tests/functional/flakes/run.sh b/tests/functional/flakes/run.sh index c2c345ed2..61af6049a 100755 --- a/tests/functional/flakes/run.sh +++ b/tests/functional/flakes/run.sh @@ -29,5 +29,26 @@ nix run --no-write-lock-file .#pkgAsPkg ! nix run --no-write-lock-file .#pkgAsApp || fail "'nix run' shouldn’t accept an 'app' defined under 'packages'" ! nix run --no-write-lock-file .#appAsPkg || fail "elements of 'apps' should be of type 'app'" +# Test that we're not setting any more environment variables than necessary. +# For instance, we might set an environment variable temporarily to affect some +# initialization or whatnot, but this must not leak into the environment of the +# command being run. +env > $TEST_ROOT/expected-env +nix run -f shell-hello.nix env > $TEST_ROOT/actual-env +# Remove/reset variables we expect to be different. +# - PATH is modified by nix shell +# - _ is set by bash and is expected to differ because it contains the original command +# - __CF_USER_TEXT_ENCODING is set by macOS and is beyond our control +sed -i \ + -e 's/PATH=.*/PATH=.../' \ + -e 's/_=.*/_=.../' \ + -e '/^__CF_USER_TEXT_ENCODING=.*$/d' \ + $TEST_ROOT/expected-env $TEST_ROOT/actual-env +sort $TEST_ROOT/expected-env | uniq > $TEST_ROOT/expected-env.sorted +# nix run appears to clear _. I don't understand why. Is this ok? +echo "_=..." >> $TEST_ROOT/actual-env +sort $TEST_ROOT/actual-env | uniq > $TEST_ROOT/actual-env.sorted +diff $TEST_ROOT/expected-env.sorted $TEST_ROOT/actual-env.sorted + clearStore diff --git a/tests/functional/lang-gc.sh b/tests/functional/lang-gc.sh new file mode 100644 index 000000000..1746fa4c1 --- /dev/null +++ b/tests/functional/lang-gc.sh @@ -0,0 +1,36 @@ +# shellcheck shell=bash + +# Regression tests for the evaluator +# These are not in lang.sh because they generally only need to run in CI, +# whereas lang.sh is often run locally during development + + +source common.sh + +set -o pipefail + +skipTest "Too memory instensive for CI. Attempt to reduce memory usage was unsuccessful, because it made detection of the bug unreliable." + +# Regression test for #11141. The stack pointer corrector assigned the base +# instead of the top (which resides at the low end of the stack). Sounds confusing? +# Stacks grow downwards, so that's why this mistake happened. +# My manual testing did not uncover this, because it didn't rely on the stack enough. +# https://github.com/NixOS/nix/issues/11141 +test_issue_11141() { + mkdir -p "$TEST_ROOT/issue-11141/src" + cp lang-gc/issue-11141-gc-coroutine-test.nix "$TEST_ROOT/issue-11141/" + ( + set +x; + n=10 + echo "populating $TEST_ROOT/issue-11141/src with $((n*100)) files..." + for i in $(seq 0 $n); do + touch "$TEST_ROOT/issue-11141/src/file-$i"{0,1,2,3,4,5,6,7,8,9}{0,1,2,3,4,5,6,7,8,9} + done + ) + + GC_INITIAL_HEAP_SIZE=$((1024 * 1024)) \ + NIX_SHOW_STATS=1 \ + nix eval -vvv\ + -f "$TEST_ROOT/issue-11141/issue-11141-gc-coroutine-test.nix" +} +test_issue_11141 diff --git a/tests/functional/lang-gc/issue-11141-gc-coroutine-test.nix b/tests/functional/lang-gc/issue-11141-gc-coroutine-test.nix new file mode 100644 index 000000000..4f311af75 --- /dev/null +++ b/tests/functional/lang-gc/issue-11141-gc-coroutine-test.nix @@ -0,0 +1,65 @@ + +# Run: +# GC_INITIAL_HEAP_SIZE=$[1024 * 1024] NIX_SHOW_STATS=1 nix eval -f gc-coroutine-test.nix -vvvv + +let + inherit (builtins) + foldl' + isList + ; + + # Generate a tree of numbers, n deep, such that the numbers add up to (1 + salt) * 10^n. + # The salting makes the numbers all different, increasing the likelihood of catching + # any memory corruptions that might be caused by the GC or otherwise. + garbage = salt: n: + if n == 0 + then [(1 + salt)] + else [ + (garbage (10 * salt + 1) (n - 1)) + (garbage (10 * salt - 1) (n - 1)) + (garbage (10 * salt + 2) (n - 1)) + (garbage (10 * salt - 2) (n - 1)) + (garbage (10 * salt + 3) (n - 1)) + (garbage (10 * salt - 3) (n - 1)) + (garbage (10 * salt + 4) (n - 1)) + (garbage (10 * salt - 4) (n - 1)) + (garbage (10 * salt + 5) (n - 1)) + (garbage (10 * salt - 5) (n - 1)) + ]; + + pow = base: n: + if n == 0 + then 1 + else base * (pow base (n - 1)); + + sumNestedLists = l: + if isList l + then foldl' (a: b: a + sumNestedLists b) 0 l + else l; + +in + assert sumNestedLists (garbage 0 3) == pow 10 3; + assert sumNestedLists (garbage 0 6) == pow 10 6; + builtins.foldl' + (a: b: + assert + "${ + builtins.path { + path = ./src; + filter = path: type: + # We're not doing common subexpression elimination, so this reallocates + # the fairly big tree over and over, producing a lot of garbage during + # source filtering, whose filter runs in a coroutine. + assert sumNestedLists (garbage 0 3) == pow 10 3; + true; + } + }" + == "${./src}"; + + # These asserts don't seem necessary, as the lambda value get corrupted first + assert a.okay; + assert b.okay; + { okay = true; } + ) + { okay = true; } + [ { okay = true; } { okay = true; } { okay = true; } ] diff --git a/tests/functional/lang.sh b/tests/functional/lang.sh index 8cb8e98fb..46cf3f1fe 100755 --- a/tests/functional/lang.sh +++ b/tests/functional/lang.sh @@ -50,11 +50,25 @@ set +x badDiff=0 badExitCode=0 +# Extra post-processing that's specific to each test case +postprocess() { + if [[ -e "lang/$1.postprocess" ]]; then + ( + # We could allow arbitrary interpreters in .postprocess, but that + # just exposes us to the complexity of not having /usr/bin/env in + # the sandbox. So let's just hardcode bash for now. + set -x; + bash "lang/$1.postprocess" "lang/$1" + ) + fi +} + for i in lang/parse-fail-*.nix; do echo "parsing $i (should fail)"; i=$(basename "$i" .nix) if expectStderr 1 nix-instantiate --parse - < "lang/$i.nix" > "lang/$i.err" then + postprocess "$i" diffAndAccept "$i" err err.exp else echo "FAIL: $i shouldn't parse" @@ -71,6 +85,7 @@ for i in lang/parse-okay-*.nix; do 2> "lang/$i.err" then sed "s!$(pwd)!/pwd!g" "lang/$i.out" "lang/$i.err" + postprocess "$i" diffAndAccept "$i" out exp diffAndAccept "$i" err err.exp else @@ -94,6 +109,7 @@ for i in lang/eval-fail-*.nix; do expectStderr 1 nix-instantiate $flags "lang/$i.nix" \ | sed "s!$(pwd)!/pwd!g" > "lang/$i.err" then + postprocess "$i" diffAndAccept "$i" err err.exp else echo "FAIL: $i shouldn't evaluate" @@ -109,6 +125,7 @@ for i in lang/eval-okay-*.nix; do if expect 0 nix-instantiate --eval --xml --no-location --strict \ "lang/$i.nix" > "lang/$i.out.xml" then + postprocess "$i" diffAndAccept "$i" out.xml exp.xml else echo "FAIL: $i should evaluate" @@ -129,6 +146,7 @@ for i in lang/eval-okay-*.nix; do 2> "lang/$i.err" then sed -i "s!$(pwd)!/pwd!g" "lang/$i.out" "lang/$i.err" + postprocess "$i" diffAndAccept "$i" out exp diffAndAccept "$i" err err.exp else diff --git a/tests/functional/lang/eval-fail-assert-equal-attrs-names-2.err.exp b/tests/functional/lang/eval-fail-assert-equal-attrs-names-2.err.exp new file mode 100644 index 000000000..4b68d97c2 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-attrs-names-2.err.exp @@ -0,0 +1,8 @@ +error: + … while evaluating the condition of the assertion '({ a = true; } == { a = true; b = true; })' + at /pwd/lang/eval-fail-assert-equal-attrs-names-2.nix:1:1: + 1| assert { a = true; } == { a = true; b = true; }; + | ^ + 2| throw "unreachable" + + error: attribute names of attribute set '{ a = true; }' differs from attribute set '{ a = true; b = true; }' diff --git a/tests/functional/lang/eval-fail-assert-equal-attrs-names-2.nix b/tests/functional/lang/eval-fail-assert-equal-attrs-names-2.nix new file mode 100644 index 000000000..8e7ac9cf2 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-attrs-names-2.nix @@ -0,0 +1,2 @@ +assert { a = true; } == { a = true; b = true; }; +throw "unreachable" diff --git a/tests/functional/lang/eval-fail-assert-equal-attrs-names.err.exp b/tests/functional/lang/eval-fail-assert-equal-attrs-names.err.exp new file mode 100644 index 000000000..bc61ca63a --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-attrs-names.err.exp @@ -0,0 +1,8 @@ +error: + … while evaluating the condition of the assertion '({ a = true; b = true; } == { a = true; })' + at /pwd/lang/eval-fail-assert-equal-attrs-names.nix:1:1: + 1| assert { a = true; b = true; } == { a = true; }; + | ^ + 2| throw "unreachable" + + error: attribute names of attribute set '{ a = true; b = true; }' differs from attribute set '{ a = true; }' diff --git a/tests/functional/lang/eval-fail-assert-equal-attrs-names.nix b/tests/functional/lang/eval-fail-assert-equal-attrs-names.nix new file mode 100644 index 000000000..e2f53a85a --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-attrs-names.nix @@ -0,0 +1,2 @@ +assert { a = true; b = true; } == { a = true; }; +throw "unreachable" diff --git a/tests/functional/lang/eval-fail-assert-equal-derivations-extra.err.exp b/tests/functional/lang/eval-fail-assert-equal-derivations-extra.err.exp new file mode 100644 index 000000000..7f4924074 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-derivations-extra.err.exp @@ -0,0 +1,26 @@ +error: + … while evaluating the condition of the assertion '({ foo = { outPath = "/nix/store/0"; type = "derivation"; }; } == { foo = { devious = true; outPath = "/nix/store/1"; type = "derivation"; }; })' + at /pwd/lang/eval-fail-assert-equal-derivations-extra.nix:1:1: + 1| assert + | ^ + 2| { foo = { type = "derivation"; outPath = "/nix/store/0"; }; } + + … while comparing attribute 'foo' + + … where left hand side is + at /pwd/lang/eval-fail-assert-equal-derivations-extra.nix:2:5: + 1| assert + 2| { foo = { type = "derivation"; outPath = "/nix/store/0"; }; } + | ^ + 3| == + + … where right hand side is + at /pwd/lang/eval-fail-assert-equal-derivations-extra.nix:4:5: + 3| == + 4| { foo = { type = "derivation"; outPath = "/nix/store/1"; devious = true; }; }; + | ^ + 5| throw "unreachable" + + … while comparing a derivation by its 'outPath' attribute + + error: string '"/nix/store/0"' is not equal to string '"/nix/store/1"' diff --git a/tests/functional/lang/eval-fail-assert-equal-derivations-extra.nix b/tests/functional/lang/eval-fail-assert-equal-derivations-extra.nix new file mode 100644 index 000000000..fd8bc3f26 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-derivations-extra.nix @@ -0,0 +1,5 @@ +assert + { foo = { type = "derivation"; outPath = "/nix/store/0"; }; } + == + { foo = { type = "derivation"; outPath = "/nix/store/1"; devious = true; }; }; +throw "unreachable" \ No newline at end of file diff --git a/tests/functional/lang/eval-fail-assert-equal-derivations.err.exp b/tests/functional/lang/eval-fail-assert-equal-derivations.err.exp new file mode 100644 index 000000000..d7f0face0 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-derivations.err.exp @@ -0,0 +1,26 @@ +error: + … while evaluating the condition of the assertion '({ foo = { ignored = (abort "not ignored"); outPath = "/nix/store/0"; type = "derivation"; }; } == { foo = { ignored = (abort "not ignored"); outPath = "/nix/store/1"; type = "derivation"; }; })' + at /pwd/lang/eval-fail-assert-equal-derivations.nix:1:1: + 1| assert + | ^ + 2| { foo = { type = "derivation"; outPath = "/nix/store/0"; ignored = abort "not ignored"; }; } + + … while comparing attribute 'foo' + + … where left hand side is + at /pwd/lang/eval-fail-assert-equal-derivations.nix:2:5: + 1| assert + 2| { foo = { type = "derivation"; outPath = "/nix/store/0"; ignored = abort "not ignored"; }; } + | ^ + 3| == + + … where right hand side is + at /pwd/lang/eval-fail-assert-equal-derivations.nix:4:5: + 3| == + 4| { foo = { type = "derivation"; outPath = "/nix/store/1"; ignored = abort "not ignored"; }; }; + | ^ + 5| throw "unreachable" + + … while comparing a derivation by its 'outPath' attribute + + error: string '"/nix/store/0"' is not equal to string '"/nix/store/1"' diff --git a/tests/functional/lang/eval-fail-assert-equal-derivations.nix b/tests/functional/lang/eval-fail-assert-equal-derivations.nix new file mode 100644 index 000000000..c648eae37 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-derivations.nix @@ -0,0 +1,5 @@ +assert + { foo = { type = "derivation"; outPath = "/nix/store/0"; ignored = abort "not ignored"; }; } + == + { foo = { type = "derivation"; outPath = "/nix/store/1"; ignored = abort "not ignored"; }; }; +throw "unreachable" \ No newline at end of file diff --git a/tests/functional/lang/eval-fail-assert-equal-floats.err.exp b/tests/functional/lang/eval-fail-assert-equal-floats.err.exp new file mode 100644 index 000000000..d8545e2db --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-floats.err.exp @@ -0,0 +1,22 @@ +error: + … while evaluating the condition of the assertion '({ b = 1; } == { b = 1.01; })' + at /pwd/lang/eval-fail-assert-equal-floats.nix:1:1: + 1| assert { b = 1.0; } == { b = 1.01; }; + | ^ + 2| abort "unreachable" + + … while comparing attribute 'b' + + … where left hand side is + at /pwd/lang/eval-fail-assert-equal-floats.nix:1:10: + 1| assert { b = 1.0; } == { b = 1.01; }; + | ^ + 2| abort "unreachable" + + … where right hand side is + at /pwd/lang/eval-fail-assert-equal-floats.nix:1:26: + 1| assert { b = 1.0; } == { b = 1.01; }; + | ^ + 2| abort "unreachable" + + error: a float with value '1' is not equal to a float with value '1.01' diff --git a/tests/functional/lang/eval-fail-assert-equal-floats.nix b/tests/functional/lang/eval-fail-assert-equal-floats.nix new file mode 100644 index 000000000..438e85abf --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-floats.nix @@ -0,0 +1,2 @@ +assert { b = 1.0; } == { b = 1.01; }; +abort "unreachable" diff --git a/tests/functional/lang/eval-fail-assert-equal-function-direct.err.exp b/tests/functional/lang/eval-fail-assert-equal-function-direct.err.exp new file mode 100644 index 000000000..f06d79698 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-function-direct.err.exp @@ -0,0 +1,9 @@ +error: + … while evaluating the condition of the assertion '((x: x) == (x: x))' + at /pwd/lang/eval-fail-assert-equal-function-direct.nix:3:1: + 2| # This only compares a direct comparison and makes no claims about functions in nested structures. + 3| assert + | ^ + 4| (x: x) + + error: distinct functions and immediate comparisons of identical functions compare as unequal diff --git a/tests/functional/lang/eval-fail-assert-equal-function-direct.nix b/tests/functional/lang/eval-fail-assert-equal-function-direct.nix new file mode 100644 index 000000000..68e5e3908 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-function-direct.nix @@ -0,0 +1,7 @@ +# Note: functions in nested structures, e.g. attributes, may be optimized away by pointer identity optimization. +# This only compares a direct comparison and makes no claims about functions in nested structures. +assert + (x: x) + == + (x: x); +abort "unreachable" \ No newline at end of file diff --git a/tests/functional/lang/eval-fail-assert-equal-int-float.err.exp b/tests/functional/lang/eval-fail-assert-equal-int-float.err.exp new file mode 100644 index 000000000..c927e38d6 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-int-float.err.exp @@ -0,0 +1,8 @@ +error: + … while evaluating the condition of the assertion '(1 == 1.1)' + at /pwd/lang/eval-fail-assert-equal-int-float.nix:1:1: + 1| assert 1 == 1.1; + | ^ + 2| throw "unreachable" + + error: an integer with value '1' is not equal to a float with value '1.1' diff --git a/tests/functional/lang/eval-fail-assert-equal-int-float.nix b/tests/functional/lang/eval-fail-assert-equal-int-float.nix new file mode 100644 index 000000000..1dfdf2bda --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-int-float.nix @@ -0,0 +1,2 @@ +assert 1 == 1.1; +throw "unreachable" diff --git a/tests/functional/lang/eval-fail-assert-equal-ints.err.exp b/tests/functional/lang/eval-fail-assert-equal-ints.err.exp new file mode 100644 index 000000000..d6219e200 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-ints.err.exp @@ -0,0 +1,22 @@ +error: + … while evaluating the condition of the assertion '({ b = 1; } == { b = 2; })' + at /pwd/lang/eval-fail-assert-equal-ints.nix:1:1: + 1| assert { b = 1; } == { b = 2; }; + | ^ + 2| abort "unreachable" + + … while comparing attribute 'b' + + … where left hand side is + at /pwd/lang/eval-fail-assert-equal-ints.nix:1:10: + 1| assert { b = 1; } == { b = 2; }; + | ^ + 2| abort "unreachable" + + … where right hand side is + at /pwd/lang/eval-fail-assert-equal-ints.nix:1:24: + 1| assert { b = 1; } == { b = 2; }; + | ^ + 2| abort "unreachable" + + error: an integer with value '1' is not equal to an integer with value '2' diff --git a/tests/functional/lang/eval-fail-assert-equal-ints.nix b/tests/functional/lang/eval-fail-assert-equal-ints.nix new file mode 100644 index 000000000..645258ea6 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-ints.nix @@ -0,0 +1,2 @@ +assert { b = 1; } == { b = 2; }; +abort "unreachable" diff --git a/tests/functional/lang/eval-fail-assert-equal-list-length.err.exp b/tests/functional/lang/eval-fail-assert-equal-list-length.err.exp new file mode 100644 index 000000000..90108552c --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-list-length.err.exp @@ -0,0 +1,8 @@ +error: + … while evaluating the condition of the assertion '([ (1) (0) ] == [ (10) ])' + at /pwd/lang/eval-fail-assert-equal-list-length.nix:1:1: + 1| assert [ 1 0 ] == [ 10 ]; + | ^ + 2| throw "unreachable" + + error: list of size '2' is not equal to list of size '1', left hand side is '[ 1 0 ]', right hand side is '[ 10 ]' diff --git a/tests/functional/lang/eval-fail-assert-equal-list-length.nix b/tests/functional/lang/eval-fail-assert-equal-list-length.nix new file mode 100644 index 000000000..6d40f4d8e --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-list-length.nix @@ -0,0 +1,2 @@ +assert [ 1 0 ] == [ 10 ]; +throw "unreachable" \ No newline at end of file diff --git a/tests/functional/lang/eval-fail-assert-equal-paths.err.exp b/tests/functional/lang/eval-fail-assert-equal-paths.err.exp new file mode 100644 index 000000000..66c34e971 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-paths.err.exp @@ -0,0 +1,8 @@ +error: + … while evaluating the condition of the assertion '(/pwd/lang/foo == /pwd/lang/bar)' + at /pwd/lang/eval-fail-assert-equal-paths.nix:1:1: + 1| assert ./foo == ./bar; + | ^ + 2| throw "unreachable" + + error: path '/pwd/lang/foo' is not equal to path '/pwd/lang/bar' diff --git a/tests/functional/lang/eval-fail-assert-equal-paths.nix b/tests/functional/lang/eval-fail-assert-equal-paths.nix new file mode 100644 index 000000000..ef0b67024 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-paths.nix @@ -0,0 +1,2 @@ +assert ./foo == ./bar; +throw "unreachable" \ No newline at end of file diff --git a/tests/functional/lang/eval-fail-assert-equal-type-nested.err.exp b/tests/functional/lang/eval-fail-assert-equal-type-nested.err.exp new file mode 100644 index 000000000..f78badd25 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-type-nested.err.exp @@ -0,0 +1,22 @@ +error: + … while evaluating the condition of the assertion '({ ding = false; } == { ding = null; })' + at /pwd/lang/eval-fail-assert-equal-type-nested.nix:1:1: + 1| assert { ding = false; } == { ding = null; }; + | ^ + 2| abort "unreachable" + + … while comparing attribute 'ding' + + … where left hand side is + at /pwd/lang/eval-fail-assert-equal-type-nested.nix:1:10: + 1| assert { ding = false; } == { ding = null; }; + | ^ + 2| abort "unreachable" + + … where right hand side is + at /pwd/lang/eval-fail-assert-equal-type-nested.nix:1:31: + 1| assert { ding = false; } == { ding = null; }; + | ^ + 2| abort "unreachable" + + error: a Boolean of value 'false' is not equal to null of value 'null' diff --git a/tests/functional/lang/eval-fail-assert-equal-type-nested.nix b/tests/functional/lang/eval-fail-assert-equal-type-nested.nix new file mode 100644 index 000000000..3fbd14ce6 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-type-nested.nix @@ -0,0 +1,2 @@ +assert { ding = false; } == { ding = null; }; +abort "unreachable" diff --git a/tests/functional/lang/eval-fail-assert-equal-type.err.exp b/tests/functional/lang/eval-fail-assert-equal-type.err.exp new file mode 100644 index 000000000..4dc3f2ece --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-type.err.exp @@ -0,0 +1,8 @@ +error: + … while evaluating the condition of the assertion '(false == null)' + at /pwd/lang/eval-fail-assert-equal-type.nix:1:1: + 1| assert false == null; + | ^ + 2| abort "unreachable" + + error: a Boolean of value 'false' is not equal to null of value 'null' diff --git a/tests/functional/lang/eval-fail-assert-equal-type.nix b/tests/functional/lang/eval-fail-assert-equal-type.nix new file mode 100644 index 000000000..7023ea007 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-type.nix @@ -0,0 +1,2 @@ +assert false == null; +abort "unreachable" diff --git a/tests/functional/lang/eval-fail-assert-nested-bool.err.exp b/tests/functional/lang/eval-fail-assert-nested-bool.err.exp new file mode 100644 index 000000000..1debb668c --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-nested-bool.err.exp @@ -0,0 +1,74 @@ +error: + … while evaluating the condition of the assertion '({ a = { b = [ ({ c = { d = true; }; }) ]; }; } == { a = { b = [ ({ c = { d = false; }; }) ]; }; })' + at /pwd/lang/eval-fail-assert-nested-bool.nix:1:1: + 1| assert + | ^ + 2| { a.b = [ { c.d = true; } ]; } + + … while comparing attribute 'a' + + … where left hand side is + at /pwd/lang/eval-fail-assert-nested-bool.nix:2:5: + 1| assert + 2| { a.b = [ { c.d = true; } ]; } + | ^ + 3| == + + … where right hand side is + at /pwd/lang/eval-fail-assert-nested-bool.nix:4:5: + 3| == + 4| { a.b = [ { c.d = false; } ]; }; + | ^ + 5| + + … while comparing attribute 'b' + + … where left hand side is + at /pwd/lang/eval-fail-assert-nested-bool.nix:2:5: + 1| assert + 2| { a.b = [ { c.d = true; } ]; } + | ^ + 3| == + + … where right hand side is + at /pwd/lang/eval-fail-assert-nested-bool.nix:4:5: + 3| == + 4| { a.b = [ { c.d = false; } ]; }; + | ^ + 5| + + … while comparing list element 0 + + … while comparing attribute 'c' + + … where left hand side is + at /pwd/lang/eval-fail-assert-nested-bool.nix:2:15: + 1| assert + 2| { a.b = [ { c.d = true; } ]; } + | ^ + 3| == + + … where right hand side is + at /pwd/lang/eval-fail-assert-nested-bool.nix:4:15: + 3| == + 4| { a.b = [ { c.d = false; } ]; }; + | ^ + 5| + + … while comparing attribute 'd' + + … where left hand side is + at /pwd/lang/eval-fail-assert-nested-bool.nix:2:15: + 1| assert + 2| { a.b = [ { c.d = true; } ]; } + | ^ + 3| == + + … where right hand side is + at /pwd/lang/eval-fail-assert-nested-bool.nix:4:15: + 3| == + 4| { a.b = [ { c.d = false; } ]; }; + | ^ + 5| + + error: boolean 'true' is not equal to boolean 'false' diff --git a/tests/functional/lang/eval-fail-assert-nested-bool.nix b/tests/functional/lang/eval-fail-assert-nested-bool.nix new file mode 100644 index 000000000..228576983 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-nested-bool.nix @@ -0,0 +1,6 @@ +assert + { a.b = [ { c.d = true; } ]; } + == + { a.b = [ { c.d = false; } ]; }; + +abort "unreachable" \ No newline at end of file diff --git a/tests/functional/lang/eval-fail-assert.err.exp b/tests/functional/lang/eval-fail-assert.err.exp index 0656ec81c..7be9e2387 100644 --- a/tests/functional/lang/eval-fail-assert.err.exp +++ b/tests/functional/lang/eval-fail-assert.err.exp @@ -20,9 +20,11 @@ error: | ^ 3| - error: assertion '(arg == "y")' failed - at /pwd/lang/eval-fail-assert.nix:2:12: + … while evaluating the condition of the assertion '(arg == "y")' + at /pwd/lang/eval-fail-assert.nix:2:12: 1| let { 2| x = arg: assert arg == "y"; 123; | ^ 3| + + error: string '"x"' is not equal to string '"y"' diff --git a/tests/functional/lang/eval-fail-derivation-name.err.exp b/tests/functional/lang/eval-fail-derivation-name.err.exp index eb2206df1..0ef98674d 100644 --- a/tests/functional/lang/eval-fail-derivation-name.err.exp +++ b/tests/functional/lang/eval-fail-derivation-name.err.exp @@ -1,26 +1,26 @@ error: … while evaluating the attribute 'outPath' - at :19:9: - 18| value = commonAttrs // { - 19| outPath = builtins.getAttr outputName strict; + at ::: + | value = commonAttrs // { + | outPath = builtins.getAttr outputName strict; | ^ - 20| drvPath = strict.drvPath; + | drvPath = strict.drvPath; … while calling the 'getAttr' builtin - at :19:19: - 18| value = commonAttrs // { - 19| outPath = builtins.getAttr outputName strict; + at ::: + | value = commonAttrs // { + | outPath = builtins.getAttr outputName strict; | ^ - 20| drvPath = strict.drvPath; + | drvPath = strict.drvPath; … while calling the 'derivationStrict' builtin - at :9:12: - 8| - 9| strict = derivationStrict drvAttrs; + at ::: + | + | strict = derivationStrict drvAttrs; | ^ - 10| + | … while evaluating derivation '~jiggle~' - whose name attribute is located at /pwd/lang/eval-fail-derivation-name.nix:2:3 + whose name attribute is located at /pwd/lang/eval-fail-derivation-name.nix:: error: invalid derivation name: name '~jiggle~' contains illegal character '~'. Please pass a different 'name'. diff --git a/tests/functional/lang/eval-fail-derivation-name.postprocess b/tests/functional/lang/eval-fail-derivation-name.postprocess new file mode 100644 index 000000000..ffbc2b5d4 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivation-name.postprocess @@ -0,0 +1,9 @@ +# shellcheck shell=bash +set -euo pipefail +testcaseBasename=$1 + +# Line numbers change when derivation.nix docs are updated. +sed -i "$testcaseBasename.err" \ + -e 's/[0-9 ][0-9 ][0-9 ][0-9 ][0-9 ][0-9 ][0-9 ][0-9]\([^0-9]\)/\1/g' \ + -e 's/[0-9][0-9]*//g' \ + ; diff --git a/tests/functional/lang/eval-fail-pipe-operators.err.exp b/tests/functional/lang/eval-fail-pipe-operators.err.exp new file mode 100644 index 000000000..49f3fa8ad --- /dev/null +++ b/tests/functional/lang/eval-fail-pipe-operators.err.exp @@ -0,0 +1,5 @@ +error: experimental Nix feature 'pipe-operators' is disabled; add '--extra-experimental-features pipe-operators' to enable it + at /pwd/lang/eval-fail-pipe-operators.nix:1:3: + 1| 1 |> 2 + | ^ + 2| diff --git a/tests/functional/lang/eval-fail-pipe-operators.nix b/tests/functional/lang/eval-fail-pipe-operators.nix new file mode 100644 index 000000000..433e0fd7f --- /dev/null +++ b/tests/functional/lang/eval-fail-pipe-operators.nix @@ -0,0 +1 @@ +1 |> 2 diff --git a/tests/functional/lang/parse-okay-ind-string.exp b/tests/functional/lang/parse-okay-ind-string.exp new file mode 100644 index 000000000..82e9940a2 --- /dev/null +++ b/tests/functional/lang/parse-okay-ind-string.exp @@ -0,0 +1 @@ +(let string = "str"; in [ (/some/path) ((/some/path)) ((/some/path)) ((/some/path + "\n end")) (string) ((string)) ((string)) ((string + "\n end")) ("") ("") ("end") ]) diff --git a/tests/functional/lang/parse-okay-ind-string.nix b/tests/functional/lang/parse-okay-ind-string.nix new file mode 100644 index 000000000..97c9de3cd --- /dev/null +++ b/tests/functional/lang/parse-okay-ind-string.nix @@ -0,0 +1,31 @@ +let + string = "str"; +in [ + /some/path + + ''${/some/path}'' + + '' + ${/some/path}'' + + ''${/some/path} + end'' + + string + + ''${string}'' + + '' + ${string}'' + + ''${string} + end'' + + '''' + + '' + '' + + '' + end'' +] diff --git a/tests/functional/local.mk b/tests/functional/local.mk index 49ee31284..8b4945cac 100644 --- a/tests/functional/local.mk +++ b/tests/functional/local.mk @@ -1,28 +1,11 @@ nix_tests = \ test-infra.sh \ - flakes/flakes.sh \ - flakes/develop.sh \ - flakes/edit.sh \ - flakes/run.sh \ - flakes/mercurial.sh \ - flakes/circular.sh \ - flakes/init.sh \ - flakes/inputs.sh \ - flakes/follow-paths.sh \ - flakes/bundle.sh \ - flakes/check.sh \ - flakes/unlocked-override.sh \ - flakes/absolute-paths.sh \ - flakes/absolute-attr-paths.sh \ - flakes/build-paths.sh \ - flakes/flake-in-submodule.sh \ - flakes/prefetch.sh \ - flakes/eval-cache.sh \ gc.sh \ nix-collect-garbage-d.sh \ remote-store.sh \ legacy-ssh-store.sh \ lang.sh \ + lang-gc.sh \ characterisation-test-infra.sh \ experimental-features.sh \ fetchMercurial.sh \ @@ -60,7 +43,6 @@ nix_tests = \ restricted.sh \ fetchGitSubmodules.sh \ fetchGitVerification.sh \ - flakes/search-root.sh \ readfile-context.sh \ nix-channel.sh \ recursive.sh \ @@ -101,7 +83,6 @@ nix_tests = \ nix-copy-ssh-ng.sh \ post-hook.sh \ function-trace.sh \ - flakes/config.sh \ fmt.sh \ eval-store.sh \ why-depends.sh \ @@ -124,7 +105,6 @@ nix_tests = \ store-info.sh \ fetchClosure.sh \ completions.sh \ - flakes/show.sh \ impure-derivations.sh \ path-from-hash-part.sh \ path-info.sh \ diff --git a/tests/functional/nix-shell.sh b/tests/functional/nix-shell.sh index 65ff279f8..b9625eb66 100755 --- a/tests/functional/nix-shell.sh +++ b/tests/functional/nix-shell.sh @@ -65,6 +65,25 @@ chmod a+rx $TEST_ROOT/shell.shebang.sh output=$($TEST_ROOT/shell.shebang.sh abc def) [ "$output" = "foo bar abc def" ] +# Test nix-shell shebang mode with an alternate working directory +sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.expr > $TEST_ROOT/shell.shebang.expr +chmod a+rx $TEST_ROOT/shell.shebang.expr +# Should fail due to expressions using relative path +! $TEST_ROOT/shell.shebang.expr bar +cp shell.nix config.nix $TEST_ROOT +# Should succeed +echo "cwd: $PWD" +output=$($TEST_ROOT/shell.shebang.expr bar) +[ "$output" = foo ] + +# Test nix-shell shebang mode with an alternate working directory +sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.legacy.expr > $TEST_ROOT/shell.shebang.legacy.expr +chmod a+rx $TEST_ROOT/shell.shebang.legacy.expr +# Should fail due to expressions using relative path +mkdir -p "$TEST_ROOT/somewhere-unrelated" +output="$(cd "$TEST_ROOT/somewhere-unrelated"; $TEST_ROOT/shell.shebang.legacy.expr bar;)" +[[ $(realpath "$output") = $(realpath "$TEST_ROOT/somewhere-unrelated") ]] + # Test nix-shell shebang mode again with metacharacters in the filename. # First word of filename is chosen to not match any file in the test root. sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.sh > $TEST_ROOT/spaced\ \\\'\"shell.shebang.sh diff --git a/tests/functional/nix_path.sh b/tests/functional/nix_path.sh index e6a2193f3..90cba1f0c 100755 --- a/tests/functional/nix_path.sh +++ b/tests/functional/nix_path.sh @@ -14,3 +14,75 @@ nix-instantiate --eval -E '' --restrict-eval [[ $(nix-instantiate --find-file by-absolute-path/simple.nix) = $PWD/simple.nix ]] [[ $(nix-instantiate --find-file by-relative-path/simple.nix) = $PWD/simple.nix ]] + +# this is the human-readable specification for the following test cases of interactions between various ways of specifying NIX_PATH. +# TODO: the actual tests are incomplete and too manual. +# there should be 43 of them, since the table has 9 rows and columns, and 2 interactions are meaningless +# ideally they would work off the table programmatically. +# +# | precedence | hard-coded | nix-path in file | extra-nix-path in file | nix-path in env | extra-nix-path in env | NIX_PATH | nix-path | extra-nix-path | -I | +# |------------------------|------------|------------------|------------------------|-----------------|-----------------------|-----------|-----------|-----------------|-----------------| +# | hard-coded | x | ^override | ^append | ^override | ^append | ^override | ^override | ^append | ^prepend | +# | nix-path in file | | last wins | ^append | ^override | ^append | ^override | ^override | ^append | ^prepend | +# | extra-nix-path in file | | | append in order | ^override | ^append | ^override | ^override | ^append | ^prepend | +# | nix-path in env | | | | last wins | ^append | ^override | ^override | ^append | ^prepend | +# | extra-nix-path in env | | | | | append in order | ^override | ^override | ^append | ^prepend | +# | NIX_PATH | | | | | | x | ^override | ^append | ^prepend | +# | nix-path | | | | | | | last wins | ^append | ^prepend | +# | extra-nix-path | | | | | | | | append in order | append in order | +# | -I | | | | | | | | | append in order | + +unset NIX_PATH + +mkdir -p $TEST_ROOT/{from-nix-path-file,from-NIX_PATH,from-nix-path,from-extra-nix-path,from-I} +for i in from-nix-path-file from-NIX_PATH from-nix-path from-extra-nix-path from-I; do + touch $TEST_ROOT/$i/only-$i.nix +done + +# finding something that's not in any of the default paths fails +( ! $(nix-instantiate --find-file test) ) + +echo "nix-path = test=$TEST_ROOT/from-nix-path-file" >> "$test_nix_conf" + +# Use nix.conf in absence of NIX_PATH +[[ $(nix-instantiate --find-file test) = $TEST_ROOT/from-nix-path-file ]] + +# NIX_PATH overrides nix.conf +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate --find-file test) = $TEST_ROOT/from-NIX_PATH ]] +# if NIX_PATH does not have the desired entry, it fails +(! NIX_PATH=test=$TEST_ROOT nix-instantiate --find-file test/only-from-nix-path-file.nix) + +# -I extends nix.conf +[[ $(nix-instantiate -I test=$TEST_ROOT/from-I --find-file test/only-from-I.nix) = $TEST_ROOT/from-I/only-from-I.nix ]] +# if -I does not have the desired entry, the value from nix.conf is used +[[ $(nix-instantiate -I test=$TEST_ROOT/from-I --find-file test/only-from-nix-path-file.nix) = $TEST_ROOT/from-nix-path-file/only-from-nix-path-file.nix ]] + +# -I extends NIX_PATH +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate -I test=$TEST_ROOT/from-I --find-file test/only-from-I.nix) = $TEST_ROOT/from-I/only-from-I.nix ]] +# -I takes precedence over NIX_PATH +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate -I test=$TEST_ROOT/from-I --find-file test) = $TEST_ROOT/from-I ]] +# if -I does not have the desired entry, the value from NIX_PATH is used +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate -I test=$TEST_ROOT/from-I --find-file test/only-from-NIX_PATH.nix) = $TEST_ROOT/from-NIX_PATH/only-from-NIX_PATH.nix ]] + +# --extra-nix-path extends NIX_PATH +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate --extra-nix-path test=$TEST_ROOT/from-extra-nix-path --find-file test/only-from-extra-nix-path.nix) = $TEST_ROOT/from-extra-nix-path/only-from-extra-nix-path.nix ]] +# if --extra-nix-path does not have the desired entry, the value from NIX_PATH is used +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate --extra-nix-path test=$TEST_ROOT/from-extra-nix-path --find-file test/only-from-NIX_PATH.nix) = $TEST_ROOT/from-NIX_PATH/only-from-NIX_PATH.nix ]] + +# --nix-path overrides NIX_PATH +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path --find-file test) = $TEST_ROOT/from-nix-path ]] +# if --nix-path does not have the desired entry, it fails +(! NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path --find-file test/only-from-NIX_PATH.nix) + +# --nix-path overrides nix.conf +[[ $(nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path --find-file test) = $TEST_ROOT/from-nix-path ]] +(! nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path --find-file test/only-from-nix-path-file.nix) + +# --extra-nix-path extends nix.conf +[[ $(nix-instantiate --extra-nix-path test=$TEST_ROOT/from-extra-nix-path --find-file test/only-from-extra-nix-path.nix) = $TEST_ROOT/from-extra-nix-path/only-from-extra-nix-path.nix ]] +# if --extra-nix-path does not have the desired entry, it is taken from nix.conf +[[ $(nix-instantiate --extra-nix-path test=$TEST_ROOT/from-extra-nix-path --find-file test) = $TEST_ROOT/from-nix-path-file ]] + +# -I extends --nix-path +[[ $(nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path -I test=$TEST_ROOT/from-I --find-file test/only-from-I.nix) = $TEST_ROOT/from-I/only-from-I.nix ]] +[[ $(nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path -I test=$TEST_ROOT/from-I --find-file test/only-from-nix-path.nix) = $TEST_ROOT/from-nix-path/only-from-nix-path.nix ]] diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index 6d00e944b..365bd5197 100755 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -262,6 +262,23 @@ badExitCode=0 nixVersion="$(nix eval --impure --raw --expr 'builtins.nixVersion' --extra-experimental-features nix-command)" +# TODO: write a repl interacter for testing. Papering over the differences between readline / editline and between platforms is a pain. + +# I couldn't get readline and editline to agree on the newline before the prompt, +# so let's just force it to be one empty line. +stripEmptyLinesBeforePrompt() { + # --null-data: treat input as NUL-terminated instead of newline-terminated + sed --null-data 's/\n\n*nix-repl>/\n\nnix-repl>/g' +} + +# We don't get a final prompt on darwin, so we strip this as well. +stripFinalPrompt() { + # Strip the final prompt and/or any trailing spaces + sed --null-data \ + -e 's/\(.*[^\n]\)\n\n*nix-repl>[ \n]*$/\1/' \ + -e 's/[ \n]*$/\n/' +} + runRepl () { # That is right, we are also filtering out the testdir _without underscores_. @@ -273,8 +290,13 @@ runRepl () { testDirNoUnderscores="${testDir//_/}" # TODO: pass arguments to nix repl; see lang.sh + _NIX_TEST_RAW_MARKDOWN=1 \ + _NIX_TEST_REPL_ECHO=1 \ nix repl 2>&1 \ | stripColors \ + | tr -d '\0' \ + | stripEmptyLinesBeforePrompt \ + | stripFinalPrompt \ | sed \ -e "s@$testDir@/path/to/tests/functional@g" \ -e "s@$testDirNoUnderscores@/path/to/tests/functional@g" \ @@ -289,7 +311,10 @@ for test in $(cd "$testDir/repl"; echo *.in); do in="$testDir/repl/$test.in" actual="$testDir/repl/$test.actual" expected="$testDir/repl/$test.expected" - (cd "$testDir/repl"; set +x; runRepl 2>&1) < "$in" > "$actual" + (cd "$testDir/repl"; set +x; runRepl 2>&1) < "$in" > "$actual" || { + echo "FAIL: $test (exit code $?)" >&2 + badExitCode=1 + } diffAndAcceptInner "$test" "$actual" "$expected" done diff --git a/tests/functional/repl/doc-comment-curried-args.expected b/tests/functional/repl/doc-comment-curried-args.expected index c10c171e1..56607e911 100644 --- a/tests/functional/repl/doc-comment-curried-args.expected +++ b/tests/functional/repl/doc-comment-curried-args.expected @@ -1,24 +1,28 @@ Nix Type :? for help. + +nix-repl> :l doc-comments.nix Added variables. -Function curriedArgs - … defined at - /path/to/tests/functional/repl/doc-comments.nix:48:5 +nix-repl> :doc curriedArgs +Function `curriedArgs`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:48:5 - A documented function. +A documented function. +nix-repl> x = curriedArgs 1 +nix-repl> "Note that users may not expect this to behave as it currently does" "Note that users may not expect this to behave as it currently does" -Function curriedArgs - … defined at - /path/to/tests/functional/repl/doc-comments.nix:50:5 +nix-repl> :doc x +Function `curriedArgs`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:50:5 - The function returned by applying once +The function returned by applying once -"This won't produce documentation, because we can't actually document arbitrary values" +nix-repl> "This won't produce docs; no support for arbitrary values" +"This won't produce docs; no support for arbitrary values" +nix-repl> :doc x 2 error: value does not have documentation - - diff --git a/tests/functional/repl/doc-comment-curried-args.in b/tests/functional/repl/doc-comment-curried-args.in index 8dbbfc370..06ba21dcc 100644 --- a/tests/functional/repl/doc-comment-curried-args.in +++ b/tests/functional/repl/doc-comment-curried-args.in @@ -3,5 +3,5 @@ x = curriedArgs 1 "Note that users may not expect this to behave as it currently does" :doc x -"This won't produce documentation, because we can't actually document arbitrary values" +"This won't produce docs; no support for arbitrary values" :doc x 2 diff --git a/tests/functional/repl/doc-comment-formals.expected b/tests/functional/repl/doc-comment-formals.expected index 704c0050b..1024919f4 100644 --- a/tests/functional/repl/doc-comment-formals.expected +++ b/tests/functional/repl/doc-comment-formals.expected @@ -1,13 +1,14 @@ Nix Type :? for help. + +nix-repl> :l doc-comments.nix Added variables. +nix-repl> "Note that this is not yet complete" "Note that this is not yet complete" -Function documentedFormals - … defined at - /path/to/tests/functional/repl/doc-comments.nix:57:5 - - Finds x - +nix-repl> :doc documentedFormals +Function `documentedFormals`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:57:5 +Finds x diff --git a/tests/functional/repl/doc-comment-function.expected b/tests/functional/repl/doc-comment-function.expected index 5ec465a96..3889c4f78 100644 --- a/tests/functional/repl/doc-comment-function.expected +++ b/tests/functional/repl/doc-comment-function.expected @@ -1,8 +1,7 @@ Nix Type :? for help. -Function defined at - /path/to/tests/functional/repl/doc-comment-function.nix:2:1 - - A doc comment for a file that only contains a function +nix-repl> :doc import ./doc-comment-function.nix +Function defined at /path/to/tests/functional/repl/doc-comment-function.nix:2:1 +A doc comment for a file that only contains a function diff --git a/tests/functional/repl/doc-compact.expected b/tests/functional/repl/doc-compact.expected index 4b05b653c..79f1fd44f 100644 --- a/tests/functional/repl/doc-compact.expected +++ b/tests/functional/repl/doc-compact.expected @@ -1,11 +1,11 @@ Nix Type :? for help. + +nix-repl> :l doc-comments.nix Added variables. -Function compact - … defined at - /path/to/tests/functional/repl/doc-comments.nix:18:20 - - boom - +nix-repl> :doc compact +Function `compact`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:18:20 +boom diff --git a/tests/functional/repl/doc-constant.expected b/tests/functional/repl/doc-constant.expected index c66558333..5787e04dc 100644 --- a/tests/functional/repl/doc-constant.expected +++ b/tests/functional/repl/doc-constant.expected @@ -1,23 +1,27 @@ Nix Type :? for help. + +nix-repl> :l doc-comments.nix Added variables. +nix-repl> :doc constant error: value does not have documentation -Attribute version +nix-repl> :doc lib.version +Attribute `version` - … defined at - /path/to/tests/functional/repl/doc-comments.nix:30:3 + … defined at /path/to/tests/functional/repl/doc-comments.nix:30:3 - Immovably fixed. +Immovably fixed. -Attribute empty +nix-repl> :doc lib.attr.empty +Attribute `empty` - … defined at - /path/to/tests/functional/repl/doc-comments.nix:33:3 + … defined at /path/to/tests/functional/repl/doc-comments.nix:33:3 - Unchangeably constant. +Unchangeably constant. +nix-repl> :doc lib.attr.undocument error: … while evaluating the attribute 'attr.undocument' at /path/to/tests/functional/repl/doc-comments.nix:33:3: @@ -32,59 +36,65 @@ error: | ^ Did you mean undocumented? -Attribute constant +nix-repl> :doc (import ./doc-comments.nix).constant +Attribute `constant` - … defined at - /path/to/tests/functional/repl/doc-comments.nix:27:3 + … defined at /path/to/tests/functional/repl/doc-comments.nix:27:3 - Firmly rigid. +Firmly rigid. -Attribute version +nix-repl> :doc (import ./doc-comments.nix).lib.version +Attribute `version` - … defined at - /path/to/tests/functional/repl/doc-comments.nix:30:3 + … defined at /path/to/tests/functional/repl/doc-comments.nix:30:3 - Immovably fixed. +Immovably fixed. -Attribute empty +nix-repl> :doc (import ./doc-comments.nix).lib.attr.empty +Attribute `empty` - … defined at - /path/to/tests/functional/repl/doc-comments.nix:33:3 + … defined at /path/to/tests/functional/repl/doc-comments.nix:33:3 - Unchangeably constant. +Unchangeably constant. -Attribute undocumented +nix-repl> :doc (import ./doc-comments.nix).lib.attr.undocumented +Attribute `undocumented` - … defined at - /path/to/tests/functional/repl/doc-comments.nix:35:3 + … defined at /path/to/tests/functional/repl/doc-comments.nix:35:3 - No documentation found. +No documentation found. +nix-repl> :doc missing error: undefined variable 'missing' at «string»:1:1: 1| missing | ^ +nix-repl> :doc constanz error: undefined variable 'constanz' at «string»:1:1: 1| constanz | ^ +nix-repl> :doc missing.attr error: undefined variable 'missing' at «string»:1:1: 1| missing.attr | ^ +nix-repl> :doc lib.missing error: attribute 'missing' missing at «string»:1:1: 1| lib.missing | ^ +nix-repl> :doc lib.missing.attr error: attribute 'missing' missing at «string»:1:1: 1| lib.missing.attr | ^ +nix-repl> :doc lib.attr.undocumental error: … while evaluating the attribute 'attr.undocumental' at /path/to/tests/functional/repl/doc-comments.nix:33:3: @@ -98,5 +108,3 @@ error: 1| lib.attr.undocumental | ^ Did you mean undocumented? - - diff --git a/tests/functional/repl/doc-floatedIn.expected b/tests/functional/repl/doc-floatedIn.expected index 30f135725..82bb80b95 100644 --- a/tests/functional/repl/doc-floatedIn.expected +++ b/tests/functional/repl/doc-floatedIn.expected @@ -1,11 +1,11 @@ Nix Type :? for help. + +nix-repl> :l doc-comments.nix Added variables. -Function floatedIn - … defined at - /path/to/tests/functional/repl/doc-comments.nix:16:5 - - This also works. - +nix-repl> :doc floatedIn +Function `floatedIn`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:16:5 +This also works. diff --git a/tests/functional/repl/doc-lambda-flavors.expected b/tests/functional/repl/doc-lambda-flavors.expected index 43f483ce9..ab5c95639 100644 --- a/tests/functional/repl/doc-lambda-flavors.expected +++ b/tests/functional/repl/doc-lambda-flavors.expected @@ -1,29 +1,29 @@ Nix Type :? for help. + +nix-repl> :l doc-comments.nix Added variables. -Function nonStrict - … defined at - /path/to/tests/functional/repl/doc-comments.nix:37:70 +nix-repl> :doc nonStrict +Function `nonStrict`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:37:70 - My syntax is not strict, but I'm strict anyway. +My syntax is not strict, but I'm strict anyway. -Function strict - … defined at - /path/to/tests/functional/repl/doc-comments.nix:38:63 +nix-repl> :doc strict +Function `strict`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:38:63 - I don't have to be strict, but I am anyway. +I don't have to be strict, but I am anyway. -Function strictPre - … defined at - /path/to/tests/functional/repl/doc-comments.nix:40:48 +nix-repl> :doc strictPre +Function `strictPre`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:40:48 - Here's one way to do this - -Function strictPost - … defined at - /path/to/tests/functional/repl/doc-comments.nix:41:53 - - Here's another way to do this +Here's one way to do this +nix-repl> :doc strictPost +Function `strictPost`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:41:53 +Here's another way to do this diff --git a/tests/functional/repl/doc-measurement.expected b/tests/functional/repl/doc-measurement.expected index 8598aaedb..555cac9a2 100644 --- a/tests/functional/repl/doc-measurement.expected +++ b/tests/functional/repl/doc-measurement.expected @@ -1,11 +1,11 @@ Nix Type :? for help. + +nix-repl> :l doc-comments.nix Added variables. -Function measurement - … defined at - /path/to/tests/functional/repl/doc-comments.nix:13:17 - - 👈 precisely this wide 👉 - +nix-repl> :doc measurement +Function `measurement`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:13:17 +👈 precisely this wide 👉 diff --git a/tests/functional/repl/doc-multiply.expected b/tests/functional/repl/doc-multiply.expected index db82af91f..21523e24c 100644 --- a/tests/functional/repl/doc-multiply.expected +++ b/tests/functional/repl/doc-multiply.expected @@ -1,15 +1,17 @@ Nix Type :? for help. + +nix-repl> :l doc-comments.nix Added variables. -Function multiply - … defined at - /path/to/tests/functional/repl/doc-comments.nix:10:14 - - Perform arithmetic multiplication. It's kind of like - repeated addition, very neat. - - | multiply 2 3 - | => 6 +nix-repl> :doc multiply +Function `multiply`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:10:14 +Perform *arithmetic* multiplication. It's kind of like repeated **addition**, very neat. + +```nix +multiply 2 3 +=> 6 +``` diff --git a/tests/functional/repl/doc-unambiguous.expected b/tests/functional/repl/doc-unambiguous.expected index 825aa1ee1..0db5505d7 100644 --- a/tests/functional/repl/doc-unambiguous.expected +++ b/tests/functional/repl/doc-unambiguous.expected @@ -1,11 +1,11 @@ Nix Type :? for help. + +nix-repl> :l doc-comments.nix Added variables. -Function unambiguous - … defined at - /path/to/tests/functional/repl/doc-comments.nix:24:5 - - Very close - +nix-repl> :doc unambiguous +Function `unambiguous`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:24:5 +Very close diff --git a/tests/functional/repl/pretty-print-idempotent.expected b/tests/functional/repl/pretty-print-idempotent.expected new file mode 100644 index 000000000..311855dae --- /dev/null +++ b/tests/functional/repl/pretty-print-idempotent.expected @@ -0,0 +1,37 @@ +Nix +Type :? for help. + +nix-repl> :l pretty-print-idempotent.nix +Added variables. + +nix-repl> oneDeep +{ homepage = "https://example.com"; } + +nix-repl> oneDeep +{ homepage = "https://example.com"; } + +nix-repl> twoDeep +{ + layerOne = { ... }; +} + +nix-repl> twoDeep +{ + layerOne = { ... }; +} + +nix-repl> oneDeepList +[ "https://example.com" ] + +nix-repl> oneDeepList +[ "https://example.com" ] + +nix-repl> twoDeepList +[ + [ ... ] +] + +nix-repl> twoDeepList +[ + [ ... ] +] diff --git a/tests/functional/repl/pretty-print-idempotent.in b/tests/functional/repl/pretty-print-idempotent.in new file mode 100644 index 000000000..5f865316f --- /dev/null +++ b/tests/functional/repl/pretty-print-idempotent.in @@ -0,0 +1,9 @@ +:l pretty-print-idempotent.nix +oneDeep +oneDeep +twoDeep +twoDeep +oneDeepList +oneDeepList +twoDeepList +twoDeepList diff --git a/tests/functional/repl/pretty-print-idempotent.nix b/tests/functional/repl/pretty-print-idempotent.nix new file mode 100644 index 000000000..68929f387 --- /dev/null +++ b/tests/functional/repl/pretty-print-idempotent.nix @@ -0,0 +1,19 @@ +{ + oneDeep = { + homepage = "https://" + "example.com"; + }; + twoDeep = { + layerOne = { + homepage = "https://" + "example.com"; + }; + }; + + oneDeepList = [ + ("https://" + "example.com") + ]; + twoDeepList = [ + [ + ("https://" + "example.com") + ] + ]; +} diff --git a/tests/functional/restricted.sh b/tests/functional/restricted.sh index 915d973b0..591367e9f 100755 --- a/tests/functional/restricted.sh +++ b/tests/functional/restricted.sh @@ -10,6 +10,9 @@ nix-instantiate --restrict-eval --eval -E '1 + 2' nix-instantiate --restrict-eval ./simple.nix -I src=. nix-instantiate --restrict-eval ./simple.nix -I src1=simple.nix -I src2=config.nix -I src3=./simple.builder.sh +# no default NIX_PATH +(unset NIX_PATH; ! nix-instantiate --restrict-eval --find-file .) + (! nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix') nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix' -I src=../.. diff --git a/tests/functional/shell-hello.nix b/tests/functional/shell-hello.nix index c46fdec8a..c920d7cb4 100644 --- a/tests/functional/shell-hello.nix +++ b/tests/functional/shell-hello.nix @@ -55,4 +55,26 @@ rec { chmod +x $out/bin/hello ''; }; + + # execs env from PATH, so that we can probe the environment + # does not allow arguments, because we don't need them + env = mkDerivation { + name = "env"; + outputs = [ "out" ]; + buildCommand = + '' + mkdir -p $out/bin + + cat > $out/bin/env <&2 + exit 1 + fi + exec env + EOF + chmod +x $out/bin/env + ''; + }; + } diff --git a/tests/functional/shell.nix b/tests/functional/shell.nix index 750cdf0bc..9cae14b78 100644 --- a/tests/functional/shell.nix +++ b/tests/functional/shell.nix @@ -46,6 +46,7 @@ let pkgs = rec { ASCII_PERCENT = "%"; ASCII_AT = "@"; TEST_inNixShell = if inNixShell then "true" else "false"; + FOO = fooContents; inherit stdenv; outputs = ["dev" "out"]; } // { diff --git a/tests/functional/shell.sh b/tests/functional/shell.sh index b4c03f547..c2ac3b24d 100755 --- a/tests/functional/shell.sh +++ b/tests/functional/shell.sh @@ -23,6 +23,25 @@ nix shell -f shell-hello.nix hello-symlink -c hello | grep 'Hello World' # Test that symlinks outside of the store don't work. expect 1 nix shell -f shell-hello.nix forbidden-symlink -c hello 2>&1 | grepQuiet "is not in the Nix store" +# Test that we're not setting any more environment variables than necessary. +# For instance, we might set an environment variable temporarily to affect some +# initialization or whatnot, but this must not leak into the environment of the +# command being run. +env > $TEST_ROOT/expected-env +nix shell -f shell-hello.nix hello -c env > $TEST_ROOT/actual-env +# Remove/reset variables we expect to be different. +# - PATH is modified by nix shell +# - _ is set by bash and is expectedf to differ because it contains the original command +# - __CF_USER_TEXT_ENCODING is set by macOS and is beyond our control +sed -i \ + -e 's/PATH=.*/PATH=.../' \ + -e 's/_=.*/_=.../' \ + -e '/^__CF_USER_TEXT_ENCODING=.*$/d' \ + $TEST_ROOT/expected-env $TEST_ROOT/actual-env +sort $TEST_ROOT/expected-env > $TEST_ROOT/expected-env.sorted +sort $TEST_ROOT/actual-env > $TEST_ROOT/actual-env.sorted +diff $TEST_ROOT/expected-env.sorted $TEST_ROOT/actual-env.sorted + if isDaemonNewer "2.20.0pre20231220"; then # Test that command line attribute ordering is reflected in the PATH # https://github.com/NixOS/nix/issues/7905 diff --git a/tests/functional/shell.shebang.expr b/tests/functional/shell.shebang.expr new file mode 100755 index 000000000..c602dedbf --- /dev/null +++ b/tests/functional/shell.shebang.expr @@ -0,0 +1,9 @@ +#! @ENV_PROG@ nix-shell +#! nix-shell "{ script, path, ... }: assert path == ./shell.nix; script { }" +#! nix-shell --no-substitute +#! nix-shell --expr +#! nix-shell --arg script "import ./shell.nix" +#! nix-shell --arg path "./shell.nix" +#! nix-shell -A shellDrv +#! nix-shell -i bash +echo "$FOO" diff --git a/tests/functional/shell.shebang.legacy.expr b/tests/functional/shell.shebang.legacy.expr new file mode 100755 index 000000000..490542f43 --- /dev/null +++ b/tests/functional/shell.shebang.legacy.expr @@ -0,0 +1,10 @@ +#! @ENV_PROG@ nix-shell +#! nix-shell "{ script, path, ... }: assert path == ./shell.nix; script { fooContents = toString ./.; }" +#! nix-shell --no-substitute +#! nix-shell --expr +#! nix-shell --arg script "import ((builtins.getEnv ''TEST_ROOT'')+''/shell.nix'')" +#! nix-shell --arg path "./shell.nix" +#! nix-shell -A shellDrv +#! nix-shell -i bash +#! nix-shell --option nix-shell-shebang-arguments-relative-to-script false +echo "$FOO" diff --git a/tests/functional/tarball.sh b/tests/functional/tarball.sh index ab357ac78..4d8945625 100755 --- a/tests/functional/tarball.sh +++ b/tests/functional/tarball.sh @@ -83,3 +83,20 @@ path="$(nix flake prefetch --json "tarball+file://$(pwd)/tree.tar.gz" | jq -r .s [[ $(cat "$path/a/zzz") = bar ]] [[ $(cat "$path/c/aap") = bar ]] [[ $(cat "$path/fnord") = bar ]] + +# Test a tarball that has multiple top-level directories. +rm -rf "$TEST_ROOT/tar_root" +mkdir -p "$TEST_ROOT/tar_root" "$TEST_ROOT/tar_root/foo" "$TEST_ROOT/tar_root/bar" +tar cvf "$TEST_ROOT/tar.tar" -C "$TEST_ROOT/tar_root" . +path="$(nix flake prefetch --json "tarball+file://$TEST_ROOT/tar.tar" | jq -r .storePath)" +[[ -d "$path/foo" ]] +[[ -d "$path/bar" ]] + +# Test a tarball that has a single regular file. +rm -rf "$TEST_ROOT/tar_root" +mkdir -p "$TEST_ROOT/tar_root" +echo bar > "$TEST_ROOT/tar_root/foo" +chmod +x "$TEST_ROOT/tar_root/foo" +tar cvf "$TEST_ROOT/tar.tar" -C "$TEST_ROOT/tar_root" . +path="$(nix flake prefetch --refresh --json "tarball+file://$TEST_ROOT/tar.tar" | jq -r .storePath)" +[[ $(cat "$path/foo") = bar ]] diff --git a/tests/functional/test-infra.sh b/tests/functional/test-infra.sh index 37322b356..d02a11b46 100755 --- a/tests/functional/test-infra.sh +++ b/tests/functional/test-infra.sh @@ -13,6 +13,25 @@ expect 1 false # `expect` will fail when we get it wrong expect 1 expect 0 false +function ret() { + return $1 +} + +# `expect` can call functions, not just executables +expect 0 ret 0 +expect 1 ret 1 + +# `expect` supports negative exit codes +expect -1 ret -1 + +# or high positive ones, equivalent to negative ones +expect 255 ret 255 +expect 255 ret -1 +expect -1 ret 255 + +# but it doesn't confuse negative exit codes with positive ones +expect 1 expect -10 ret 10 + noisyTrue () { echo YAY! >&2 true @@ -69,6 +88,10 @@ funBang () { expect 1 funBang unset funBang +# callerPrefix can be used by the test framework to improve error messages +# it reports about our call site here +echo "<[$(callerPrefix)]>" | grepQuiet -F "<[test-infra.sh:$LINENO: ]>" + # `grep -v -q` is not what we want for exit codes, but `grepInverse` is # Avoid `grep -v -q`. The following line proves the point, and if it fails, # we'll know that `grep` had a breaking change or `-v -q` may not be portable. @@ -85,3 +108,12 @@ unset res res=$(set -eu -o pipefail; echo foo | expect 1 grepQuietInverse foo | wc -c) (( res == 0 )) unset res + +# `grepQuiet` does not allow newlines in its arguments, because grep quietly +# treats them as multiple queries. +{ echo foo; echo bar; } | expectStderr -101 grepQuiet $'foo\nbar' \ + | grepQuiet -E 'test-infra\.sh:[0-9]+: in call to grepQuiet: newline not allowed in arguments; grep would try each line individually as if connected by an OR operator' + +# We took the blue pill and woke up in a world where `grep` is moderately safe. +expectStderr -101 grep $'foo\nbar' \ + | grepQuiet -E 'test-infra\.sh:[0-9]+: in call to grep: newline not allowed in arguments; grep would try each line individually as if connected by an OR operator' diff --git a/tests/nixos/remote-builds.nix b/tests/nixos/remote-builds.nix index 1661203ec..8ddf6ad02 100644 --- a/tests/nixos/remote-builds.nix +++ b/tests/nixos/remote-builds.nix @@ -104,7 +104,15 @@ in builder.succeed("mkdir -p -m 700 /root/.ssh") builder.copy_from_host("key.pub", "/root/.ssh/authorized_keys") builder.wait_for_unit("sshd") - client.succeed(f"ssh -o StrictHostKeyChecking=no {builder.name} 'echo hello world'") + # Make sure the builder can handle our login correctly + builder.wait_for_unit("multi-user.target") + # Make sure there's no funny business on the client either + # (should not be necessary, but we have reason to be careful) + client.wait_for_unit("multi-user.target") + client.succeed(f""" + ssh -o StrictHostKeyChecking=no {builder.name} \ + 'echo hello world on $(hostname)' >&2 + """) # Perform a build and check that it was performed on the builder. out = client.succeed( diff --git a/tests/unit/libexpr-support/meson.build b/tests/unit/libexpr-support/meson.build index 705672204..4f50478aa 100644 --- a/tests/unit/libexpr-support/meson.build +++ b/tests/unit/libexpr-support/meson.build @@ -27,6 +27,8 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') +subdir('build-utils-meson/threads') + rapidcheck = dependency('rapidcheck') deps_public += rapidcheck diff --git a/tests/unit/libexpr-support/package.nix b/tests/unit/libexpr-support/package.nix index 0c966c55a..f53aa842f 100644 --- a/tests/unit/libexpr-support/package.nix +++ b/tests/unit/libexpr-support/package.nix @@ -66,12 +66,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/tests/unit/libexpr/main.cc b/tests/unit/libexpr/main.cc index cf7fcf5a3..e3412d9ef 100644 --- a/tests/unit/libexpr/main.cc +++ b/tests/unit/libexpr/main.cc @@ -34,6 +34,9 @@ int main (int argc, char **argv) { setEnv("_NIX_TEST_NO_SANDBOX", "1"); #endif + // For pipe operator tests in trivial.cc + experimentalFeatureSettings.set("experimental-features", "pipe-operators"); + ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } diff --git a/tests/unit/libexpr/meson.build b/tests/unit/libexpr/meson.build index ee35258cf..21c321334 100644 --- a/tests/unit/libexpr/meson.build +++ b/tests/unit/libexpr/meson.build @@ -25,6 +25,8 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') +subdir('build-utils-meson/threads') + subdir('build-utils-meson/export-all-symbols') rapidcheck = dependency('rapidcheck') diff --git a/tests/unit/libexpr/package.nix b/tests/unit/libexpr/package.nix index 6b7e12c4a..e70ed7836 100644 --- a/tests/unit/libexpr/package.nix +++ b/tests/unit/libexpr/package.nix @@ -1,4 +1,5 @@ { lib +, buildPackages , stdenv , mkMesonDerivation , releaseTools @@ -41,8 +42,6 @@ mkMesonDerivation (finalAttrs: { (fileset.fileFilter (file: file.hasExt "hh") ./.) ]; - outputs = [ "out" "dev" ]; - nativeBuildInputs = [ meson ninja @@ -72,28 +71,28 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; passthru = { tests = { run = runCommand "${finalAttrs.pname}-run" { - } '' - PATH="${lib.makeBinPath [ finalAttrs.finalPackage ]}:$PATH" + meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages; + } (lib.optionalString stdenv.hostPlatform.isWindows '' + export HOME="$PWD/home-dir" + mkdir -p "$HOME" + '' + '' export _NIX_TEST_UNIT_DATA=${resolvePath ./data} - nix-expr-tests + ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} touch $out - ''; + ''); }; }; meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; + mainProgram = finalAttrs.pname + stdenv.hostPlatform.extensions.executable; }; }) diff --git a/tests/unit/libexpr/trivial.cc b/tests/unit/libexpr/trivial.cc index 61ea71a0f..e455a571b 100644 --- a/tests/unit/libexpr/trivial.cc +++ b/tests/unit/libexpr/trivial.cc @@ -182,6 +182,60 @@ namespace nix { ASSERT_THAT(v, IsIntEq(15)); } + TEST_F(TrivialExpressionTest, forwardPipe) { + auto v = eval("1 |> builtins.add 2 |> builtins.mul 3"); + ASSERT_THAT(v, IsIntEq(9)); + } + + TEST_F(TrivialExpressionTest, backwardPipe) { + auto v = eval("builtins.add 1 <| builtins.mul 2 <| 3"); + ASSERT_THAT(v, IsIntEq(7)); + } + + TEST_F(TrivialExpressionTest, forwardPipeEvaluationOrder) { + auto v = eval("1 |> null |> (x: 2)"); + ASSERT_THAT(v, IsIntEq(2)); + } + + TEST_F(TrivialExpressionTest, backwardPipeEvaluationOrder) { + auto v = eval("(x: 1) <| null <| 2"); + ASSERT_THAT(v, IsIntEq(1)); + } + + TEST_F(TrivialExpressionTest, differentPipeOperatorsDoNotAssociate) { + ASSERT_THROW(eval("(x: 1) <| 2 |> (x: 3)"), ParseError); + } + + TEST_F(TrivialExpressionTest, differentPipeOperatorsParensLeft) { + auto v = eval("((x: 1) <| 2) |> (x: 3)"); + ASSERT_THAT(v, IsIntEq(3)); + } + + TEST_F(TrivialExpressionTest, differentPipeOperatorsParensRight) { + auto v = eval("(x: 1) <| (2 |> (x: 3))"); + ASSERT_THAT(v, IsIntEq(1)); + } + + TEST_F(TrivialExpressionTest, forwardPipeLowestPrecedence) { + auto v = eval("false -> true |> (x: !x)"); + ASSERT_THAT(v, IsFalse()); + } + + TEST_F(TrivialExpressionTest, backwardPipeLowestPrecedence) { + auto v = eval("(x: !x) <| false -> true"); + ASSERT_THAT(v, IsFalse()); + } + + TEST_F(TrivialExpressionTest, forwardPipeStrongerThanElse) { + auto v = eval("if true then 1 else 2 |> 3"); + ASSERT_THAT(v, IsIntEq(1)); + } + + TEST_F(TrivialExpressionTest, backwardPipeStrongerThanElse) { + auto v = eval("if true then 1 else 2 <| 3"); + ASSERT_THAT(v, IsIntEq(1)); + } + TEST_F(TrivialExpressionTest, bindOr) { auto v = eval("{ or = 1; }"); ASSERT_THAT(v, IsAttrsOfSize(1)); diff --git a/tests/unit/libfetchers/git-utils.cc b/tests/unit/libfetchers/git-utils.cc index d3547ec6a..de5110cc3 100644 --- a/tests/unit/libfetchers/git-utils.cc +++ b/tests/unit/libfetchers/git-utils.cc @@ -77,7 +77,7 @@ TEST_F(GitUtilsTest, sink_basic) // sink->createHardlink("foo-1.1/links/foo-2", CanonPath("foo-1.1/hello")); - auto result = sink->sync(); + auto result = repo->dereferenceSingletonDirectory(sink->sync()); auto accessor = repo->getAccessor(result, false); auto entries = accessor->readDirectory(CanonPath::root); ASSERT_EQ(entries.size(), 5); @@ -103,7 +103,7 @@ TEST_F(GitUtilsTest, sink_hardlink) sink->createHardlink(CanonPath("foo-1.1/link"), CanonPath("hello")); FAIL() << "Expected an exception"; } catch (const nix::Error & e) { - ASSERT_THAT(e.msg(), testing::HasSubstr("invalid hard link target")); + ASSERT_THAT(e.msg(), testing::HasSubstr("cannot find hard link target")); ASSERT_THAT(e.msg(), testing::HasSubstr("/hello")); ASSERT_THAT(e.msg(), testing::HasSubstr("foo-1.1/link")); } diff --git a/tests/unit/libfetchers/meson.build b/tests/unit/libfetchers/meson.build index d2de93829..dc9818e27 100644 --- a/tests/unit/libfetchers/meson.build +++ b/tests/unit/libfetchers/meson.build @@ -24,6 +24,8 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') +subdir('build-utils-meson/threads') + subdir('build-utils-meson/export-all-symbols') rapidcheck = dependency('rapidcheck') diff --git a/tests/unit/libfetchers/package.nix b/tests/unit/libfetchers/package.nix index 9522f9639..ad512f562 100644 --- a/tests/unit/libfetchers/package.nix +++ b/tests/unit/libfetchers/package.nix @@ -1,4 +1,5 @@ { lib +, buildPackages , stdenv , mkMesonDerivation , releaseTools @@ -40,8 +41,6 @@ mkMesonDerivation (finalAttrs: { (fileset.fileFilter (file: file.hasExt "hh") ./.) ]; - outputs = [ "out" "dev" ]; - nativeBuildInputs = [ meson ninja @@ -70,28 +69,28 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; passthru = { tests = { run = runCommand "${finalAttrs.pname}-run" { - } '' - PATH="${lib.makeBinPath [ finalAttrs.finalPackage ]}:$PATH" + meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages; + } (lib.optionalString stdenv.hostPlatform.isWindows '' + export HOME="$PWD/home-dir" + mkdir -p "$HOME" + '' + '' export _NIX_TEST_UNIT_DATA=${resolvePath ./data} - nix-fetchers-tests + ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} touch $out - ''; + ''); }; }; meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; + mainProgram = finalAttrs.pname + stdenv.hostPlatform.extensions.executable; }; }) diff --git a/tests/unit/libflake/meson.build b/tests/unit/libflake/meson.build index 2d6bbca0f..c022d7f41 100644 --- a/tests/unit/libflake/meson.build +++ b/tests/unit/libflake/meson.build @@ -24,6 +24,8 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') +subdir('build-utils-meson/threads') + subdir('build-utils-meson/export-all-symbols') rapidcheck = dependency('rapidcheck') diff --git a/tests/unit/libflake/package.nix b/tests/unit/libflake/package.nix index 859bc49d0..0d63d2ff7 100644 --- a/tests/unit/libflake/package.nix +++ b/tests/unit/libflake/package.nix @@ -1,4 +1,5 @@ { lib +, buildPackages , stdenv , mkMesonDerivation , releaseTools @@ -40,8 +41,6 @@ mkMesonDerivation (finalAttrs: { (fileset.fileFilter (file: file.hasExt "hh") ./.) ]; - outputs = [ "out" "dev" ]; - nativeBuildInputs = [ meson ninja @@ -70,28 +69,28 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; passthru = { tests = { run = runCommand "${finalAttrs.pname}-run" { - } '' - PATH="${lib.makeBinPath [ finalAttrs.finalPackage ]}:$PATH" + meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages; + } (lib.optionalString stdenv.hostPlatform.isWindows '' + export HOME="$PWD/home-dir" + mkdir -p "$HOME" + '' + '' export _NIX_TEST_UNIT_DATA=${resolvePath ./data} - nix-flake-tests + ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} touch $out - ''; + ''); }; }; meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; + mainProgram = finalAttrs.pname + stdenv.hostPlatform.extensions.executable; }; }) diff --git a/tests/unit/libstore-support/meson.build b/tests/unit/libstore-support/meson.build index ddb067c1b..f09d26a31 100644 --- a/tests/unit/libstore-support/meson.build +++ b/tests/unit/libstore-support/meson.build @@ -25,6 +25,8 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') +subdir('build-utils-meson/threads') + rapidcheck = dependency('rapidcheck') deps_public += rapidcheck diff --git a/tests/unit/libstore-support/package.nix b/tests/unit/libstore-support/package.nix index cb15cdd5f..f512db3ee 100644 --- a/tests/unit/libstore-support/package.nix +++ b/tests/unit/libstore-support/package.nix @@ -66,12 +66,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/tests/unit/libstore/http-binary-cache-store.cc b/tests/unit/libstore/http-binary-cache-store.cc new file mode 100644 index 000000000..1e415f625 --- /dev/null +++ b/tests/unit/libstore/http-binary-cache-store.cc @@ -0,0 +1,21 @@ +#include + +#include "http-binary-cache-store.hh" + +namespace nix { + +TEST(HttpBinaryCacheStore, constructConfig) +{ + HttpBinaryCacheStoreConfig config{"http", "foo.bar.baz", {}}; + + EXPECT_EQ(config.cacheUri, "http://foo.bar.baz"); +} + +TEST(HttpBinaryCacheStore, constructConfigNoTrailingSlash) +{ + HttpBinaryCacheStoreConfig config{"https", "foo.bar.baz/a/b/", {}}; + + EXPECT_EQ(config.cacheUri, "https://foo.bar.baz/a/b"); +} + +} // namespace nix diff --git a/tests/unit/libstore/local-binary-cache-store.cc b/tests/unit/libstore/local-binary-cache-store.cc new file mode 100644 index 000000000..2e840228d --- /dev/null +++ b/tests/unit/libstore/local-binary-cache-store.cc @@ -0,0 +1,14 @@ +#include + +#include "local-binary-cache-store.hh" + +namespace nix { + +TEST(LocalBinaryCacheStore, constructConfig) +{ + LocalBinaryCacheStoreConfig config{"local", "/foo/bar/baz", {}}; + + EXPECT_EQ(config.binaryCacheDir, "/foo/bar/baz"); +} + +} // namespace nix diff --git a/tests/unit/libstore/local-overlay-store.cc b/tests/unit/libstore/local-overlay-store.cc new file mode 100644 index 000000000..b34ca9237 --- /dev/null +++ b/tests/unit/libstore/local-overlay-store.cc @@ -0,0 +1,34 @@ +// FIXME: Odd failures for templates that are causing the PR to break +// for now with discussion with @Ericson2314 to comment out. +#if 0 +# include + +# include "local-overlay-store.hh" + +namespace nix { + +TEST(LocalOverlayStore, constructConfig_rootQueryParam) +{ + LocalOverlayStoreConfig config{ + "local-overlay", + "", + { + { + "root", + "/foo/bar", + }, + }, + }; + + EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); +} + +TEST(LocalOverlayStore, constructConfig_rootPath) +{ + LocalOverlayStoreConfig config{"local-overlay", "/foo/bar", {}}; + + EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); +} + +} // namespace nix +#endif diff --git a/tests/unit/libstore/local-store.cc b/tests/unit/libstore/local-store.cc new file mode 100644 index 000000000..abc3ea796 --- /dev/null +++ b/tests/unit/libstore/local-store.cc @@ -0,0 +1,40 @@ +// FIXME: Odd failures for templates that are causing the PR to break +// for now with discussion with @Ericson2314 to comment out. +#if 0 +# include + +# include "local-store.hh" + +// Needed for template specialisations. This is not good! When we +// overhaul how store configs work, this should be fixed. +# include "args.hh" +# include "config-impl.hh" +# include "abstract-setting-to-json.hh" + +namespace nix { + +TEST(LocalStore, constructConfig_rootQueryParam) +{ + LocalStoreConfig config{ + "local", + "", + { + { + "root", + "/foo/bar", + }, + }, + }; + + EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); +} + +TEST(LocalStore, constructConfig_rootPath) +{ + LocalStoreConfig config{"local", "/foo/bar", {}}; + + EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); +} + +} // namespace nix +#endif diff --git a/tests/unit/libstore/meson.build b/tests/unit/libstore/meson.build index 41b2fb0ab..3b36cd62f 100644 --- a/tests/unit/libstore/meson.build +++ b/tests/unit/libstore/meson.build @@ -25,6 +25,8 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') +subdir('build-utils-meson/threads') + subdir('build-utils-meson/export-all-symbols') sqlite = dependency('sqlite3', 'sqlite', version : '>=3.6.19') @@ -58,7 +60,11 @@ sources = files( 'derivation.cc', 'derived-path.cc', 'downstream-placeholder.cc', + 'http-binary-cache-store.cc', 'legacy-ssh-store.cc', + 'local-binary-cache-store.cc', + 'local-overlay-store.cc', + 'local-store.cc', 'machines.cc', 'nar-info-disk-cache.cc', 'nar-info.cc', @@ -67,9 +73,11 @@ sources = files( 'path-info.cc', 'path.cc', 'references.cc', + 's3-binary-cache-store.cc', 'serve-protocol.cc', 'ssh-store.cc', 'store-reference.cc', + 'uds-remote-store.cc', 'worker-protocol.cc', ) diff --git a/tests/unit/libstore/package.nix b/tests/unit/libstore/package.nix index efffd0063..7560a5b79 100644 --- a/tests/unit/libstore/package.nix +++ b/tests/unit/libstore/package.nix @@ -1,4 +1,5 @@ { lib +, buildPackages , stdenv , mkMesonDerivation , releaseTools @@ -42,8 +43,6 @@ mkMesonDerivation (finalAttrs: { (fileset.fileFilter (file: file.hasExt "hh") ./.) ]; - outputs = [ "out" "dev" ]; - nativeBuildInputs = [ meson ninja @@ -74,12 +73,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; passthru = { @@ -94,17 +89,22 @@ mkMesonDerivation (finalAttrs: { ../../functional/derivation ]; }; - in runCommand "${finalAttrs.pname}-run" {} '' - PATH="${lib.makeBinPath [ finalAttrs.finalPackage ]}:$PATH" + in runCommand "${finalAttrs.pname}-run" { + meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages; + } (lib.optionalString stdenv.hostPlatform.isWindows '' + export HOME="$PWD/home-dir" + mkdir -p "$HOME" + '' + '' export _NIX_TEST_UNIT_DATA=${data + "/unit/libstore/data"} - nix-store-tests + ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} touch $out - ''; + ''); }; }; meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; + mainProgram = finalAttrs.pname + stdenv.hostPlatform.extensions.executable; }; }) diff --git a/tests/unit/libstore/s3-binary-cache-store.cc b/tests/unit/libstore/s3-binary-cache-store.cc new file mode 100644 index 000000000..7aa5f2f2c --- /dev/null +++ b/tests/unit/libstore/s3-binary-cache-store.cc @@ -0,0 +1,18 @@ +#if ENABLE_S3 + +# include + +# include "s3-binary-cache-store.hh" + +namespace nix { + +TEST(S3BinaryCacheStore, constructConfig) +{ + S3BinaryCacheStoreConfig config{"s3", "foobar", {}}; + + EXPECT_EQ(config.bucketName, "foobar"); +} + +} // namespace nix + +#endif diff --git a/tests/unit/libstore/ssh-store.cc b/tests/unit/libstore/ssh-store.cc index 7010ad157..b853a5f1f 100644 --- a/tests/unit/libstore/ssh-store.cc +++ b/tests/unit/libstore/ssh-store.cc @@ -1,6 +1,9 @@ -#include +// FIXME: Odd failures for templates that are causing the PR to break +// for now with discussion with @Ericson2314 to comment out. +#if 0 +# include -#include "ssh-store.hh" +# include "ssh-store.hh" namespace nix { @@ -15,7 +18,9 @@ TEST(SSHStore, constructConfig) // TODO #11106, no more split on space "foo bar", }, - }}; + }, + }; + EXPECT_EQ( config.remoteProgram.get(), (Strings{ @@ -23,4 +28,28 @@ TEST(SSHStore, constructConfig) "bar", })); } + +TEST(MountedSSHStore, constructConfig) +{ + MountedSSHStoreConfig config{ + "mounted-ssh", + "localhost", + StoreConfig::Params{ + { + "remote-program", + // TODO #11106, no more split on space + "foo bar", + }, + }, + }; + + EXPECT_EQ( + config.remoteProgram.get(), + (Strings{ + "foo", + "bar", + })); } + +} +#endif diff --git a/tests/unit/libstore/uds-remote-store.cc b/tests/unit/libstore/uds-remote-store.cc new file mode 100644 index 000000000..5ccb20871 --- /dev/null +++ b/tests/unit/libstore/uds-remote-store.cc @@ -0,0 +1,23 @@ +// FIXME: Odd failures for templates that are causing the PR to break +// for now with discussion with @Ericson2314 to comment out. +#if 0 +# include + +# include "uds-remote-store.hh" + +namespace nix { + +TEST(UDSRemoteStore, constructConfig) +{ + UDSRemoteStoreConfig config{"unix", "/tmp/socket", {}}; + + EXPECT_EQ(config.path, "/tmp/socket"); +} + +TEST(UDSRemoteStore, constructConfigWrongScheme) +{ + EXPECT_THROW(UDSRemoteStoreConfig("http", "/tmp/socket", {}), UsageError); +} + +} // namespace nix +#endif diff --git a/tests/unit/libstore/worker-protocol.cc b/tests/unit/libstore/worker-protocol.cc index c15120010..bbea9ed75 100644 --- a/tests/unit/libstore/worker-protocol.cc +++ b/tests/unit/libstore/worker-protocol.cc @@ -658,15 +658,15 @@ TEST_F(WorkerProtoTest, handshake_log) FdSink out { toServer.writeSide.get() }; FdSource in0 { toClient.readSide.get() }; TeeSource in { in0, toClientLog }; - clientResult = WorkerProto::BasicClientConnection::handshake( - out, in, defaultVersion); + clientResult = std::get<0>(WorkerProto::BasicClientConnection::handshake( + out, in, defaultVersion, {})); }); { FdSink out { toClient.writeSide.get() }; FdSource in { toServer.readSide.get() }; WorkerProto::BasicServerConnection::handshake( - out, in, defaultVersion); + out, in, defaultVersion, {}); }; thread.join(); @@ -675,6 +675,33 @@ TEST_F(WorkerProtoTest, handshake_log) }); } +TEST_F(WorkerProtoTest, handshake_features) +{ + Pipe toClient, toServer; + toClient.create(); + toServer.create(); + + std::tuple> clientResult; + + auto clientThread = std::thread([&]() { + FdSink out { toServer.writeSide.get() }; + FdSource in { toClient.readSide.get() }; + clientResult = WorkerProto::BasicClientConnection::handshake( + out, in, 123, {"bar", "aap", "mies", "xyzzy"}); + }); + + FdSink out { toClient.writeSide.get() }; + FdSource in { toServer.readSide.get() }; + auto daemonResult = WorkerProto::BasicServerConnection::handshake( + out, in, 456, {"foo", "bar", "xyzzy"}); + + clientThread.join(); + + EXPECT_EQ(clientResult, daemonResult); + EXPECT_EQ(std::get<0>(clientResult), 123); + EXPECT_EQ(std::get<1>(clientResult), std::set({"bar", "xyzzy"})); +} + /// Has to be a `BufferedSink` for handshake. struct NullBufferedSink : BufferedSink { void writeUnbuffered(std::string_view data) override { } @@ -686,8 +713,8 @@ TEST_F(WorkerProtoTest, handshake_client_replay) NullBufferedSink nullSink; StringSource in { toClientLog }; - auto clientResult = WorkerProto::BasicClientConnection::handshake( - nullSink, in, defaultVersion); + auto clientResult = std::get<0>(WorkerProto::BasicClientConnection::handshake( + nullSink, in, defaultVersion, {})); EXPECT_EQ(clientResult, defaultVersion); }); @@ -705,13 +732,13 @@ TEST_F(WorkerProtoTest, handshake_client_truncated_replay_throws) if (len < 8) { EXPECT_THROW( WorkerProto::BasicClientConnection::handshake( - nullSink, in, defaultVersion), + nullSink, in, defaultVersion, {}), EndOfFile); } else { // Not sure why cannot keep on checking for `EndOfFile`. EXPECT_THROW( WorkerProto::BasicClientConnection::handshake( - nullSink, in, defaultVersion), + nullSink, in, defaultVersion, {}), Error); } } @@ -734,17 +761,17 @@ TEST_F(WorkerProtoTest, handshake_client_corrupted_throws) // magic bytes don't match EXPECT_THROW( WorkerProto::BasicClientConnection::handshake( - nullSink, in, defaultVersion), + nullSink, in, defaultVersion, {}), Error); } else if (idx < 8 || idx >= 12) { // Number out of bounds EXPECT_THROW( WorkerProto::BasicClientConnection::handshake( - nullSink, in, defaultVersion), + nullSink, in, defaultVersion, {}), SerialisationError); } else { - auto ver = WorkerProto::BasicClientConnection::handshake( - nullSink, in, defaultVersion); + auto ver = std::get<0>(WorkerProto::BasicClientConnection::handshake( + nullSink, in, defaultVersion, {})); // `std::min` of this and the other version saves us EXPECT_EQ(ver, defaultVersion); } diff --git a/tests/unit/libutil-support/meson.build b/tests/unit/libutil-support/meson.build index 7d0e9c2fc..6be4972c6 100644 --- a/tests/unit/libutil-support/meson.build +++ b/tests/unit/libutil-support/meson.build @@ -23,6 +23,8 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') +subdir('build-utils-meson/threads') + rapidcheck = dependency('rapidcheck') deps_public += rapidcheck diff --git a/tests/unit/libutil-support/package.nix b/tests/unit/libutil-support/package.nix index fdecdec72..1665804cb 100644 --- a/tests/unit/libutil-support/package.nix +++ b/tests/unit/libutil-support/package.nix @@ -64,12 +64,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/tests/unit/libutil-support/tests/string_callback.cc b/tests/unit/libutil-support/tests/string_callback.cc index 2d0e0dad0..7a13bd4ff 100644 --- a/tests/unit/libutil-support/tests/string_callback.cc +++ b/tests/unit/libutil-support/tests/string_callback.cc @@ -2,9 +2,10 @@ namespace nix::testing { -void observe_string_cb(const char * start, unsigned int n, std::string * user_data) +void observe_string_cb(const char * start, unsigned int n, void * user_data) { - *user_data = std::string(start); + auto user_data_casted = reinterpret_cast(user_data); + *user_data_casted = std::string(start); } } diff --git a/tests/unit/libutil-support/tests/string_callback.hh b/tests/unit/libutil-support/tests/string_callback.hh index a02ea3a1b..9a7e8d85d 100644 --- a/tests/unit/libutil-support/tests/string_callback.hh +++ b/tests/unit/libutil-support/tests/string_callback.hh @@ -3,14 +3,13 @@ namespace nix::testing { -void observe_string_cb(const char * start, unsigned int n, std::string * user_data); +void observe_string_cb(const char * start, unsigned int n, void * user_data); inline void * observe_string_cb_data(std::string & out) { return (void *) &out; }; -#define OBSERVE_STRING(str) \ - (nix_get_string_callback) nix::testing::observe_string_cb, nix::testing::observe_string_cb_data(str) +#define OBSERVE_STRING(str) nix::testing::observe_string_cb, nix::testing::observe_string_cb_data(str) } diff --git a/tests/unit/libutil/meson.build b/tests/unit/libutil/meson.build index 4f055cabd..7f024e6f2 100644 --- a/tests/unit/libutil/meson.build +++ b/tests/unit/libutil/meson.build @@ -25,6 +25,8 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') +subdir('build-utils-meson/threads') + subdir('build-utils-meson/export-all-symbols') rapidcheck = dependency('rapidcheck') diff --git a/tests/unit/libutil/package.nix b/tests/unit/libutil/package.nix index ad5ff7d1e..2fce5bfa8 100644 --- a/tests/unit/libutil/package.nix +++ b/tests/unit/libutil/package.nix @@ -1,4 +1,5 @@ { lib +, buildPackages , stdenv , mkMesonDerivation , releaseTools @@ -40,8 +41,6 @@ mkMesonDerivation (finalAttrs: { (fileset.fileFilter (file: file.hasExt "hh") ./.) ]; - outputs = [ "out" "dev" ]; - nativeBuildInputs = [ meson ninja @@ -71,28 +70,28 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; passthru = { tests = { run = runCommand "${finalAttrs.pname}-run" { - } '' - PATH="${lib.makeBinPath [ finalAttrs.finalPackage ]}:$PATH" + meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages; + } (lib.optionalString stdenv.hostPlatform.isWindows '' + export HOME="$PWD/home-dir" + mkdir -p "$HOME" + '' + '' export _NIX_TEST_UNIT_DATA=${./data} - nix-util-tests + ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} touch $out - ''; + ''); }; }; meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; + mainProgram = finalAttrs.pname + stdenv.hostPlatform.extensions.executable; }; }) diff --git a/tests/unit/libutil/tests.cc b/tests/unit/libutil/tests.cc index 8a3ca8561..2b73d323b 100644 --- a/tests/unit/libutil/tests.cc +++ b/tests/unit/libutil/tests.cc @@ -17,6 +17,10 @@ # define FS_ROOT FS_SEP #endif +#ifndef PATH_MAX +# define PATH_MAX 4096 +#endif + namespace nix { /* ----------- tests for util.hh ------------------------------------------------*/