${name}
${name} ${listArgs args}
${name}${listArgs args}
; } - > $ nix-instantiate --eval --expr '{ a = ; }' - > { a = ; } - > ``` - > - > For human-readable output, `nix eval` (experimental) is more informative: - > - > ```console - > $ nix-instantiate --eval --expr 'a: a' - > - > $ nix eval --expr 'a: a' - > «lambda @ «string»:1:1» - > ``` - > - > For machine-readable output, the `--xml` option produces unambiguous - > output: - > - > ```console - > $ nix-instantiate --eval --xml --expr '{ foo = ; }' - > - > - > - > - > - > - > - > - > ``` + Just parse the input files, and print their abstract syntax trees on + standard output as a Nix expression. - - `--find-file`\ - Look up the given files in Nix’s search path (as specified by the - `NIX_PATH` environment variable). If found, print the corresponding - absolute paths on standard output. For instance, if `NIX_PATH` is - `nixpkgs=/home/alice/nixpkgs`, then `nix-instantiate --find-file - nixpkgs/default.nix` will print `/home/alice/nixpkgs/default.nix`. +- `--eval` - - `--strict`\ - When used with `--eval`, recursively evaluate list elements and - attributes. Normally, such sub-expressions are left unevaluated - (since the Nix language is lazy). + Just parse and evaluate the input files, and print the resulting + values on standard output. No instantiation of store derivations + takes place. - > **Warning** - > - > This option can cause non-termination, because lazy data - > structures can be infinitely large. + > **Warning** + > + > This option produces output which can be parsed as a Nix expression which + > will produce a different result than the input expression when evaluated. + > For example, these two Nix expressions print the same result despite + > having different meaning: + > + > ```console + > $ nix-instantiate --eval --expr '{ a = {}; }' + > { a = ; } + > $ nix-instantiate --eval --expr '{ a = ; }' + > { a = ; } + > ``` + > + > For human-readable output, `nix eval` (experimental) is more informative: + > + > ```console + > $ nix-instantiate --eval --expr 'a: a' + > + > $ nix eval --expr 'a: a' + > «lambda @ «string»:1:1» + > ``` + > + > For machine-readable output, the `--xml` option produces unambiguous + > output: + > + > ```console + > $ nix-instantiate --eval --xml --expr '{ foo = ; }' + > + > + > + > + > + > + > + > + > ``` - - `--json`\ - When used with `--eval`, print the resulting value as an JSON - representation of the abstract syntax tree rather than as a Nix expression. +- `--find-file` - - `--xml`\ - When used with `--eval`, print the resulting value as an XML - representation of the abstract syntax tree rather than as a Nix expression. - The schema is the same as that used by the [`toXML` - built-in](../language/builtins.md). + Look up the given files in Nix’s search path (as specified by the + `NIX_PATH` environment variable). If found, print the corresponding + absolute paths on standard output. For instance, if `NIX_PATH` is + `nixpkgs=/home/alice/nixpkgs`, then `nix-instantiate --find-file + nixpkgs/default.nix` will print `/home/alice/nixpkgs/default.nix`. - - `--read-write-mode`\ - When used with `--eval`, perform evaluation in read/write mode so - nix language features that require it will still work (at the cost - of needing to do instantiation of every evaluated derivation). If - this option is not enabled, there may be uninstantiated store paths - in the final output. +- `--strict` + + When used with `--eval`, recursively evaluate list elements and + attributes. Normally, such sub-expressions are left unevaluated + (since the Nix language is lazy). + + > **Warning** + > + > This option can cause non-termination, because lazy data + > structures can be infinitely large. + +- `--json` + + When used with `--eval`, print the resulting value as an JSON + representation of the abstract syntax tree rather than as a Nix expression. + +- `--xml` + + When used with `--eval`, print the resulting value as an XML + representation of the abstract syntax tree rather than as a Nix expression. + The schema is the same as that used by the [`toXML` + built-in](../language/builtins.md). + +- `--read-write-mode` + + When used with `--eval`, perform evaluation in read/write mode so + nix language features that require it will still work (at the cost + of needing to do instantiation of every evaluated derivation). If + this option is not enabled, there may be uninstantiated store paths + in the final output. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-prefetch-url.md b/doc/manual/src/command-ref/nix-prefetch-url.md index 45ef01e02..ffab94b8a 100644 --- a/doc/manual/src/command-ref/nix-prefetch-url.md +++ b/doc/manual/src/command-ref/nix-prefetch-url.md @@ -39,27 +39,32 @@ the path of the downloaded file in the Nix store is also printed. # Options - - `--type` *hashAlgo*\ - Use the specified cryptographic hash algorithm, - which can be one of `md5`, `sha1`, `sha256`, and `sha512`. - The default is `sha256`. +- `--type` *hashAlgo* - - `--print-path`\ - Print the store path of the downloaded file on standard output. + Use the specified cryptographic hash algorithm, + which can be one of `md5`, `sha1`, `sha256`, and `sha512`. + The default is `sha256`. - - `--unpack`\ - Unpack the archive (which must be a tarball or zip file) and add the - result to the Nix store. The resulting hash can be used with - functions such as Nixpkgs’s `fetchzip` or `fetchFromGitHub`. +- `--print-path` - - `--executable`\ - Set the executable bit on the downloaded file. + Print the store path of the downloaded file on standard output. - - `--name` *name*\ - Override the name of the file in the Nix store. By default, this is - `hash-basename`, where *basename* is the last component of *url*. - Overriding the name is necessary when *basename* contains characters - that are not allowed in Nix store paths. +- `--unpack` + + Unpack the archive (which must be a tarball or zip file) and add the + result to the Nix store. The resulting hash can be used with + functions such as Nixpkgs’s `fetchzip` or `fetchFromGitHub`. + +- `--executable` + + Set the executable bit on the downloaded file. + +- `--name` *name* + + Override the name of the file in the Nix store. By default, this is + `hash-basename`, where *basename* is the last component of *url*. + Overriding the name is necessary when *basename* contains characters + that are not allowed in Nix store paths. # Examples diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index 1eaf3c36a..69a711bd5 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -60,55 +60,63 @@ All options not listed here are passed to `nix-store --realise`, except for `--arg` and `--attr` / `-A` which are passed to `nix-instantiate`. - - `--command` *cmd*\ - In the environment of the derivation, run the shell command *cmd*. - This command is executed in an interactive shell. (Use `--run` to - use a non-interactive shell instead.) However, a call to `exit` is - implicitly added to the command, so the shell will exit after - running the command. To prevent this, add `return` at the end; - e.g. `--command "echo Hello; return"` will print `Hello` and then - drop you into the interactive shell. This can be useful for doing - any additional initialisation. +- `--command` *cmd* - - `--run` *cmd*\ - Like `--command`, but executes the command in a non-interactive - shell. This means (among other things) that if you hit Ctrl-C while - the command is running, the shell exits. + In the environment of the derivation, run the shell command *cmd*. + This command is executed in an interactive shell. (Use `--run` to + use a non-interactive shell instead.) However, a call to `exit` is + implicitly added to the command, so the shell will exit after + running the command. To prevent this, add `return` at the end; + e.g. `--command "echo Hello; return"` will print `Hello` and then + drop you into the interactive shell. This can be useful for doing + any additional initialisation. - - `--exclude` *regexp*\ - Do not build any dependencies whose store path matches the regular - expression *regexp*. This option may be specified multiple times. +- `--run` *cmd* - - `--pure`\ - If this flag is specified, the environment is almost entirely - cleared before the interactive shell is started, so you get an - environment that more closely corresponds to the “real†Nix build. A - few variables, in particular `HOME`, `USER` and `DISPLAY`, are - retained. + Like `--command`, but executes the command in a non-interactive + shell. This means (among other things) that if you hit Ctrl-C while + the command is running, the shell exits. - - `--packages` / `-p` *packages*…\ - Set up an environment in which the specified packages are present. - The command line arguments are interpreted as attribute names inside - the Nix Packages collection. Thus, `nix-shell --packages libjpeg openjdk` - will start a shell in which the packages denoted by the attribute - names `libjpeg` and `openjdk` are present. +- `--exclude` *regexp* - - `-i` *interpreter*\ - The chained script interpreter to be invoked by `nix-shell`. Only - applicable in `#!`-scripts (described below). + Do not build any dependencies whose store path matches the regular + expression *regexp*. This option may be specified multiple times. - - `--keep` *name*\ - When a `--pure` shell is started, keep the listed environment - variables. +- `--pure` + + If this flag is specified, the environment is almost entirely + cleared before the interactive shell is started, so you get an + environment that more closely corresponds to the “real†Nix build. A + few variables, in particular `HOME`, `USER` and `DISPLAY`, are + retained. + +- `--packages` / `-p` *packages*… + + Set up an environment in which the specified packages are present. + The command line arguments are interpreted as attribute names inside + the Nix Packages collection. Thus, `nix-shell --packages libjpeg openjdk` + will start a shell in which the packages denoted by the attribute + names `libjpeg` and `openjdk` are present. + +- `-i` *interpreter* + + The chained script interpreter to be invoked by `nix-shell`. Only + applicable in `#!`-scripts (described below). + +- `--keep` *name* + + When a `--pure` shell is started, keep the listed environment + variables. {{#include ./opt-common.md}} # Environment variables - - `NIX_BUILD_SHELL`\ - Shell used to start the interactive environment. Defaults to the - `bash` found in ``, falling back to the `bash` found in - `PATH` if not found. +- `NIX_BUILD_SHELL` + + Shell used to start the interactive environment. Defaults to the + `bash` found in ``, falling back to the `bash` found in + `PATH` if not found. {{#include ./env-common.md}} @@ -202,14 +210,14 @@ For example, here is a Python script that depends on Python and the ```python #! /usr/bin/env nix-shell -#! nix-shell -i python --packages python pythonPackages.prettytable +#! nix-shell -i python3 --packages python3 python3Packages.prettytable import prettytable # Print a simple table. t = prettytable.PrettyTable(["N", "N^2"]) for n in range(1, 10): t.add_row([n, n * n]) -print t +print(t) ``` Similarly, the following is a Perl script that specifies that it @@ -289,3 +297,8 @@ with import {}; runCommand "dummy" { buildInputs = [ python pythonPackages.prettytable ]; } "" ``` + +The script's file name is passed as the first argument to the interpreter specified by the `-i` flag. + +Aside from the very first line, which is a directive to the operating system, the additional `#! nix-shell` lines do not need to be at the beginning of the file. +This allows wrapping them in block comments for languages where `#` does not start a comment, such as ECMAScript, Erlang, PHP, or Ruby. diff --git a/doc/manual/src/command-ref/nix-store/add-fixed.md b/doc/manual/src/command-ref/nix-store/add-fixed.md index d25db091c..bebf15026 100644 --- a/doc/manual/src/command-ref/nix-store/add-fixed.md +++ b/doc/manual/src/command-ref/nix-store/add-fixed.md @@ -16,9 +16,10 @@ public url or broke since the download expression was written. This operation has the following options: - - `--recursive`\ - Use recursive instead of flat hashing mode, used when adding - directories to the store. +- `--recursive` + + Use recursive instead of flat hashing mode, used when adding + directories to the store. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/dump.md b/doc/manual/src/command-ref/nix-store/dump.md index c2f3c42ef..3de0e27b0 100644 --- a/doc/manual/src/command-ref/nix-store/dump.md +++ b/doc/manual/src/command-ref/nix-store/dump.md @@ -1,6 +1,6 @@ # Name -`nix-store --dump` - write a single path to a Nix Archive +`nix-store --dump` - write a single path to a [Nix Archive] ## Synopsis @@ -8,7 +8,7 @@ ## Description -The operation `--dump` produces a NAR (Nix ARchive) file containing the +The operation `--dump` produces a [Nix archive](@docroot@/glossary.md#gloss-nar) (NAR) file containing the contents of the file system tree rooted at *path*. The archive is written to standard output. @@ -30,8 +30,9 @@ NAR archives support filenames of unlimited length and 64-bit file sizes. They can contain regular files, directories, and symbolic links, but not other types of files (such as device nodes). -A Nix archive can be unpacked using `nix-store ---restore`. +A Nix archive can be unpacked using [`nix-store --restore`](@docroot@/command-ref/nix-store/restore.md). + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/export.md b/doc/manual/src/command-ref/nix-store/export.md index 1bc46f53b..ba772eb43 100644 --- a/doc/manual/src/command-ref/nix-store/export.md +++ b/doc/manual/src/command-ref/nix-store/export.md @@ -1,6 +1,6 @@ # Name -`nix-store --export` - export store paths to a Nix Archive +`nix-store --export` - export store paths to a [Nix Archive] ## Synopsis @@ -8,16 +8,22 @@ ## Description -The operation `--export` writes a serialisation of the specified store -paths to standard output in a format that can be imported into another -Nix store with `nix-store --import`. This is like `nix-store ---dump`, except that the NAR archive produced by that command doesn’t -contain the necessary meta-information to allow it to be imported into -another Nix store (namely, the set of references of the path). +The operation `--export` writes a serialisation of the given [store objects](@docroot@/glossary.md#gloss-store-object) to standard output in a format that can be imported into another [Nix store](@docroot@/store/index.md) with [`nix-store --import`](./import.md). -This command does not produce a *closure* of the specified paths, so if -a store path references other store paths that are missing in the target -Nix store, the import will fail. +> **Warning** +> +> This command *does not* produce a [closure](@docroot@/glossary.md#gloss-closure) of the specified store paths. +> Trying to import a store object that refers to store paths not available in the target Nix store will fail. +> +> Use [`nix-store --query`](@docroot@/command-ref/nix-store/query.md) to obtain the closure of a store path. + +This command is different from [`nix-store --dump`](./dump.md), which produces a [Nix archive](@docroot@/glossary.md#gloss-nar) that *does not* contain the set of [references](@docroot@/glossary.md#gloss-reference) of a given store path. + +> **Note** +> +> For efficient transfer of closures to remote machines over SSH, use [`nix-copy-closure`](@docroot@/command-ref/nix-copy-closure.md). + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive {{#include ./opt-common.md}} @@ -27,15 +33,21 @@ Nix store, the import will fail. # Examples -To copy a whole closure, do something -like: - -```console -$ nix-store --export $(nix-store --query --requisites paths) > out -``` - -To import the whole closure again, run: - -```console -$ nix-store --import < out -``` +> **Example** +> +> Deploy GNU Hello to an airgapped machine via USB stick. +> +> Write the closure to the block device on a machine with internet connection: +> +> ```shell-session +> [alice@itchy]$ storePath=$(nix-build '' -I nixpkgs=channel:nixpkgs-unstable -A hello --no-out-link) +> [alice@itchy]$ nix-store --export $(nix-store --query --requisites $storePath) | sudo dd of=/dev/usb +> ``` +> +> Read the closure from the block device on the machine without internet connection: +> +> ```shell-session +> [bob@scratchy]$ hello=$(sudo dd if=/dev/usb | nix-store --import | tail -1) +> [bob@scratchy]$ $hello/bin/hello +> Hello, world! +> ``` diff --git a/doc/manual/src/command-ref/nix-store/gc.md b/doc/manual/src/command-ref/nix-store/gc.md index 7be0d559a..f432e00eb 100644 --- a/doc/manual/src/command-ref/nix-store/gc.md +++ b/doc/manual/src/command-ref/nix-store/gc.md @@ -14,30 +14,34 @@ reachable via file system references from a set of “rootsâ€, are deleted. The following suboperations may be specified: - - `--print-roots`\ - This operation prints on standard output the set of roots used by - the garbage collector. +- `--print-roots` - - `--print-live`\ - This operation prints on standard output the set of “live†store - paths, which are all the store paths reachable from the roots. Live - paths should never be deleted, since that would break consistency — - it would become possible that applications are installed that - reference things that are no longer present in the store. + This operation prints on standard output the set of roots used by + the garbage collector. - - `--print-dead`\ - This operation prints out on standard output the set of “dead†store - paths, which is just the opposite of the set of live paths: any path - in the store that is not live (with respect to the roots) is dead. +- `--print-live` + + This operation prints on standard output the set of “live†store + paths, which are all the store paths reachable from the roots. Live + paths should never be deleted, since that would break consistency — + it would become possible that applications are installed that + reference things that are no longer present in the store. + +- `--print-dead` + + This operation prints out on standard output the set of “dead†store + paths, which is just the opposite of the set of live paths: any path + in the store that is not live (with respect to the roots) is dead. By default, all unreachable paths are deleted. The following options control what gets deleted and in what order: - - `--max-freed` *bytes*\ - Keep deleting paths until at least *bytes* bytes have been deleted, - then stop. The argument *bytes* can be followed by the - multiplicative suffix `K`, `M`, `G` or `T`, denoting KiB, MiB, GiB - or TiB units. +- `--max-freed` *bytes* + + Keep deleting paths until at least *bytes* bytes have been deleted, + then stop. The argument *bytes* can be followed by the + multiplicative suffix `K`, `M`, `G` or `T`, denoting KiB, MiB, GiB + or TiB units. The behaviour of the collector is also influenced by the `keep-outputs` and `keep-derivations` settings in the Nix diff --git a/doc/manual/src/command-ref/nix-store/import.md b/doc/manual/src/command-ref/nix-store/import.md index 2711316a7..3f6b3d076 100644 --- a/doc/manual/src/command-ref/nix-store/import.md +++ b/doc/manual/src/command-ref/nix-store/import.md @@ -1,6 +1,8 @@ # Name -`nix-store --import` - import Nix Archive into the store +`nix-store --import` - import [Nix Archive] into the store + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive # Synopsis @@ -8,14 +10,34 @@ # Description -The operation `--import` reads a serialisation of a set of store paths -produced by `nix-store --export` from standard input and adds those -store paths to the Nix store. Paths that already exist in the Nix store -are ignored. If a path refers to another path that doesn’t exist in the -Nix store, the import fails. +The operation `--import` reads a serialisation of a set of [store objects](@docroot@/glossary.md#gloss-store-object) produced by [`nix-store --export`](./export.md) from standard input, and adds those store objects to the specified [Nix store](@docroot@/store/index.md). +Paths that already exist in the target Nix store are ignored. +If a path [refers](@docroot@/glossary.md#gloss-reference) to another path that doesn’t exist in the target Nix store, the import fails. + +> **Note** +> +> For efficient transfer of closures to remote machines over SSH, use [`nix-copy-closure`](@docroot@/command-ref/nix-copy-closure.md). {{#include ./opt-common.md}} {{#include ../opt-common.md}} {{#include ../env-common.md}} + +# Examples + +> **Example** +> +> Given a closure of GNU Hello as a file: +> +> ```shell-session +> $ storePath="$(nix-build '' -I nixpkgs=channel:nixpkgs-unstable -A hello --no-out-link)" +> $ nix-store --export $(nix-store --query --requisites $storePath) > hello.closure +> ``` +> +> Import the closure into a [remote SSH store](@docroot@/store/types/ssh-store.md) using the [`--store`](@docroot@/command-ref/conf-file.md#conf-store) option: +> +> ```console +> $ nix-store --import --store ssh://alice@itchy.example.org < hello.closure +> ``` + diff --git a/doc/manual/src/command-ref/nix-store/optimise.md b/doc/manual/src/command-ref/nix-store/optimise.md index dc392aeb8..b257466b2 100644 --- a/doc/manual/src/command-ref/nix-store/optimise.md +++ b/doc/manual/src/command-ref/nix-store/optimise.md @@ -12,7 +12,7 @@ The operation `--optimise` reduces Nix store disk space usage by finding identical files in the store and hard-linking them to each other. It typically reduces the size of the store by something like 25-35%. Only regular files and symlinks are hard-linked in this manner. Files are -considered identical when they have the same NAR archive serialisation: +considered identical when they have the same [Nix Archive (NAR)][Nix Archive] serialisation: that is, regular files must have the same contents and permission (executable or non-executable), and symlinks must have the same contents. @@ -38,3 +38,4 @@ hashing files in `/nix/store/qhqx7l2f1kmwihc9bnxs7rc159hsxnf3-gcc-4.1.1' there are 114486 files with equal contents out of 215894 files in total ``` +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive diff --git a/doc/manual/src/command-ref/nix-store/query.md b/doc/manual/src/command-ref/nix-store/query.md index a158c76aa..b4efa734e 100644 --- a/doc/manual/src/command-ref/nix-store/query.md +++ b/doc/manual/src/command-ref/nix-store/query.md @@ -24,122 +24,138 @@ symlink. # Common query options - - `--use-output`; `-u`\ - For each argument to the query that is a [store derivation], apply the - query to the output path of the derivation instead. +- `--use-output` / `-u` - - `--force-realise`; `-f`\ - Realise each argument to the query first (see [`nix-store --realise`](./realise.md)). + For each argument to the query that is a [store derivation], apply the + query to the output path of the derivation instead. + +- `--force-realise` / `-f` + + Realise each argument to the query first (see [`nix-store --realise`](./realise.md)). [store derivation]: @docroot@/glossary.md#gloss-store-derivation # Queries - - `--outputs`\ - Prints out the [output paths] of the store - derivations *paths*. These are the paths that will be produced when - the derivation is built. +- `--outputs` - [output paths]: ../../glossary.md#gloss-output-path + Prints out the [output paths] of the store + derivations *paths*. These are the paths that will be produced when + the derivation is built. - - `--requisites`; `-R`\ - Prints out the [closure] of the store path *paths*. + [output paths]: @docroot@/glossary.md#gloss-output-path - [closure]: ../../glossary.md#gloss-closure +- `--requisites` / `-R` - This query has one option: + Prints out the [closure] of the store path *paths*. - - `--include-outputs` - Also include the existing output paths of [store derivation]s, - and their closures. + [closure]: @docroot@/glossary.md#gloss-closure - This query can be used to implement various kinds of deployment. A - *source deployment* is obtained by distributing the closure of a - store derivation. A *binary deployment* is obtained by distributing - the closure of an output path. A *cache deployment* (combined - source/binary deployment, including binaries of build-time-only - dependencies) is obtained by distributing the closure of a store - derivation and specifying the option `--include-outputs`. + This query has one option: - - `--references`\ - Prints the set of [references] of the store paths - *paths*, that is, their immediate dependencies. (For *all* - dependencies, use `--requisites`.) + - `--include-outputs` + Also include the existing output paths of [store derivation]s, + and their closures. - [references]: ../../glossary.md#gloss-reference + This query can be used to implement various kinds of deployment. A + *source deployment* is obtained by distributing the closure of a + store derivation. A *binary deployment* is obtained by distributing + the closure of an output path. A *cache deployment* (combined + source/binary deployment, including binaries of build-time-only + dependencies) is obtained by distributing the closure of a store + derivation and specifying the option `--include-outputs`. - - `--referrers`\ - Prints the set of *referrers* of the store paths *paths*, that is, - the store paths currently existing in the Nix store that refer to - one of *paths*. Note that contrary to the references, the set of - referrers is not constant; it can change as store paths are added or - removed. +- `--references` - - `--referrers-closure`\ - Prints the closure of the set of store paths *paths* under the - referrers relation; that is, all store paths that directly or - indirectly refer to one of *paths*. These are all the path currently - in the Nix store that are dependent on *paths*. + Prints the set of [references] of the store paths + *paths*, that is, their immediate dependencies. (For *all* + dependencies, use `--requisites`.) - - `--deriver`; `-d`\ - Prints the [deriver] that was used to build the store paths *paths*. If - the path has no deriver (e.g., if it is a source file), or if the - deriver is not known (e.g., in the case of a binary-only - deployment), the string `unknown-deriver` is printed. - The returned deriver is not guaranteed to exist in the local store, for - example when *paths* were substituted from a binary cache. - Use `--valid-derivers` instead to obtain valid paths only. + [references]: @docroot@/glossary.md#gloss-reference - [deriver]: ../../glossary.md#gloss-deriver +- `--referrers` - - `--valid-derivers`\ - Prints a set of derivation files (`.drv`) which are supposed produce - said paths when realized. Might print nothing, for example for source paths - or paths subsituted from a binary cache. + Prints the set of *referrers* of the store paths *paths*, that is, + the store paths currently existing in the Nix store that refer to + one of *paths*. Note that contrary to the references, the set of + referrers is not constant; it can change as store paths are added or + removed. - - `--graph`\ - Prints the references graph of the store paths *paths* in the format - of the `dot` tool of AT\&T's [Graphviz - package](http://www.graphviz.org/). This can be used to visualise - dependency graphs. To obtain a build-time dependency graph, apply - this to a store derivation. To obtain a runtime dependency graph, - apply it to an output path. +- `--referrers-closure` - - `--tree`\ - Prints the references graph of the store paths *paths* as a nested - ASCII tree. References are ordered by descending closure size; this - tends to flatten the tree, making it more readable. The query only - recurses into a store path when it is first encountered; this - prevents a blowup of the tree representation of the graph. + Prints the closure of the set of store paths *paths* under the + referrers relation; that is, all store paths that directly or + indirectly refer to one of *paths*. These are all the path currently + in the Nix store that are dependent on *paths*. - - `--graphml`\ - Prints the references graph of the store paths *paths* in the - [GraphML](http://graphml.graphdrawing.org/) file format. This can be - used to visualise dependency graphs. To obtain a build-time - dependency graph, apply this to a [store derivation]. To obtain a - runtime dependency graph, apply it to an output path. +- `--deriver` / `-d` - - `--binding` *name*; `-b` *name*\ - Prints the value of the attribute *name* (i.e., environment - variable) of the [store derivation]s *paths*. It is an error for a - derivation to not have the specified attribute. + Prints the [deriver] that was used to build the store paths *paths*. If + the path has no deriver (e.g., if it is a source file), or if the + deriver is not known (e.g., in the case of a binary-only + deployment), the string `unknown-deriver` is printed. + The returned deriver is not guaranteed to exist in the local store, for + example when *paths* were substituted from a binary cache. + Use `--valid-derivers` instead to obtain valid paths only. - - `--hash`\ - Prints the SHA-256 hash of the contents of the store paths *paths* - (that is, the hash of the output of `nix-store --dump` on the given - paths). Since the hash is stored in the Nix database, this is a fast - operation. + [deriver]: @docroot@/glossary.md#gloss-deriver - - `--size`\ - Prints the size in bytes of the contents of the store paths *paths* - — to be precise, the size of the output of `nix-store --dump` on - the given paths. Note that the actual disk space required by the - store paths may be higher, especially on filesystems with large - cluster sizes. +- `--valid-derivers` - - `--roots`\ - Prints the garbage collector roots that point, directly or - indirectly, at the store paths *paths*. + Prints a set of derivation files (`.drv`) which are supposed produce + said paths when realized. Might print nothing, for example for source paths + or paths subsituted from a binary cache. + +- `--graph` + + Prints the references graph of the store paths *paths* in the format + of the `dot` tool of AT\&T's [Graphviz + package](http://www.graphviz.org/). This can be used to visualise + dependency graphs. To obtain a build-time dependency graph, apply + this to a store derivation. To obtain a runtime dependency graph, + apply it to an output path. + +- `--tree` + + Prints the references graph of the store paths *paths* as a nested + ASCII tree. References are ordered by descending closure size; this + tends to flatten the tree, making it more readable. The query only + recurses into a store path when it is first encountered; this + prevents a blowup of the tree representation of the graph. + +- `--graphml` + + Prints the references graph of the store paths *paths* in the + [GraphML](http://graphml.graphdrawing.org/) file format. This can be + used to visualise dependency graphs. To obtain a build-time + dependency graph, apply this to a [store derivation]. To obtain a + runtime dependency graph, apply it to an output path. + +- `--binding` *name* / `-b` *name* + + Prints the value of the attribute *name* (i.e., environment + variable) of the [store derivation]s *paths*. It is an error for a + derivation to not have the specified attribute. + +- `--hash` + + Prints the SHA-256 hash of the contents of the store paths *paths* + (that is, the hash of the output of `nix-store --dump` on the given + paths). Since the hash is stored in the Nix database, this is a fast + operation. + +- `--size` + + Prints the size in bytes of the contents of the store paths *paths* + — to be precise, the size of the output of `nix-store --dump` on + the given paths. Note that the actual disk space required by the + store paths may be higher, especially on filesystems with large + cluster sizes. + +- `--roots` + + Prints the garbage collector roots that point, directly or + indirectly, at the store paths *paths*. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index 5428d57fa..a899758df 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -25,14 +25,14 @@ Each of *paths* is processed as follows: If no substitutes are available and no store derivation is given, realisation fails. -[store paths]: @docroot@/glossary.md#gloss-store-path +[store paths]: @docroot@/store/store-path.md [valid]: @docroot@/glossary.md#gloss-validity [store derivation]: @docroot@/glossary.md#gloss-store-derivation [output paths]: @docroot@/glossary.md#gloss-output-path -[store objects]: @docroot@/glossary.md#gloss-store-object +[store objects]: @docroot@/store/store-object.md [closure]: @docroot@/glossary.md#gloss-closure [substituters]: @docroot@/command-ref/conf-file.md#conf-substituters -[content-addressed derivations]: @docroot@/contributing/experimental-features.md#xp-feature-ca-derivations +[content-addressed derivations]: @docroot@/development/experimental-features.md#xp-feature-ca-derivations [Nix database]: @docroot@/glossary.md#gloss-nix-database The resulting paths are printed on standard output. @@ -42,23 +42,26 @@ For non-derivation arguments, the argument itself is printed. # Options - - `--dry-run`\ - Print on standard error a description of what packages would be - built or downloaded, without actually performing the operation. +- `--dry-run` - - `--ignore-unknown`\ - If a non-derivation path does not have a substitute, then silently - ignore it. + Print on standard error a description of what packages would be + built or downloaded, without actually performing the operation. - - `--check`\ - This option allows you to check whether a derivation is - deterministic. It rebuilds the specified derivation and checks - whether the result is bitwise-identical with the existing outputs, - printing an error if that’s not the case. The outputs of the - specified derivation must already exist. When used with `-K`, if an - output path is not identical to the corresponding output from the - previous build, the new output path is left in - `/nix/store/name.check.` +- `--ignore-unknown` + + If a non-derivation path does not have a substitute, then silently + ignore it. + +- `--check` + + This option allows you to check whether a derivation is + deterministic. It rebuilds the specified derivation and checks + whether the result is bitwise-identical with the existing outputs, + printing an error if that’s not the case. The outputs of the + specified derivation must already exist. When used with `-K`, if an + output path is not identical to the corresponding output from the + previous build, the new output path is left in + `/nix/store/name.check.` {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/restore.md b/doc/manual/src/command-ref/nix-store/restore.md index fcba43df4..2d0aa3127 100644 --- a/doc/manual/src/command-ref/nix-store/restore.md +++ b/doc/manual/src/command-ref/nix-store/restore.md @@ -8,9 +8,11 @@ ## Description -The operation `--restore` unpacks a NAR archive to *path*, which must +The operation `--restore` unpacks a [Nix Archive (NAR)][Nix Archive] to *path*, which must not already exist. The archive is read from standard input. +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive + {{#include ./opt-common.md}} {{#include ../opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/serve.md b/doc/manual/src/command-ref/nix-store/serve.md index 0f90f65ae..9a4cf5216 100644 --- a/doc/manual/src/command-ref/nix-store/serve.md +++ b/doc/manual/src/command-ref/nix-store/serve.md @@ -14,10 +14,11 @@ access to a restricted ssh user. The following flags are available: - - `--write`\ - Allow the connected client to request the realization of - derivations. In effect, this can be used to make the host act as a - remote builder. +- `--write` + + Allow the connected client to request the realization of + derivations. In effect, this can be used to make the host act as a + remote builder. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/verify.md b/doc/manual/src/command-ref/nix-store/verify.md index 2695b3361..40c9180db 100644 --- a/doc/manual/src/command-ref/nix-store/verify.md +++ b/doc/manual/src/command-ref/nix-store/verify.md @@ -16,18 +16,20 @@ being modified by non-Nix tools, or of bugs in Nix itself. This operation has the following options: - - `--check-contents`\ - Checks that the contents of every valid store path has not been - altered by computing a SHA-256 hash of the contents and comparing it - with the hash stored in the Nix database at build time. Paths that - have been modified are printed out. For large stores, - `--check-contents` is obviously quite slow. +- `--check-contents` - - `--repair`\ - If any valid path is missing from the store, or (if - `--check-contents` is given) the contents of a valid path has been - modified, then try to repair the path by redownloading it. See - `nix-store --repair-path` for details. + Checks that the contents of every valid store path has not been + altered by computing a SHA-256 hash of the contents and comparing it + with the hash stored in the Nix database at build time. Paths that + have been modified are printed out. For large stores, + `--check-contents` is obviously quite slow. + +- `--repair` + + If any valid path is missing from the store, or (if + `--check-contents` is given) the contents of a valid path has been + modified, then try to repair the path by redownloading it. See + `nix-store --repair-path` for details. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index 114b292f9..69a700207 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -37,7 +37,7 @@ Most Nix commands accept the following command-line options: Print even more informational messages. - `4` “Debug†- + Print debug information. - `5` “Vomit†@@ -143,7 +143,7 @@ Most Nix commands accept the following command-line options: This option is accepted by `nix-env`, `nix-instantiate`, `nix-shell` and `nix-build`. When evaluating Nix expressions, the expression evaluator will automatically try to call functions that it encounters. - It can automatically call functions for which every argument has a [default value](@docroot@/language/constructs.md#functions) (e.g., `{ argName ? defaultValue }: ...`). + It can automatically call functions for which every argument has a [default value](@docroot@/language/syntax.md#functions) (e.g., `{ argName ? defaultValue }: ...`). With `--arg`, you can also call functions that have arguments without a default value (or override a default value). That is, if the evaluator encounters a function with an argument named *name*, it will call it with value *value*. @@ -187,11 +187,12 @@ Most Nix commands accept the following command-line options: For `nix-shell`, this option is commonly used to give you a shell in which you can build the packages returned by the expression. If you want to get a shell which contain the *built* packages ready for use, give your expression to the `nix-shell --packages ` convenience flag instead. -- [`-I`](#opt-I) *path* +- [`-I` / `--include`](#opt-I) *path* - Add an entry to the [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path). + Add an entry to the list of search paths used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). This option may be given multiple times. - Paths added through `-I` take precedence over [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH). + + Paths added through `-I` take precedence over the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path) and the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH). - [`--option`](#opt-option) *name* *value* diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/development/building.md similarity index 67% rename from doc/manual/src/contributing/hacking.md rename to doc/manual/src/development/building.md index 916ec3077..5a5fb3368 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/development/building.md @@ -1,24 +1,67 @@ -# Hacking +# Building Nix -This section provides some notes on how to hack on Nix. To get the -latest version of Nix from GitHub: +This section provides some notes on how to start hacking on Nix. +To get the latest version of Nix from GitHub: ```console $ git clone https://github.com/NixOS/nix.git $ cd nix ``` -The following instructions assume you already have some version of Nix installed locally, so that you can use it to set up the development environment. If you don't have it installed, follow the [installation instructions]. +> **Note** +> +> The following instructions assume you already have some version of Nix installed locally, so that you can use it to set up the development environment. +> If you don't have it installed, follow the [installation instructions](../installation/index.md). -[installation instructions]: ../installation/index.md + +To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: + +```console +$ nix-shell +``` + +To get a shell with one of the other [supported compilation environments](#compilation-environments): + +```console +$ nix-shell --attr devShells.x86_64-linux.native-clangStdenvPackages +``` + +> **Note** +> +> You can use `native-ccacheStdenvPackages` to drastically improve rebuild time. +> By default, [ccache](https://ccache.dev) keeps artifacts in `~/.cache/ccache/`. + +To build Nix itself in this shell: + +```console +[nix-shell]$ autoreconfPhase +[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out +[nix-shell]$ make -j $NIX_BUILD_CORES +``` + +To install it in `$(pwd)/outputs` and test it: + +```console +[nix-shell]$ make install +[nix-shell]$ make installcheck -j $NIX_BUILD_CORES +[nix-shell]$ ./outputs/out/bin/nix --version +nix (Nix) 2.12 +``` + +To build a release version of Nix for the current operating system and CPU architecture: + +```console +$ nix-build +``` + +You can also build Nix for one of the [supported platforms](#platforms). ## Building Nix with flakes This section assumes you are using Nix with the [`flakes`] and [`nix-command`] experimental features enabled. -See the [Building Nix](#building-nix) section for equivalent instructions using stable Nix interfaces. -[`flakes`]: @docroot@/contributing/experimental-features.md#xp-feature-flakes -[`nix-command`]: @docroot@/contributing/experimental-features.md#xp-nix-command +[`flakes`]: @docroot@/development/experimental-features.md#xp-feature-flakes +[`nix-command`]: @docroot@/development/experimental-features.md#xp-nix-command To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: @@ -67,50 +110,6 @@ $ nix build You can also build Nix for one of the [supported platforms](#platforms). -## Building Nix - -To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: - -```console -$ nix-shell -``` - -To get a shell with one of the other [supported compilation environments](#compilation-environments): - -```console -$ nix-shell --attr devShells.x86_64-linux.native-clangStdenvPackages -``` - -> **Note** -> -> You can use `native-ccacheStdenvPackages` to drastically improve rebuild time. -> By default, [ccache](https://ccache.dev) keeps artifacts in `~/.cache/ccache/`. - -To build Nix itself in this shell: - -```console -[nix-shell]$ autoreconfPhase -[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out -[nix-shell]$ make -j $NIX_BUILD_CORES -``` - -To install it in `$(pwd)/outputs` and test it: - -```console -[nix-shell]$ make install -[nix-shell]$ make installcheck -j $NIX_BUILD_CORES -[nix-shell]$ ./outputs/out/bin/nix --version -nix (Nix) 2.12 -``` - -To build a release version of Nix for the current operating system and CPU architecture: - -```console -$ nix-build -``` - -You can also build Nix for one of the [supported platforms](#platforms). - ## Makefile variables You may need `profiledir=$out/etc/profile.d` and `sysconfdir=$out/etc` to run `make install`. @@ -122,7 +121,6 @@ Run `make` with [`-e` / `--environment-overrides`](https://www.gnu.org/software/ The docs can take a while to build, so you may want to disable this for local development. - `ENABLE_FUNCTIONAL_TESTS=yes` to enable building the functional tests. -- `ENABLE_UNIT_TESTS=yes` to enable building the unit tests. - `OPTIMIZE=1` to enable optimizations. - `libraries=libutil programs=` to only build a specific library. @@ -144,6 +142,7 @@ Nix can be built for various platforms, as specified in [`flake.nix`]: - `aarch64-darwin` - `armv6l-linux` - `armv7l-linux` +- `riscv64-linux` In order to build Nix for a different platform than the one you're currently on, you need a way for your current Nix installation to build code for that @@ -166,7 +165,10 @@ or for Nix with the [`flakes`] and [`nix-command`] experimental features enabled $ nix build .#packages.aarch64-linux.default ``` -Cross-compiled builds are available for ARMv6 (`armv6l-linux`) and ARMv7 (`armv7l-linux`). +Cross-compiled builds are available for: +- `armv6l-linux` +- `armv7l-linux` +- `riscv64-linux` Add more [system types](#system-type) to `crossSystems` in `flake.nix` to bootstrap Nix on unsupported platforms. ### Building for multiple platforms at once @@ -196,7 +198,7 @@ In order to facilitate this, Nix has some support for being built out of tree ## System type -Nix uses a string with he following format to identify the *system type* or *platform* it runs on: +Nix uses a string with the following format to identify the *system type* or *platform* it runs on: ``` -[-] @@ -258,10 +260,10 @@ See [supported compilation environments](#compilation-environments) and instruct To use the LSP with your editor, you first need to [set up `clangd`](https://clangd.llvm.org/installation#project-setup) by running: ```console -make clean && bear -- make -j$NIX_BUILD_CORES default check install +make compile_commands.json ``` -Configure your editor to use the `clangd` from the shell, either by running it inside the development shell, or by using [nix-direnv](https://github.com/nix-community/nix-direnv) and [the appropriate editor plugin](https://github.com/direnv/direnv/wiki#editor-integration). +Configure your editor to use the `clangd` from the `.#native-clangStdenvPackages` shell. You can do that either by running it inside the development shell, or by using [nix-direnv](https://github.com/nix-community/nix-direnv) and [the appropriate editor plugin](https://github.com/direnv/direnv/wiki#editor-integration). > **Note** > @@ -269,80 +271,25 @@ Configure your editor to use the `clangd` from the shell, either by running it i > Some other editors (e.g. Emacs, Vim) need a plugin to support LSP servers in general (e.g. [lsp-mode](https://github.com/emacs-lsp/lsp-mode) for Emacs and [vim-lsp](https://github.com/prabirshrestha/vim-lsp) for vim). > Editor-specific setup is typically opinionated, so we will not cover it here in more detail. -## Add a release note +## Formatting and pre-commit hooks -`doc/manual/rl-next` contains release notes entries for all unreleased changes. +You may run the formatters as a one-off using: -User-visible changes should come with a release note. - -### Add an entry - -Here's what a complete entry looks like. The file name is not incorporated in the document. - -``` ---- -synopsis: Basically a title -issues: 1234 -prs: 1238 ---- - -Here's one or more paragraphs that describe the change. - -- It's markdown -- Add references to the manual using @docroot@ +```console +make format ``` -Significant changes should add the following header, which moves them to the top. +If you'd like to run the formatters before every commit, install the hooks: ``` -significance: significant +pre-commit-hooks-install ``` - -See also the [format documentation](https://github.com/haskell/cabal/blob/master/CONTRIBUTING.md#changelog). +This installs [pre-commit](https://pre-commit.com) using [cachix/git-hooks.nix](https://github.com/cachix/git-hooks.nix). -### Build process +When making a commit, pay attention to the console output. +If it fails, run `git add --patch` to approve the suggestions _and commit again_. -Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`. - -## Branches - -- [`master`](https://github.com/NixOS/nix/commits/master) - - The main development branch. All changes are approved and merged here. - When developing a change, create a branch based on the latest `master`. - - Maintainers try to [keep it in a release-worthy state](#reverting). - -- [`maintenance-*.*`](https://github.com/NixOS/nix/branches/all?query=maintenance) - - These branches are the subject of backports only, and are - also [kept](#reverting) in a release-worthy state. - - See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) - -- [`latest-release`](https://github.com/NixOS/nix/tree/latest-release) - - The latest patch release of the latest minor version. - - See [`maintainers/release-process.md`](https://github.com/NixOS/nix/blob/master/maintainers/release-process.md) - -- [`backport-*-to-*`](https://github.com/NixOS/nix/branches/all?query=backport) - - Generally branches created by the backport action. - - See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) - -- [_other_](https://github.com/NixOS/nix/branches/all) - - Branches that do not conform to the above patterns should be feature branches. - -## Reverting - -If a change turns out to be merged by mistake, or contain a regression, it may be reverted. -A revert is not a rejection of the contribution, but merely part of an effective development process. -It makes sure that development keeps running smoothly, with minimal uncertainty, and less overhead. -If maintainers have to worry too much about avoiding reverts, they would not be able to merge as much. -By embracing reverts as a good part of the development process, everyone wins. - -However, taking a step back may be frustrating, so maintainers will be extra supportive on the next try. +To refresh pre-commit hook's config file, do the following: +1. Exit the development shell and start it again by running `nix develop`. +2. If you also use the pre-commit hook, also run `pre-commit-hooks-install` again. diff --git a/doc/manual/src/contributing/cli-guideline.md b/doc/manual/src/development/cli-guideline.md similarity index 88% rename from doc/manual/src/contributing/cli-guideline.md rename to doc/manual/src/development/cli-guideline.md index e90d6de8d..23df844ec 100644 --- a/doc/manual/src/contributing/cli-guideline.md +++ b/doc/manual/src/development/cli-guideline.md @@ -389,88 +389,6 @@ colors, no emojis and using ASCII instead of Unicode symbols). The same should happen when TTY is not detected on STDERR. We should not display progress / status section, but only print warnings and errors. -## Returning future proof JSON - -The schema of JSON output should allow for backwards compatible extension. This section explains how to achieve this. - -Two definitions are helpful here, because while JSON only defines one "key-value" -object type, we use it to cover two use cases: - - - **dictionary**: a map from names to value that all have the same type. In - C++ this would be a `std::map` with string keys. - - **record**: a fixed set of attributes each with their own type. In C++, this - would be represented by a `struct`. - -It is best not to mix these use cases, as that may lead to incompatibilities when the schema changes. For example, adding a record field to a dictionary breaks consumers that assume all JSON object fields to have the same meaning and type. - -This leads to the following guidelines: - - - The top-level (root) value must be a record. - - Otherwise, one can not change the structure of a command's output. - - - The value of a dictionary item must be a record. - - Otherwise, the item type can not be extended. - - - List items should be records. - - Otherwise, one can not change the structure of the list items. - - If the order of the items does not matter, and each item has a unique key that is a string, consider representing the list as a dictionary instead. If the order of the items needs to be preserved, return a list of records. - - - Streaming JSON should return records. - - An example of a streaming JSON format is [JSON lines](https://jsonlines.org/), where each line represents a JSON value. These JSON values can be considered top-level values or list items, and they must be records. - -### Examples - - -This is bad, because all keys must be assumed to be store types: - -```json -{ - "local": { ... }, - "remote": { ... }, - "http": { ... } -} -``` - -This is good, because the it is extensible at the root, and is somewhat self-documenting: - -```json -{ - "storeTypes": { "local": { ... }, ... }, - "pluginSupport": true -} -``` - -While the dictionary of store types seems like a very complete response at first, a use case may arise that warrants returning additional information. -For example, the presence of plugin support may be crucial information for a client to proceed when their desired store type is missing. - - - -The following representation is bad because it is not extensible: - -```json -{ "outputs": [ "out" "bin" ] } -``` - -However, simply converting everything to records is not enough, because the order of outputs must be preserved: - -```json -{ "outputs": { "bin": {}, "out": {} } } -``` - -The first item is the default output. Deriving this information from the outputs ordering is not great, but this is how Nix currently happens to work. -While it is possible for a JSON parser to preserve the order of fields, we can not rely on this capability to be present in all JSON libraries. - -This representation is extensible and preserves the ordering: - -```json -{ "outputs": [ { "outputName": "out" }, { "outputName": "bin" } ] } -``` - ## Dialog with the user CLIs don't always make it clear when an action has taken place. For every diff --git a/doc/manual/src/development/contributing.md b/doc/manual/src/development/contributing.md new file mode 100644 index 000000000..7de7489dc --- /dev/null +++ b/doc/manual/src/development/contributing.md @@ -0,0 +1,79 @@ +# Contributing + +## Add a release note + +`doc/manual/rl-next` contains release notes entries for all unreleased changes. + +User-visible changes should come with a release note. + +### Add an entry + +Here's what a complete entry looks like. The file name is not incorporated in the document. + +``` +--- +synopsis: Basically a title +issues: 1234 +prs: 1238 +--- + +Here's one or more paragraphs that describe the change. + +- It's markdown +- Add references to the manual using @docroot@ +``` + +Significant changes should add the following header, which moves them to the top. + +``` +significance: significant +``` + + +See also the [format documentation](https://github.com/haskell/cabal/blob/master/CONTRIBUTING.md#changelog). + +### Build process + +Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`. + +## Branches + +- [`master`](https://github.com/NixOS/nix/commits/master) + + The main development branch. All changes are approved and merged here. + When developing a change, create a branch based on the latest `master`. + + Maintainers try to [keep it in a release-worthy state](#reverting). + +- [`maintenance-*.*`](https://github.com/NixOS/nix/branches/all?query=maintenance) + + These branches are the subject of backports only, and are + also [kept](#reverting) in a release-worthy state. + + See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) + +- [`latest-release`](https://github.com/NixOS/nix/tree/latest-release) + + The latest patch release of the latest minor version. + + See [`maintainers/release-process.md`](https://github.com/NixOS/nix/blob/master/maintainers/release-process.md) + +- [`backport-*-to-*`](https://github.com/NixOS/nix/branches/all?query=backport) + + Generally branches created by the backport action. + + See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) + +- [_other_](https://github.com/NixOS/nix/branches/all) + + Branches that do not conform to the above patterns should be feature branches. + +## Reverting + +If a change turns out to be merged by mistake, or contain a regression, it may be reverted. +A revert is not a rejection of the contribution, but merely part of an effective development process. +It makes sure that development keeps running smoothly, with minimal uncertainty, and less overhead. +If maintainers have to worry too much about avoiding reverts, they would not be able to merge as much. +By embracing reverts as a good part of the development process, everyone wins. + +However, taking a step back may be frustrating, so maintainers will be extra supportive on the next try. diff --git a/doc/manual/src/contributing/cxx.md b/doc/manual/src/development/cxx.md similarity index 100% rename from doc/manual/src/contributing/cxx.md rename to doc/manual/src/development/cxx.md diff --git a/doc/manual/src/contributing/documentation.md b/doc/manual/src/development/documentation.md similarity index 89% rename from doc/manual/src/contributing/documentation.md rename to doc/manual/src/development/documentation.md index 1dddb207c..63f574ab7 100644 --- a/doc/manual/src/contributing/documentation.md +++ b/doc/manual/src/development/documentation.md @@ -24,14 +24,12 @@ nix build .#^doc and open `./result-doc/share/doc/nix/manual/index.html`. -To build the manual incrementally, [enter the development shell](./hacking.md) and run: +To build the manual incrementally, [enter the development shell](./building.md) and run: ```console -make manual-html -j $NIX_BUILD_CORES +make manual-html-open -j $NIX_BUILD_CORES ``` -and open `./outputs/out/share/doc/nix/manual/language/index.html`. - In order to reflect changes to the [Makefile for the manual], clear all generated files before re-building: [Makefile for the manual]: https://github.com/NixOS/nix/blob/master/doc/manual/local.mk @@ -149,7 +147,7 @@ Please observe these guidelines to ease reviews: ``` A [store object] contains a [file system object] and [references] to other store objects. - [store object]: @docroot@/glossary.md#gloss-store-object + [store object]: @docroot@/store/store-object.md [file system object]: @docroot@/architecture/file-system-object.md [references]: @docroot@/glossary.md#gloss-reference ``` @@ -198,13 +196,35 @@ You can also build and view it yourself: [Doxygen API documentation]: https://hydra.nixos.org/job/nix/master/internal-api-docs/latest/download-by-type/doc/internal-api-docs ```console -# nix build .#hydraJobs.internal-api-docs -# xdg-open ./result/share/doc/nix/internal-api/html/index.html +$ nix build .#hydraJobs.internal-api-docs +$ xdg-open ./result/share/doc/nix/internal-api/html/index.html +``` + +or inside `nix-shell` or `nix develop`: + +```console +$ mesonConfigurePhase +$ ninja src/internal-api-docs/html +$ xdg-open src/internal-api-docs/html/index.html +``` + +## C API documentation + +Note that the C API is not yet stable. +[C API documentation] is available online. +You can also build and view it yourself: + +[C API documentation]: https://hydra.nixos.org/job/nix/master/external-api-docs/latest/download-by-type/doc/external-api-docs + +```console +$ nix build .#hydraJobs.external-api-docs +$ xdg-open ./result/share/doc/nix/external-api/html/index.html ``` or inside `nix-shell` or `nix develop`: ``` -# make internal-api-html -# xdg-open ./outputs/doc/share/doc/nix/internal-api/html/index.html +$ mesonConfigurePhase +$ ninja src/external-api-docs/html +$ xdg-open src/external-api-docs/html/index.html ``` diff --git a/doc/manual/src/contributing/experimental-features.md b/doc/manual/src/development/experimental-features.md similarity index 100% rename from doc/manual/src/contributing/experimental-features.md rename to doc/manual/src/development/experimental-features.md diff --git a/doc/manual/src/contributing/index.md b/doc/manual/src/development/index.md similarity index 77% rename from doc/manual/src/contributing/index.md rename to doc/manual/src/development/index.md index 4d55c17a4..6403c3e66 100644 --- a/doc/manual/src/contributing/index.md +++ b/doc/manual/src/development/index.md @@ -5,4 +5,4 @@ Check the [contributing guide](https://github.com/NixOS/nix/blob/master/CONTRIBU This chapter is a collection of guides for making changes to the code and documentation. -If you're not sure where to start, try to [compile Nix from source](./hacking.md) and consider [making improvements to documentation](./documentation.md). +If you're not sure where to start, try to [compile Nix from source](./building.md) and consider [making improvements to documentation](./documentation.md). diff --git a/doc/manual/src/development/json-guideline.md b/doc/manual/src/development/json-guideline.md new file mode 100644 index 000000000..b4bc92af9 --- /dev/null +++ b/doc/manual/src/development/json-guideline.md @@ -0,0 +1,128 @@ +# JSON guideline + +Nix consumes and produces JSON in a variety of contexts. +These guidelines ensure consistent practices for all our JSON interfaces, for ease of use, and so that experience in one part carries over to another. + +## Extensibility + +The schema of JSON input and output should allow for backwards compatible extension. +This section explains how to achieve this. + +Two definitions are helpful here, because while JSON only defines one "key-value" object type, we use it to cover two use cases: + + - **dictionary**: a map from names to value that all have the same type. + In C++ this would be a `std::map` with string keys. + + - **record**: a fixed set of attributes each with their own type. + In C++, this would be represented by a `struct`. + +It is best not to mix these use cases, as that may lead to incompatibilities when the schema changes. +For example, adding a record field to a dictionary breaks consumers that assume all JSON object fields to have the same meaning and type, and dictionary items with a colliding name can not be represented anymore. + +This leads to the following guidelines: + + - The top-level (root) value must be a record. + + Otherwise, one can not change the structure of a command's output. + + - The value of a dictionary item must be a record. + + Otherwise, the item type can not be extended. + + - List items should be records. + + Otherwise, one can not change the structure of the list items. + + If the order of the items does not matter, and each item has a unique key that is a string, consider representing the list as a dictionary instead. + If the order of the items needs to be preserved, return a list of records. + + - Streaming JSON should return records. + + An example of a streaming JSON format is [JSON lines](https://jsonlines.org/), where each line represents a JSON value. + These JSON values can be considered top-level values or list items, and they must be records. + +### Examples + +This is bad, because all keys must be assumed to be store types: + +```json +{ + "local": { ... }, + "remote": { ... }, + "http": { ... } +} +``` + +This is good, because the it is extensible at the root, and is somewhat self-documenting: + +```json +{ + "storeTypes": { "local": { ... }, ... }, + "pluginSupport": true +} +``` + +While the dictionary of store types seems like a very complete response at first, a use case may arise that warrants returning additional information. +For example, the presence of plugin support may be crucial information for a client to proceed when their desired store type is missing. + + + +The following representation is bad because it is not extensible: + +```json +{ "outputs": [ "out" "bin" ] } +``` + +However, simply converting everything to records is not enough, because the order of outputs must be preserved: + +```json +{ "outputs": { "bin": {}, "out": {} } } +``` + +The first item is the default output. Deriving this information from the outputs ordering is not great, but this is how Nix currently happens to work. +While it is possible for a JSON parser to preserve the order of fields, we can not rely on this capability to be present in all JSON libraries. + +This representation is extensible and preserves the ordering: + +```json +{ "outputs": [ { "outputName": "out" }, { "outputName": "bin" } ] } +``` + +## Self-describing values + +As described in the previous section, it's crucial that schemas can be extended with with new fields without breaking compatibility. +However, that should *not* mean we use the presence/absence of fields to indicate optional information *within* a version of the schema. +Instead, always include the field, and use `null` to indicate the "nothing" case. + +### Examples + +Here are two JSON objects: + +```json +{ + "foo": {} +} +``` +```json +{ + "foo": {}, + "bar": {} +} +``` + +Since they differ in which fields they contain, they should *not* both be valid values of the same schema. +At most, they can match two different schemas where the second (with `foo` and `bar`) is considered a newer version of the first (with just `foo`). +Within each version, all fields are mandatory (always `foo`, and always `foo` and `bar`). +Only *between* each version, `bar` gets added as a new mandatory field. + +Here are another two JSON objects: + +```json +{ "foo": null } +``` +```json +{ "foo": { "bar": 1 } } +``` + +Since they both contain a `foo` field, they could be valid values of the same schema. +The schema would have `foo` has an optional field, which is either `null` or an object where `bar` is an integer. diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/development/testing.md similarity index 81% rename from doc/manual/src/contributing/testing.md rename to doc/manual/src/development/testing.md index 31c39c16c..3949164d5 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/development/testing.md @@ -59,15 +59,15 @@ The unit tests are defined using the [googletest] and [rapidcheck] frameworks. > … > ``` -The tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `tests/unit/${library_name_without-nix}`. -Given a interface (header) and implementation pair in the original library, say, `src/libexpr/value/context.{hh,cc}`, we write tests for it in `tests/unit/libexpr/tests/value/context.cc`, and (possibly) declare/define additional interfaces for testing purposes in `tests/unit/libexpr-support/tests/value/context.{hh,cc}`. +The tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `src/${library_name_without-nix}-test`. +Given an interface (header) and implementation pair in the original library, say, `src/libexpr/value/context.{hh,cc}`, we write tests for it in `src/nix-expr-tests/value/context.cc`, and (possibly) declare/define additional interfaces for testing purposes in `src/nix-expr-test-support/tests/value/context.{hh,cc}`. Data for unit tests is stored in a `data` subdir of the directory for each unit test executable. -For example, `libnixstore` code is in `src/libstore`, and its test data is in `tests/unit/libstore/data`. -The path to the `tests/unit/data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`. +For example, `libnixstore` code is in `src/libstore`, and its test data is in `src/nix-store-tests/data`. +The path to the `src/${library_name_without-nix}-test/data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`. Note that each executable only gets the data for its tests. -The unit test libraries are in `tests/unit/${library_name_without-nix}-lib`. +The unit test libraries are in `src/${library_name_without-nix}-test-support`. All headers are in a `tests` subdirectory so they are included with `#include "tests/"`. The use of all these separate directories for the unit tests might seem inconvenient, as for example the tests are not "right next to" the part of the code they are testing. @@ -76,8 +76,25 @@ there is no risk of any build-system wildcards for the library accidentally pick ### Running tests -You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. -Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option, or the `GTEST_FILTER` environment variable, e.g. `GTEST_FILTER='ErrorTraceTest.*' make check`. +You can run the whole testsuite with `meson test` from the Meson build directory, or the tests for a specific component with `meson test nix-store-tests`. +A environment variables that Google Test accepts are also worth knowing: + +1. [`GTEST_FILTER`](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) + + This is used for finer-grained filtering of which tests to run. + + +2. [`GTEST_BRIEF`](https://google.github.io/googletest/advanced.html#suppressing-test-passes) + + This is used to avoid logging passing tests. + +Putting the two together, one might run + +```bash +GTEST_BRIEF=1 GTEST_FILTER='ErrorTraceTest.*' meson test nix-expr-tests -v +``` + +for short but comprensive output. ### Characterisation testing { #characaterisation-testing-unit } @@ -86,7 +103,7 @@ See [functional characterisation testing](#characterisation-testing-functional) Like with the functional characterisation, `_NIX_TEST_ACCEPT=1` is also used. For example: ```shell-session -$ _NIX_TEST_ACCEPT=1 make libstore-tests_RUN +$ _NIX_TEST_ACCEPT=1 meson test nix-store-tests -v ... [ SKIPPED ] WorkerProtoTest.string_read [ SKIPPED ] WorkerProtoTest.string_write @@ -114,6 +131,8 @@ On other platforms they wouldn't be run at all. The functional tests reside under the `tests/functional` directory and are listed in `tests/functional/local.mk`. Each test is a bash script. +Functional tests are run during `installCheck` in the `nix` package build, as well as separately from the build, in VM tests. + ### Running the whole test suite The whole test suite can be run with: @@ -162,14 +181,14 @@ ran test tests/functional/${testName}.sh... [PASS] or without `make`: ```shell-session -$ ./mk/run-test.sh tests/functional/${testName}.sh tests/functional/init.sh +$ ./mk/run-test.sh tests/functional/${testName}.sh ran test tests/functional/${testName}.sh... [PASS] ``` To see the complete output, one can also run: ```shell-session -$ ./mk/debug-test.sh tests/functional/${testName}.sh tests/functional/init.sh +$ ./mk/debug-test.sh tests/functional/${testName}.sh +(${testName}.sh:1) foo output from foo +(${testName}.sh:2) bar @@ -204,7 +223,7 @@ edit it like so: Then, running the test with `./mk/debug-test.sh` will drop you into GDB once the script reaches that point: ```shell-session -$ ./mk/debug-test.sh tests/functional/${testName}.sh tests/functional/init.sh +$ ./mk/debug-test.sh tests/functional/${testName}.sh ... + gdb blash blub GNU gdb (GDB) 12.1 @@ -252,13 +271,30 @@ Regressions are caught, and improvements always show up in code review. To ensure that characterisation testing doesn't make it harder to intentionally change these interfaces, there always must be an easy way to regenerate the expected output, as we do with `_NIX_TEST_ACCEPT=1`. +### Running functional tests on NixOS + +We run the functional tests not just in the build, but also in VM tests. +This helps us ensure that Nix works correctly on NixOS, and environments that have similar characteristics that are hard to reproduce in a build environment. + +The recommended way to run these tests during development is: + +```shell +nix build .#hydraJobs.tests.functional_user.quickBuild +``` + +The `quickBuild` attribute configures the test to use a `nix` package that's built without integration tests, so that you can iterate on the tests without performing recompilations due to the changed sources for `installCheck`. + +Generally, this build is sufficient, but in nightly or CI we also test the attributes `functional_root` and `functional_trusted`, in which the test suite is run with different levels of authorization. + ## Integration tests The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute. These tests include everything that needs to interact with external services or run Nix in a non-trivial distributed setup. Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on ). -You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}` +You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}`. + +If you are testing a build of `nix` that you haven't compiled yet, you may iterate faster by appending the `quickBuild` attribute: `nix build .#hydraJobs.tests.{testName}.quickBuild`. ## Installer tests diff --git a/doc/manual/src/favicon.png b/doc/manual/src/favicon.png new file mode 100644 index 000000000..1ed2b5fe0 Binary files /dev/null and b/doc/manual/src/favicon.png differ diff --git a/doc/manual/src/favicon.svg b/doc/manual/src/favicon.svg new file mode 100644 index 000000000..1d2a6e835 --- /dev/null +++ b/doc/manual/src/favicon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index a71b2e2b3..877c4668b 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -1,5 +1,24 @@ # Glossary +- [content address]{#gloss-content-address} + + A + [*content address*](https://en.wikipedia.org/wiki/Content-addressable_storage) + is a secure way to reference immutable data. + The reference is calculated directly from the content of the data being referenced, which means the reference is + [*tamper proof*](https://en.wikipedia.org/wiki/Tamperproofing) + --- variations of the data should always calculate to distinct content addresses. + + For how Nix uses content addresses, see: + + - [Content-Addressing File System Objects](@docroot@/store/file-system-object/content-address.md) + - [Content-Addressing Store Objects](@docroot@/store/store-object/content-address.md) + - [content-addressed derivation](#gloss-content-addressed-derivation) + + Software Heritage's writing on [*Intrinsic and Extrinsic identifiers*](https://www.softwareheritage.org/2020/07/09/intrinsic-vs-extrinsic-identifiers) is also a good introduction to the value of content-addressing over other referencing schemes. + + Besides content addressing, the Nix store also uses [input addressing](#gloss-input-addressed-store-object). + - [derivation]{#gloss-derivation} A description of a build task. The result of a derivation is a @@ -52,10 +71,9 @@ [`__contentAddressed`](./language/advanced-attributes.md#adv-attr-__contentAddressed) attribute set to `true`. -- [fixed-output derivation]{#gloss-fixed-output-derivation} +- [fixed-output derivation]{#gloss-fixed-output-derivation} (FOD) - A derivation which includes the - [`outputHash`](./language/advanced-attributes.md#adv-attr-outputHash) attribute. + A [derivation] where a cryptographic hash of the [output] is determined in advance using the [`outputHash`](./language/advanced-attributes.md#adv-attr-outputHash) attribute, and where the [`builder`](@docroot@/language/derivations.md#attr-builder) executable has access to the network. - [store]{#gloss-store} @@ -86,7 +104,7 @@ [store path]: #gloss-store-path -- [file system object]{#gloss-store-object} +- [file system object]{#gloss-file-system-object} The Nix data model for representing simplified file system data. @@ -118,9 +136,12 @@ - [content-addressed store object]{#gloss-content-addressed-store-object} - A [store object] whose [store path] is determined by its contents. + A [store object] which is [content-addressed](#gloss-content-address), + i.e. whose [store path] is determined by its contents. This includes derivations, the outputs of [content-addressed derivations](#gloss-content-addressed-derivation), and the outputs of [fixed-output derivations](#gloss-fixed-output-derivation). + See [Content-Addressing Store Objects](@docroot@/store/store-object/content-address.md) for details. + - [substitute]{#gloss-substitute} A substitute is a command invocation stored in the [Nix database] that @@ -147,7 +168,7 @@ - [impure derivation]{#gloss-impure-derivation} - [An experimental feature](#@docroot@/contributing/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, + [An experimental feature](#@docroot@/development/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, so that they are always rebuilt, and their outputs not reused by subsequent calls to realise them. - [Nix database]{#gloss-nix-database} @@ -215,6 +236,20 @@ [output path]: #gloss-output-path +- [output closure]{#gloss-output-closure}\ + The [closure] of an [output path]. It only contains what is [reachable] from the output. + +- [deriving path]{#gloss-deriving-path} + + Deriving paths are a way to refer to [store objects][store object] that ar not yet [realised][realise]. + This is necessary because, in general and particularly for [content-addressed derivations][content-addressed derivation], the [output path] of an [output] is not known in advance. + There are two forms: + + - *constant*: just a [store path] + It can be made [valid][validity] by copying it into the store: from the evaluator, command line interface or another store. + + - *output*: a pair of a [store path] to a [derivation] and an [output] name. + - [deriver]{#gloss-deriver} The [store derivation] that produced an [output path]. @@ -252,13 +287,15 @@ See [installables](./command-ref/new-cli/nix.md#installables) for [`nix` commands](./command-ref/new-cli/nix.md) (experimental) for details. -- [NAR]{#gloss-nar} +- [Nix Archive (NAR)]{#gloss-nar} A *N*ix *AR*chive. This is a serialisation of a path in the Nix store. It can contain regular files, directories and symbolic links. NARs are generated and unpacked using `nix-store --dump` and `nix-store --restore`. + See [Nix Archive](store/file-system-object/content-address.html#serial-nix-archive) for details. + - [`∅`]{#gloss-emtpy-set} The empty set symbol. In the context of profile history, this denotes a package is not present in a particular version of the profile. @@ -275,7 +312,7 @@ - [package attribute set]{#package-attribute-set} - An [attribute set](@docroot@/language/values.md#attribute-set) containing the attribute `type = "derivation";` (derivation for historical reasons), as well as other attributes, such as + An [attribute set](@docroot@/language/types.md#attribute-set) containing the attribute `type = "derivation";` (derivation for historical reasons), as well as other attributes, such as - attributes that refer to the files of a [package], typically in the form of [derivation outputs](#output), - attributes that declare something about how the package is supposed to be installed or used, - other metadata or arbitrary attributes. @@ -288,16 +325,35 @@ See [String interpolation](./language/string-interpolation.md) for details. - [string]: ./language/values.md#type-string - [path]: ./language/values.md#type-path - [attribute name]: ./language/values.md#attribute-set + [string]: ./language/types.md#type-string + [path]: ./language/types.md#type-path + [attribute name]: ./language/types.md#attribute-set + +- [base directory]{#gloss-base-directory} + + The location from which relative paths are resolved. + + - For expressions in a file, the base directory is the directory containing that file. + This is analogous to the directory of a [base URL](https://datatracker.ietf.org/doc/html/rfc1808#section-3.3). + + + + - For expressions written in command line arguments with [`--expr`](@docroot@/command-ref/opt-common.html#opt-expr), the base directory is the current working directory. + + [base directory]: #gloss-base-directory - [experimental feature]{#gloss-experimental-feature} Not yet stabilized functionality guarded by named experimental feature flags. These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting. - See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/contributing/experimental-features.md). + See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/development/experimental-features.md). [Nix language]: ./language/index.md diff --git a/doc/manual/src/installation/env-variables.md b/doc/manual/src/installation/env-variables.md index db98f52ff..035090421 100644 --- a/doc/manual/src/installation/env-variables.md +++ b/doc/manual/src/installation/env-variables.md @@ -53,7 +53,8 @@ ssl-cert-file = /etc/ssl/my-certificate-bundle.crt The Nix installer has special handling for these proxy-related environment variables: `http_proxy`, `https_proxy`, `ftp_proxy`, -`no_proxy`, `HTTP_PROXY`, `HTTPS_PROXY`, `FTP_PROXY`, `NO_PROXY`. +`all_proxy`, `no_proxy`, `HTTP_PROXY`, `HTTPS_PROXY`, `FTP_PROXY`, +`ALL_PROXY`, `NO_PROXY`. If any of these variables are set when running the Nix installer, then the installer will create an override file at diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index 0dc989159..6a168ff3d 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -50,7 +50,7 @@ Supported systems: To explicitly instruct the installer to perform a multi-user installation on your system: ```console -$ curl -L https://nixos.org/nix/install | sh -s -- --daemon +$ bash <(curl -L https://nixos.org/nix/install) --daemon ``` You can run this under your usual user account or `root`. @@ -61,7 +61,7 @@ The script will invoke `sudo` as needed. To explicitly select a single-user installation on your system: ```console -$ curl -L https://nixos.org/nix/install | sh -s -- --no-daemon +$ bash <(curl -L https://nixos.org/nix/install) --no-daemon ``` In a single-user installation, `/nix` is owned by the invoking user. @@ -77,7 +77,7 @@ $ su root # Installing from a binary tarball You can also download a binary tarball that contains Nix and all its dependencies: -- Choose a [version](https://releases.nixos.org/?prefix=nix/) and [system type](../contributing/hacking.md#platforms) +- Choose a [version](https://releases.nixos.org/?prefix=nix/) and [system type](../development/building.md#platforms) - Download and unpack the tarball - Run the installer diff --git a/doc/manual/src/installation/uninstall.md b/doc/manual/src/installation/uninstall.md index 9ead5e53c..590327fea 100644 --- a/doc/manual/src/installation/uninstall.md +++ b/doc/manual/src/installation/uninstall.md @@ -1,16 +1,8 @@ # Uninstalling Nix -## Single User - -If you have a [single-user installation](./installing-binary.md#single-user-installation) of Nix, uninstall it by running: - -```console -$ rm -rf /nix -``` - ## Multi User -Removing a [multi-user installation](./installing-binary.md#multi-user-installation) of Nix is more involved, and depends on the operating system. +Removing a [multi-user installation](./installing-binary.md#multi-user-installation) depends on the operating system. ### Linux @@ -51,7 +43,15 @@ which you may remove. ### macOS -1. Edit `/etc/zshrc`, `/etc/bashrc`, and `/etc/bash.bashrc` to remove the lines sourcing `nix-daemon.sh`, which should look like this: +1. If system-wide shell initialisation files haven't been altered since installing Nix, use the backups made by the installer: + + ```console + sudo mv /etc/zshrc.backup-before-nix /etc/zshrc + sudo mv /etc/bashrc.backup-before-nix /etc/bashrc + sudo mv /etc/bash.bashrc.backup-before-nix /etc/bash.bashrc + ``` + + Otherwise, edit `/etc/zshrc`, `/etc/bashrc`, and `/etc/bash.bashrc` to remove the lines sourcing `nix-daemon.sh`, which should look like this: ```bash # Nix @@ -61,18 +61,6 @@ which you may remove. # End Nix ``` - If these files haven't been altered since installing Nix you can simply put - the backups back in place: - - ```console - sudo mv /etc/zshrc.backup-before-nix /etc/zshrc - sudo mv /etc/bashrc.backup-before-nix /etc/bashrc - sudo mv /etc/bash.bashrc.backup-before-nix /etc/bash.bashrc - ``` - - This will stop shells from sourcing the file and bringing everything you - installed using Nix in scope. - 2. Stop and remove the Nix daemon services: ```console @@ -82,8 +70,7 @@ which you may remove. sudo rm /Library/LaunchDaemons/org.nixos.darwin-store.plist ``` - This stops the Nix daemon and prevents it from being started next time you - boot the system. + This stops the Nix daemon and prevents it from being started next time you boot the system. 3. Remove the `nixbld` group and the `_nixbuildN` users: @@ -94,25 +81,42 @@ which you may remove. This will remove all the build users that no longer serve a purpose. -4. Edit fstab using `sudo vifs` to remove the line mounting the Nix Store - volume on `/nix`, which looks like - `UUID= /nix apfs rw,noauto,nobrowse,suid,owners` or - `LABEL=Nix\040Store /nix apfs rw,nobrowse`. This will prevent automatic - mounting of the Nix Store volume. +4. Edit fstab using `sudo vifs` to remove the line mounting the Nix Store volume on `/nix`, which looks like -5. Edit `/etc/synthetic.conf` to remove the `nix` line. If this is the only - line in the file you can remove it entirely, `sudo rm /etc/synthetic.conf`. - This will prevent the creation of the empty `/nix` directory to provide a - mountpoint for the Nix Store volume. + ``` + UUID= /nix apfs rw,noauto,nobrowse,suid,owners + ``` + or -6. Remove the files Nix added to your system: + ``` + LABEL=Nix\040Store /nix apfs rw,nobrowse + ``` + + by setting the cursor on the respective line using the error keys, and pressing `dd`, and then `:wq` to save the file. + + This will prevent automatic mounting of the Nix Store volume. + +5. Edit `/etc/synthetic.conf` to remove the `nix` line. + If this is the only line in the file you can remove it entirely: + + ```bash + if [ -f /etc/synthetic.conf ]; then + if [ "$(cat /etc/synthetic.conf)" = "nix" ]; then + sudo rm /etc/synthetic.conf + else + sudo vi /etc/synthetic.conf + fi + fi + ``` + + This will prevent the creation of the empty `/nix` directory. + +6. Remove the files Nix added to your system, except for the store: ```console sudo rm -rf /etc/nix /var/root/.nix-profile /var/root/.nix-defexpr /var/root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels ``` - This gets rid of any data Nix may have created except for the store which is - removed next. 7. Remove the Nix Store volume: @@ -120,29 +124,32 @@ which you may remove. sudo diskutil apfs deleteVolume /nix ``` - This will remove the Nix Store volume and everything that was added to the - store. + This will remove the Nix Store volume and everything that was added to the store. - If the output indicates that the command couldn't remove the volume, you should - make sure you don't have an _unmounted_ Nix Store volume. Look for a - "Nix Store" volume in the output of the following command: + If the output indicates that the command couldn't remove the volume, you should make sure you don't have an _unmounted_ Nix Store volume. + Look for a "Nix Store" volume in the output of the following command: ```console diskutil list ``` - If you _do_ see a "Nix Store" volume, delete it by re-running the diskutil - deleteVolume command, but replace `/nix` with the store volume's `diskXsY` - identifier. + If you _do_ find a "Nix Store" volume, delete it by running `diskutil deleteVolume` with the store volume's `diskXsY` identifier. > **Note** > -> After you complete the steps here, you will still have an empty `/nix` -> directory. This is an expected sign of a successful uninstall. The empty -> `/nix` directory will disappear the next time you reboot. +> After you complete the steps here, you will still have an empty `/nix` directory. +> This is an expected sign of a successful uninstall. +> The empty `/nix` directory will disappear the next time you reboot. > -> You do not have to reboot to finish uninstalling Nix. The uninstall is -> complete. macOS (Catalina+) directly controls root directories and its -> read-only root will prevent you from manually deleting the empty `/nix` -> mountpoint. +> You do not have to reboot to finish uninstalling Nix. +> The uninstall is complete. +> macOS (Catalina+) directly controls root directories, and its read-only root will prevent you from manually deleting the empty `/nix` mountpoint. +## Single User + +To remove a [single-user installation](./installing-binary.md#single-user-installation) of Nix, run: + +```console +$ rm -rf /nix ~/.nix-channels ~/.nix-defexpr ~/.nix-profile +``` +You might also want to manually remove references to Nix from your `~/.profile`. diff --git a/doc/manual/src/installation/upgrading.md b/doc/manual/src/installation/upgrading.md index 38edcdbc5..a433f1d30 100644 --- a/doc/manual/src/installation/upgrading.md +++ b/doc/manual/src/installation/upgrading.md @@ -28,7 +28,7 @@ $ sudo su ## macOS multi-user ```console -$ sudo nix-env --install --file '' --attr nix -I nixpkgs=channel:nixpkgs-unstable +$ sudo nix-env --install --file '' --attr nix cacert -I nixpkgs=channel:nixpkgs-unstable $ sudo launchctl remove org.nixos.nix-daemon $ sudo launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist ``` diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 7306fc182..51b83fc8a 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -113,19 +113,18 @@ Derivations can declare some infrequently used optional attributes. > `nix-build`. If the [`configurable-impure-env` experimental - feature](@docroot@/contributing/experimental-features.md#xp-feature-configurable-impure-env) + feature](@docroot@/development/experimental-features.md#xp-feature-configurable-impure-env) is enabled, these environment variables can also be controlled through the [`impure-env`](@docroot@/command-ref/conf-file.md#conf-impure-env) configuration setting. - [`outputHash`]{#adv-attr-outputHash}; [`outputHashAlgo`]{#adv-attr-outputHashAlgo}; [`outputHashMode`]{#adv-attr-outputHashMode}\ - These attributes declare that the derivation is a so-called - *fixed-output derivation*, which means that a cryptographic hash of - the output is already known in advance. When the build of a - fixed-output derivation finishes, Nix computes the cryptographic - hash of the output and compares it to the hash declared with these - attributes. If there is a mismatch, the build fails. + These attributes declare that the derivation is a so-called *fixed-output derivation* (FOD), which means that a cryptographic hash of the output is already known in advance. + + As opposed to regular derivations, the [`builder`] executable of a fixed-output derivation has access to the network. + Nix computes a cryptographic hash of its output and compares that to the hash declared with these attributes. + If there is a mismatch, the derivation fails. The rationale for fixed-output derivations is derivations such as those produced by the `fetchurl` function. This function downloads a @@ -188,38 +187,49 @@ Derivations can declare some infrequently used optional attributes. } ``` - The `outputHashAlgo` attribute specifies the hash algorithm used to - compute the hash. It can currently be `"sha1"`, `"sha256"` or - `"sha512"`. + The `outputHash` attribute must be a string containing the hash in either hexadecimal or "nix32" encoding, or following the format for integrity metadata as defined by [SRI](https://www.w3.org/TR/SRI/). + The "nix32" encoding is an adaptation of base-32 encoding. + The [`convertHash`](@docroot@/language/builtins.md#builtins-convertHash) function shows how to convert between different encodings, and the [`nix-hash` command](../command-ref/nix-hash.md) has information about obtaining the hash for some contents, as well as converting to and from encodings. + + The `outputHashAlgo` attribute specifies the hash algorithm used to compute the hash. + It can currently be `"sha1"`, `"sha256"`, `"sha512"`, or `null`. + `outputHashAlgo` can only be `null` when `outputHash` follows the SRI format. The `outputHashMode` attribute determines how the hash is computed. - It must be one of the following two values: + It must be one of the following values: - - `"flat"`\ - The output must be a non-executable regular file. If it isn’t, - the build fails. The hash is simply computed over the contents - of that file (so it’s equal to what Unix commands like - `sha256sum` or `sha1sum` produce). + - [`"flat"`](@docroot@/store/store-object/content-address.md#method-flat) This is the default. - - `"recursive"`\ - The hash is computed over the NAR archive dump of the output - (i.e., the result of [`nix-store --dump`](@docroot@/command-ref/nix-store/dump.md)). In - this case, the output can be anything, including a directory - tree. + - [`"recursive"` or `"nar"`](@docroot@/store/store-object/content-address.md#method-nix-archive) - The `outputHash` attribute, finally, must be a string containing - the hash in either hexadecimal or base-32 notation. (See the - [`nix-hash` command](../command-ref/nix-hash.md) for information - about converting to and from base-32 notation.) + > **Compatibility** + > + > `"recursive"` is the traditional way of indicating this, + > and is supported since 2005 (virtually the entire history of Nix). + > `"nar"` is more clear, and consistent with other parts of Nix (such as the CLI), + > however support for it is only added in Nix version 2.21. + + - [`"text"`](@docroot@/store/store-object/content-address.md#method-text) + + > **Warning** + > + > The use of this method for derivation outputs is part of the [`dynamic-derivations`][xp-feature-dynamic-derivations] experimental feature. + + - [`"git"`](@docroot@/store/store-object/content-address.md#method-git) + + > **Warning** + > + > This method is part of the [`git-hashing`][xp-feature-git-hashing] experimental feature. - [`__contentAddressed`]{#adv-attr-__contentAddressed} + > **Warning** - > This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md). + > This attribute is part of an [experimental feature](@docroot@/development/experimental-features.md). > > To use this attribute, you must enable the - > [`ca-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-ca-derivations) experimental feature. + > [`ca-derivations`][xp-feature-ca-derivations] experimental feature. > For example, in [nix.conf](../command-ref/conf-file.md) you could add: > > ``` @@ -268,7 +278,9 @@ Derivations can declare some infrequently used optional attributes. > **Note** > - > If set to `false`, the [`builder`](./derivations.md#attr-builder) should be able to run on the system type specified in the [`system` attribute](./derivations.md#attr-system), since the derivation cannot be substituted. + > If set to `false`, the [`builder`] should be able to run on the system type specified in the [`system` attribute](./derivations.md#attr-system), since the derivation cannot be substituted. + + [`builder`]: ./derivations.md#attr-builder - [`__structuredAttrs`]{#adv-attr-structuredAttrs}\ If the special attribute `__structuredAttrs` is set to `true`, the other derivation @@ -290,6 +302,12 @@ Derivations can declare some infrequently used optional attributes. (associative) arrays. For example, the attribute `hardening.format = true` ends up as the Bash associative array element `${hardening[format]}`. + > **Warning** + > + > If set to `true`, other advanced attributes such as [`allowedReferences`](#adv-attr-allowedReferences), [`allowedReferences`](#adv-attr-allowedReferences), [`allowedRequisites`](#adv-attr-allowedRequisites), + [`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites), maxSize, and maxClosureSize. + will have no effect. + - [`outputChecks`]{#adv-attr-outputChecks}\ When using [structured attributes](#adv-attr-structuredAttrs), the `outputChecks` attribute allows defining checks per-output. @@ -299,7 +317,7 @@ Derivations can declare some infrequently used optional attributes. [`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites), the following attributes are available: - - `maxSize` defines the maximum size of the resulting [store object](../glossary.md#gloss-store-object). + - `maxSize` defines the maximum size of the resulting [store object](@docroot@/store/store-object.md). - `maxClosureSize` defines the maximum size of the output's closure. - `ignoreSelfRefs` controls whether self-references should be considered when checking for allowed references/requisites. @@ -351,3 +369,7 @@ Derivations can declare some infrequently used optional attributes. ``` ensures that the derivation can only be built on a machine with the `kvm` feature. + +[xp-feature-ca-derivations]: @docroot@/development/experimental-features.md#xp-feature-ca-derivations +[xp-feature-dynamic-derivations]: @docroot@/development/experimental-features.md#xp-feature-dynamic-derivations +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/doc/manual/src/language/builtin-constants-prefix.md b/doc/manual/src/language/builtin-constants-prefix.md deleted file mode 100644 index 50f43006d..000000000 --- a/doc/manual/src/language/builtin-constants-prefix.md +++ /dev/null @@ -1,5 +0,0 @@ -# Built-in Constants - -These constants are built into the Nix language evaluator: - - diff --git a/doc/manual/src/language/builtin-constants-suffix.md b/doc/manual/src/language/builtin-constants-suffix.md deleted file mode 100644 index a74db2857..000000000 --- a/doc/manual/src/language/builtin-constants-suffix.md +++ /dev/null @@ -1 +0,0 @@ - diff --git a/doc/manual/src/language/builtins-prefix.md b/doc/manual/src/language/builtins-prefix.md index 7b2321466..fb983bb7f 100644 --- a/doc/manual/src/language/builtins-prefix.md +++ b/doc/manual/src/language/builtins-prefix.md @@ -1,9 +1,11 @@ -# Built-in Functions +# Built-ins -This section lists the functions built into the Nix language evaluator. -All built-in functions are available through the global [`builtins`](./builtin-constants.md#builtins-builtins) constant. +This section lists the values and functions built into the Nix language evaluator. +All built-ins are available through the global [`builtins`](#builtins-builtins) constant. -For convenience, some built-ins can be accessed directly: +Some built-ins are also exposed directly in the global scope: + + - [`derivation`](#builtins-derivation) - [`import`](#builtins-import) diff --git a/doc/manual/src/language/constructs.md b/doc/manual/src/language/constructs.md deleted file mode 100644 index a82ec5960..000000000 --- a/doc/manual/src/language/constructs.md +++ /dev/null @@ -1,408 +0,0 @@ -# Language Constructs - -## Recursive sets - -Recursive sets are like normal [attribute sets](./values.md#attribute-set), but the attributes can refer to each other. - -> *rec-attrset* = `rec {` [ *name* `=` *expr* `;` `]`... `}` - -Example: - -```nix -rec { - x = y; - y = 123; -}.x -``` - -This evaluates to `123`. - -Note that without `rec` the binding `x = y;` would -refer to the variable `y` in the surrounding scope, if one exists, and -would be invalid if no such variable exists. That is, in a normal -(non-recursive) set, attributes are not added to the lexical scope; in a -recursive set, they are. - -Recursive sets of course introduce the danger of infinite recursion. For -example, the expression - -```nix -rec { - x = y; - y = x; -}.x -``` - -will crash with an `infinite recursion encountered` error message. - -## Let-expressions - -A let-expression allows you to define local variables for an expression. - -> *let-in* = `let` [ *identifier* = *expr* ]... `in` *expr* - -Example: - -```nix -let - x = "foo"; - y = "bar"; -in x + y -``` - -This evaluates to `"foobar"`. - -## Inheriting attributes - -When defining an [attribute set](./values.md#attribute-set) or in a [let-expression](#let-expressions) it is often convenient to copy variables from the surrounding lexical scope (e.g., when you want to propagate attributes). -This can be shortened using the `inherit` keyword. - -Example: - -```nix -let x = 123; in -{ - inherit x; - y = 456; -} -``` - -is equivalent to - -```nix -let x = 123; in -{ - x = x; - y = 456; -} -``` - -and both evaluate to `{ x = 123; y = 456; }`. - -> **Note** -> -> This works because `x` is added to the lexical scope by the `let` construct. - -It is also possible to inherit attributes from another attribute set. - -Example: - -In this fragment from `all-packages.nix`, - -```nix -graphviz = (import ../tools/graphics/graphviz) { - inherit fetchurl stdenv libpng libjpeg expat x11 yacc; - inherit (xorg) libXaw; -}; - -xorg = { - libX11 = ...; - libXaw = ...; - ... -} - -libpng = ...; -libjpg = ...; -... -``` - -the set used in the function call to the function defined in -`../tools/graphics/graphviz` inherits a number of variables from the -surrounding scope (`fetchurl` ... `yacc`), but also inherits `libXaw` -(the X Athena Widgets) from the `xorg` set. - -Summarizing the fragment - -```nix -... -inherit x y z; -inherit (src-set) a b c; -... -``` - -is equivalent to - -```nix -... -x = x; y = y; z = z; -a = src-set.a; b = src-set.b; c = src-set.c; -... -``` - -when used while defining local variables in a let-expression or while -defining a set. - -In a `let` expression, `inherit` can be used to selectively bring specific attributes of a set into scope. For example - - -```nix -let - x = { a = 1; b = 2; }; - inherit (builtins) attrNames; -in -{ - names = attrNames x; -} -``` - -is equivalent to - -```nix -let - x = { a = 1; b = 2; }; -in -{ - names = builtins.attrNames x; -} -``` - -both evaluate to `{ names = [ "a" "b" ]; }`. - -## Functions - -Functions have the following form: - -```nix -pattern: body -``` - -The pattern specifies what the argument of the function must look like, -and binds variables in the body to (parts of) the argument. There are -three kinds of patterns: - - - If a pattern is a single identifier, then the function matches any - argument. Example: - - ```nix - let negate = x: !x; - concat = x: y: x + y; - in if negate true then concat "foo" "bar" else "" - ``` - - Note that `concat` is a function that takes one argument and returns - a function that takes another argument. This allows partial - parameterisation (i.e., only filling some of the arguments of a - function); e.g., - - ```nix - map (concat "foo") [ "bar" "bla" "abc" ] - ``` - - evaluates to `[ "foobar" "foobla" "fooabc" ]`. - - - A *set pattern* of the form `{ name1, name2, …, nameN }` matches a - set containing the listed attributes, and binds the values of those - attributes to variables in the function body. For example, the - function - - ```nix - { x, y, z }: z + y + x - ``` - - can only be called with a set containing exactly the attributes `x`, - `y` and `z`. No other attributes are allowed. If you want to allow - additional arguments, you can use an ellipsis (`...`): - - ```nix - { x, y, z, ... }: z + y + x - ``` - - This works on any set that contains at least the three named - attributes. - - It is possible to provide *default values* for attributes, in - which case they are allowed to be missing. A default value is - specified by writing `name ? e`, where *e* is an arbitrary - expression. For example, - - ```nix - { x, y ? "foo", z ? "bar" }: z + y + x - ``` - - specifies a function that only requires an attribute named `x`, but - optionally accepts `y` and `z`. - - - An `@`-pattern provides a means of referring to the whole value - being matched: - - ```nix - args@{ x, y, z, ... }: z + y + x + args.a - ``` - - but can also be written as: - - ```nix - { x, y, z, ... } @ args: z + y + x + args.a - ``` - - Here `args` is bound to the argument *as passed*, which is further - matched against the pattern `{ x, y, z, ... }`. - The `@`-pattern makes mainly sense with an ellipsis(`...`) as - you can access attribute names as `a`, using `args.a`, which was - given as an additional attribute to the function. - - > **Warning** - > - > `args@` binds the name `args` to the attribute set that is passed to the function. - > In particular, `args` does *not* include any default values specified with `?` in the function's set pattern. - > - > For instance - > - > ```nix - > let - > f = args@{ a ? 23, ... }: [ a args ]; - > in - > f {} - > ``` - > - > is equivalent to - > - > ```nix - > let - > f = args @ { ... }: [ (args.a or 23) args ]; - > in - > f {} - > ``` - > - > and both expressions will evaluate to: - > - > ```nix - > [ 23 {} ] - > ``` - -Note that functions do not have names. If you want to give them a name, -you can bind them to an attribute, e.g., - -```nix -let concat = { x, y }: x + y; -in concat { x = "foo"; y = "bar"; } -``` - -## Conditionals - -Conditionals look like this: - -```nix -if e1 then e2 else e3 -``` - -where *e1* is an expression that should evaluate to a Boolean value -(`true` or `false`). - -## Assertions - -Assertions are generally used to check that certain requirements on or -between features and dependencies hold. They look like this: - -```nix -assert e1; e2 -``` - -where *e1* is an expression that should evaluate to a Boolean value. If -it evaluates to `true`, *e2* is returned; otherwise expression -evaluation is aborted and a backtrace is printed. - -Here is a Nix expression for the Subversion package that shows how -assertions can be used:. - -```nix -{ localServer ? false -, httpServer ? false -, sslSupport ? false -, pythonBindings ? false -, javaSwigBindings ? false -, javahlBindings ? false -, stdenv, fetchurl -, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null -}: - -assert localServer -> db4 != null; â‘ -assert httpServer -> httpd != null && httpd.expat == expat; â‘¡ -assert sslSupport -> openssl != null && (httpServer -> httpd.openssl == openssl); â‘¢ -assert pythonBindings -> swig != null && swig.pythonSupport; -assert javaSwigBindings -> swig != null && swig.javaSupport; -assert javahlBindings -> j2sdk != null; - -stdenv.mkDerivation { - name = "subversion-1.1.1"; - ... - openssl = if sslSupport then openssl else null; â‘£ - ... -} -``` - -The points of interest are: - -1. This assertion states that if Subversion is to have support for - local repositories, then Berkeley DB is needed. So if the Subversion - function is called with the `localServer` argument set to `true` but - the `db4` argument set to `null`, then the evaluation fails. - - Note that `->` is the [logical - implication](https://en.wikipedia.org/wiki/Truth_table#Logical_implication) - Boolean operation. - -2. This is a more subtle condition: if Subversion is built with Apache - (`httpServer`) support, then the Expat library (an XML library) used - by Subversion should be same as the one used by Apache. This is - because in this configuration Subversion code ends up being linked - with Apache code, and if the Expat libraries do not match, a build- - or runtime link error or incompatibility might occur. - -3. This assertion says that in order for Subversion to have SSL support - (so that it can access `https` URLs), an OpenSSL library must be - passed. Additionally, it says that *if* Apache support is enabled, - then Apache's OpenSSL should match Subversion's. (Note that if - Apache support is not enabled, we don't care about Apache's - OpenSSL.) - -4. The conditional here is not really related to assertions, but is - worth pointing out: it ensures that if SSL support is disabled, then - the Subversion derivation is not dependent on OpenSSL, even if a - non-`null` value was passed. This prevents an unnecessary rebuild of - Subversion if OpenSSL changes. - -## With-expressions - -A *with-expression*, - -```nix -with e1; e2 -``` - -introduces the set *e1* into the lexical scope of the expression *e2*. -For instance, - -```nix -let as = { x = "foo"; y = "bar"; }; -in with as; x + y -``` - -evaluates to `"foobar"` since the `with` adds the `x` and `y` attributes -of `as` to the lexical scope in the expression `x + y`. The most common -use of `with` is in conjunction with the `import` function. E.g., - -```nix -with (import ./definitions.nix); ... -``` - -makes all attributes defined in the file `definitions.nix` available as -if they were defined locally in a `let`-expression. - -The bindings introduced by `with` do not shadow bindings introduced by -other means, e.g. - -```nix -let a = 3; in with { a = 1; }; let a = 4; in with { a = 2; }; ... -``` - -establishes the same scope as - -```nix -let a = 1; in let a = 2; in let a = 3; in let a = 4; in ... -``` - -## Comments - -Comments can be single-line, started with a `#` character, or -inline/multi-line, enclosed within `/* ... */`. diff --git a/doc/manual/src/language/constructs/lookup-path.md b/doc/manual/src/language/constructs/lookup-path.md index e87d2922b..11b9fe88c 100644 --- a/doc/manual/src/language/constructs/lookup-path.md +++ b/doc/manual/src/language/constructs/lookup-path.md @@ -4,9 +4,9 @@ > > *lookup-path* = `<` *identifier* [ `/` *identifier* ]... `>` -A lookup path is an identifier with an optional path suffix that resolves to a [path value](@docroot@/language/values.md#type-path) if the identifier matches a search path entry. +A lookup path is an identifier with an optional path suffix that resolves to a [path value](@docroot@/language/types.md#type-path) if the identifier matches a search path entry. -The value of a lookup path is determined by [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath). +The value of a lookup path is determined by [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath). See [`builtins.findFile`](@docroot@/language/builtins.md#builtins-findFile) for details on lookup path resolution. diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index 75f824a34..8e3f0f791 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -12,12 +12,12 @@ It outputs an attribute set, and produces a [store derivation] as a side effect ### Required -- [`name`]{#attr-name} ([String](@docroot@/language/values.md#type-string)) +- [`name`]{#attr-name} ([String](@docroot@/language/types.md#type-string)) A symbolic name for the derivation. It is added to the [store path] of the corresponding [store derivation] as well as to its [output paths](@docroot@/glossary.md#gloss-output-path). - [store path]: @docroot@/glossary.md#gloss-store-path + [store path]: @docroot@/store/store-path.md > **Example** > @@ -31,7 +31,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect > The store derivation's path will be `/nix/store/-hello.drv`. > The [output](#attr-outputs) paths will be of the form `/nix/store/-hello[-]` -- [`system`]{#attr-system} ([String](@docroot@/language/values.md#type-string)) +- [`system`]{#attr-system} ([String](@docroot@/language/types.md#type-string)) The system type on which the [`builder`](#attr-builder) executable is meant to be run. @@ -64,9 +64,9 @@ It outputs an attribute set, and produces a [store derivation] as a side effect > } > ``` > - > [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) has the value of the [`system` configuration option], and defaults to the system type of the current Nix installation. + > [`builtins.currentSystem`](@docroot@/language/builtins.md#builtins-currentSystem) has the value of the [`system` configuration option], and defaults to the system type of the current Nix installation. -- [`builder`]{#attr-builder} ([Path](@docroot@/language/values.md#type-path) | [String](@docroot@/language/values.md#type-string)) +- [`builder`]{#attr-builder} ([Path](@docroot@/language/types.md#type-path) | [String](@docroot@/language/types.md#type-string)) Path to an executable that will perform the build. @@ -113,7 +113,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect ### Optional -- [`args`]{#attr-args} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) +- [`args`]{#attr-args} ([List](@docroot@/language/types.md#list) of [String](@docroot@/language/types.md#type-string)) Default: `[ ]` @@ -132,7 +132,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect > }; > ``` -- [`outputs`]{#attr-outputs} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) +- [`outputs`]{#attr-outputs} ([List](@docroot@/language/types.md#list) of [String](@docroot@/language/types.md#type-string)) Default: `[ "out" ]` @@ -141,7 +141,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect By default, a derivation produces a single output called `out`. However, derivations can produce multiple outputs. - This allows the associated [store objects](@docroot@/glossary.md#gloss-store-object) and their [closures](@docroot@/glossary.md#gloss-closure) to be copied or garbage-collected separately. + This allows the associated [store objects](@docroot@/store/store-object.md) and their [closures](@docroot@/glossary.md#gloss-closure) to be copied or garbage-collected separately. > **Example** > diff --git a/doc/manual/src/language/identifiers.md b/doc/manual/src/language/identifiers.md new file mode 100644 index 000000000..861ee3e20 --- /dev/null +++ b/doc/manual/src/language/identifiers.md @@ -0,0 +1,51 @@ +# Identifiers + +An *identifier* is an [ASCII](https://en.wikipedia.org/wiki/ASCII) character sequence that: +- Starts with a letter (`a-z`, `A-Z`) or underscore (`_`) +- Can contain any number of: + - Letters (`a-z`, `A-Z`) + - Digits (`0-9`) + - Underscores (`_`) + - Apostrophes (`'`) + - Hyphens (`-`) +- Is not one of the [keywords](#keywords) + +> **Syntax** +> +> *identifier* ~ `[A-Za-z_][A-Za-z0-9_'-]*` + +# Names + +A *name* can be written as an [identifier](#identifier) or a [string literal](./syntax.md#string-literal). + +> **Syntax** +> +> *name* → *identifier* | *string* + +Names are used in [attribute sets](./syntax.md#attrs-literal), [`let` bindings](./syntax.md#let-expressions), and [`inherit`](./syntax.md#inheriting-attributes). +Two names are the same if they represent the same sequence of characters, regardless of whether they are written as identifiers or strings. + +# Keywords + +These keywords are reserved and cannot be used as [identifiers](#identifiers): + +- [`assert`](./syntax.md#assertions) +- [`else`][if] +- [`if`][if] +- [`in`][let] +- [`inherit`](./syntax.md#inheriting-attributes) +- [`let`][let] +- [`or`](./operators.md#attribute-selection) (see note) +- [`rec`](./syntax.md#recursive-sets) +- [`then`][if] +- [`with`](./syntax.md#with-expressions) + +[if]: ./syntax.md#conditionals +[let]: ./syntax.md#let-expressions + +> **Note** +> +> The Nix language evaluator currently allows `or` to be used as a name in some contexts, for backwards compatibility reasons. +> Users are advised not to rely on this. +> +> There are long-standing issues with how `or` is parsed as a name, which can't be resolved without making a breaking change to the language. diff --git a/doc/manual/src/language/import-from-derivation.md b/doc/manual/src/language/import-from-derivation.md index fb12ba51a..e901f5bcf 100644 --- a/doc/manual/src/language/import-from-derivation.md +++ b/doc/manual/src/language/import-from-derivation.md @@ -2,9 +2,9 @@ The value of a Nix expression can depend on the contents of a [store object]. -[store object]: @docroot@/glossary.md#gloss-store-object +[store object]: @docroot@/store/store-object.md -Passing an expression `expr` that evaluates to a [store path](@docroot@/glossary.md#gloss-store-path) to any built-in function which reads from the filesystem constitutes Import From Derivation (IFD): +Passing an expression `expr` that evaluates to a [store path](@docroot@/store/store-path.md) to any built-in function which reads from the filesystem constitutes Import From Derivation (IFD): - [`import`](./builtins.md#builtins-import)` expr` - [`builtins.readFile`](./builtins.md#builtins-readFile)` expr` diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md index a26e43a05..2bfdbb8a0 100644 --- a/doc/manual/src/language/index.md +++ b/doc/manual/src/language/index.md @@ -1,7 +1,13 @@ # Nix Language The Nix language is designed for conveniently creating and composing *derivations* – precise descriptions of how contents of existing files are used to derive new files. -It is: + +> **Tip** +> +> These pages are written as a reference. +> If you are learning Nix, nix.dev has a good [introduction to the Nix language](https://nix.dev/tutorials/nix-language). + +The language is: - *domain-specific* @@ -47,7 +53,7 @@ This is an incomplete overview of language features, by example. - *Basic values* + *Basic values ([primitives](@docroot@/language/types.md#primitives))* @@ -65,7 +71,7 @@ This is an incomplete overview of language features, by example. - A string + A [string](@docroot@/language/types.md#type-string) @@ -88,6 +94,18 @@ This is an incomplete overview of language features, by example. + + + + `# Explanation` + + + + + A [comment](@docroot@/language/syntax.md#comments). + + + @@ -100,7 +118,7 @@ This is an incomplete overview of language features, by example. - String interpolation (expands to `"hello world"`, `"1 2 3"`, `"/nix/store/-bash-/bin/sh"`) + [String interpolation](@docroot@/language/string-interpolation.md) (expands to `"hello world"`, `"1 2 3"`, `"/nix/store/-bash-/bin/sh"`) @@ -112,7 +130,7 @@ This is an incomplete overview of language features, by example. - Booleans + [Booleans](@docroot@/language/types.md#type-boolean) @@ -124,7 +142,7 @@ This is an incomplete overview of language features, by example. - Null value + [Null](@docroot@/language/types.md#type-null) value @@ -136,7 +154,7 @@ This is an incomplete overview of language features, by example. - An integer + An [integer](@docroot@/language/types.md#type-int) @@ -148,7 +166,7 @@ This is an incomplete overview of language features, by example. - A floating point number + A [floating point number](@docroot@/language/types.md#type-float) @@ -160,7 +178,7 @@ This is an incomplete overview of language features, by example. - An absolute path + An absolute [path](@docroot@/language/types.md#type-path) @@ -172,7 +190,7 @@ This is an incomplete overview of language features, by example. - A path relative to the file containing this Nix expression + A [path](@docroot@/language/types.md#type-path) relative to the file containing this Nix expression @@ -184,7 +202,7 @@ This is an incomplete overview of language features, by example. - A home path. Evaluates to the `"/.config"`. + A home [path](@docroot@/language/types.md#type-path). Evaluates to the `"/.config"`. @@ -196,7 +214,7 @@ This is an incomplete overview of language features, by example. - Search path for Nix files. Value determined by [`$NIX_PATH` environment variable](../command-ref/env-common.md#env-NIX_PATH). + A [lookup path](@docroot@/language/constructs/lookup-path.md) for Nix files. Value determined by [`$NIX_PATH` environment variable](../command-ref/env-common.md#env-NIX_PATH). @@ -220,7 +238,7 @@ This is an incomplete overview of language features, by example. - A set with attributes named `x` and `y` + An [attribute set](@docroot@/language/types.md#attribute-set) with attributes named `x` and `y` @@ -244,7 +262,7 @@ This is an incomplete overview of language features, by example. - A recursive set, equivalent to `{ x = "foo"; y = "foobar"; }` + A [recursive set](@docroot@/language/syntax.md#recursive-sets), equivalent to `{ x = "foo"; y = "foobar"; }`. @@ -260,7 +278,7 @@ This is an incomplete overview of language features, by example. - Lists with three elements. + [Lists](@docroot@/language/types.md#list) with three elements. @@ -344,7 +362,7 @@ This is an incomplete overview of language features, by example. - Attribute selection (evaluates to `1`) + [Attribute selection](@docroot@/language/types.md#attribute-set) (evaluates to `1`) @@ -356,7 +374,7 @@ This is an incomplete overview of language features, by example. - Attribute selection with default (evaluates to `3`) + [Attribute selection](@docroot@/language/types.md#attribute-set) with default (evaluates to `3`) @@ -392,7 +410,7 @@ This is an incomplete overview of language features, by example. - Conditional expression + [Conditional expression](@docroot@/language/syntax.md#conditionals). @@ -404,7 +422,7 @@ This is an incomplete overview of language features, by example. - Assertion check (evaluates to `"yes!"`). + [Assertion](@docroot@/language/syntax.md#assertions) check (evaluates to `"yes!"`). @@ -416,7 +434,7 @@ This is an incomplete overview of language features, by example. - Variable definition + Variable definition. See [`let`-expressions](@docroot@/language/syntax.md#let-expressions). @@ -428,14 +446,44 @@ This is an incomplete overview of language features, by example. - Add all attributes from the given set to the scope (evaluates to `1`) + Add all attributes from the given set to the scope (evaluates to `1`). + + See [`with`-expressions](@docroot@/language/syntax.md#with-expressions) for details and shadowing caveats. - *Functions (lambdas)* + `inherit pkgs src;` + + + + + Adds the variables to the current scope (attribute set or `let` binding). + Desugars to `pkgs = pkgs; src = src;`. + See [Inheriting attributes](@docroot@/language/syntax.md#inheriting-attributes). + + + + + + + `inherit (pkgs) lib stdenv;` + + + + + Adds the attributes, from the attribute set in parentheses, to the current scope (attribute set or `let` binding). + Desugars to `lib = pkgs.lib; stdenv = pkgs.stdenv;`. + See [Inheriting attributes](@docroot@/language/syntax.md#inheriting-attributes). + + + + + + + *[Functions](@docroot@/language/syntax.md#functions) (lambdas)* @@ -452,7 +500,7 @@ This is an incomplete overview of language features, by example. - A function that expects an integer and returns it increased by 1 + A [function](@docroot@/language/syntax.md#functions) that expects an integer and returns it increased by 1. @@ -464,7 +512,7 @@ This is an incomplete overview of language features, by example. - Curried function, equivalent to `x: (y: x + y)`. Can be used like a function that takes two arguments and returns their sum. + Curried [function](@docroot@/language/syntax.md#functions), equivalent to `x: (y: x + y)`. Can be used like a function that takes two arguments and returns their sum. @@ -476,7 +524,7 @@ This is an incomplete overview of language features, by example. - A function call (evaluates to 101) + A [function](@docroot@/language/syntax.md#functions) call (evaluates to 101) @@ -488,7 +536,7 @@ This is an incomplete overview of language features, by example. - A function bound to a variable and subsequently called by name (evaluates to 103) + A [function](@docroot@/language/syntax.md#functions) bound to a variable and subsequently called by name (evaluates to 103) @@ -500,7 +548,7 @@ This is an incomplete overview of language features, by example. - A function that expects a set with required attributes `x` and `y` and concatenates them + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attributes `x` and `y` and concatenates them @@ -512,7 +560,7 @@ This is an incomplete overview of language features, by example. - A function that expects a set with required attribute `x` and optional `y`, using `"bar"` as default value for `y` + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attribute `x` and optional `y`, using `"bar"` as default value for `y` @@ -524,7 +572,7 @@ This is an incomplete overview of language features, by example. - A function that expects a set with required attributes `x` and `y` and ignores any other attributes + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attributes `x` and `y` and ignores any other attributes @@ -538,7 +586,7 @@ This is an incomplete overview of language features, by example. - A function that expects a set with required attributes `x` and `y`, and binds the whole set to `args` + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attributes `x` and `y`, and binds the whole set to `args` @@ -562,7 +610,8 @@ This is an incomplete overview of language features, by example. - Load and return Nix expression in given file + Load and return Nix expression in given file. + See [import](@docroot@/language/builtins.md#builtins-import). @@ -574,7 +623,8 @@ This is an incomplete overview of language features, by example. - Apply a function to every element of a list (evaluates to `[ 2 4 6 ]`) + Apply a function to every element of a list (evaluates to `[ 2 4 6 ]`). + See [`map`](@docroot@/language/builtins.md#builtins-map). diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index 6fd66864b..e1c020781 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -26,12 +26,16 @@ | Logical conjunction (`AND`) | *bool* `&&` *bool* | left | 12 | | Logical disjunction (`OR`) | *bool* \|\| *bool* | left | 13 | | [Logical implication] | *bool* `->` *bool* | right | 14 | +| [Pipe operator] (experimental) | *expr* `\|>` *func* | left | 15 | +| [Pipe operator] (experimental) | *func* `<\|` *expr* | right | 15 | -[string]: ./values.md#type-string -[path]: ./values.md#type-path -[number]: ./values.md#type-number -[list]: ./values.md#list -[attribute set]: ./values.md#attribute-set +[string]: ./types.md#type-string +[path]: ./types.md#type-path +[number]: ./types.md#type-float +[list]: ./types.md#list +[attribute set]: ./types.md#attribute-set + + ## Attribute selection @@ -42,12 +46,6 @@ Select the attribute denoted by attribute path *attrpath* from [attribute set] *attrset*. If the attribute doesn’t exist, return the *expr* after `or` if provided, otherwise abort evaluation. -An attribute path is a dot-separated list of [attribute names](./values.md#attribute-set). - -> **Syntax** -> -> *attrpath* = *name* [ `.` *name* ]... - [Attribute selection]: #attribute-selection ## Has attribute @@ -61,7 +59,7 @@ The result is a [Boolean] value. See also: [`builtins.hasAttr`](@docroot@/language/builtins.md#builtins-hasAttr) -[Boolean]: ./values.md#type-boolean +[Boolean]: ./types.md#type-boolean [Has attribute]: #has-attribute @@ -128,8 +126,8 @@ The result is a string. > The file or directory at *path* must exist and is copied to the [store]. > The path appears in the result as the corresponding [store path]. -[store path]: ../glossary.md#gloss-store-path -[store]: ../glossary.md#gloss-store +[store path]: @docroot@/store/store-path.md +[store]: @docroot@/glossary.md#gloss-store [String and path concatenation]: #string-and-path-concatenation @@ -141,7 +139,7 @@ The result is a string. Update [attribute set] *attrset1* with names and values from *attrset2*. -The returned attribute set will have of all the attributes in *attrset1* and *attrset2*. +The returned attribute set will have all of the attributes in *attrset1* and *attrset2*. If an attribute name is present in both, the attribute value from the latter is taken. [Update]: #update @@ -172,7 +170,7 @@ All comparison operators are implemented in terms of `<`, and the following equi - Numbers are type-compatible, see [arithmetic] operators. - Floating point numbers only differ up to a limited precision. -[function]: ./constructs.md#functions +[function]: ./syntax.md#functions [Equality]: #equality @@ -182,3 +180,34 @@ Equivalent to `!`*b1* `||` *b2*. [Logical implication]: #logical-implication +## Pipe operators + +- *a* `|>` *b* is equivalent to *b* *a* +- *a* `<|` *b* is equivalent to *a* *b* + +> **Example** +> +> ``` +> nix-repl> 1 |> builtins.add 2 |> builtins.mul 3 +> 9 +> +> nix-repl> builtins.add 1 <| builtins.mul 2 <| 3 +> 7 +> ``` + +> **Warning** +> +> This syntax is part of an +> [experimental feature](@docroot@/development/experimental-features.md) +> and may change in future releases. +> +> To use this syntax, make sure the +> [`pipe-operators` experimental feature](@docroot@/development/experimental-features.md#xp-feature-pipe-operators) +> is enabled. +> For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): +> +> ``` +> extra-experimental-features = pipe-operators +> ``` + +[Pipe operator]: #pipe-operators diff --git a/doc/manual/src/language/scope.md b/doc/manual/src/language/scope.md new file mode 100644 index 000000000..9373324e2 --- /dev/null +++ b/doc/manual/src/language/scope.md @@ -0,0 +1,28 @@ +# Scoping rules + +A *scope* in the Nix language is a dictionary keyed by [name](./identifiers.md#names), mapping each name to an expression and a *definition type*. +The definition type is either *explicit* or *implicit*. +Each entry in this dictionary is a *definition*. + +Explicit definitions are created by the following expressions: +- [let-expressions](syntax.md#let-expressions) +- [recursive attribute set literals](syntax.md#recursive-sets) (`rec`) +- [function literals](syntax.md#functions) + +Implicit definitions are only created by [with-expressions](./syntax.md#with-expressions). + +Every expression is *enclosed* by a scope. +The outermost expression is enclosed by the [built-in, global scope](./builtins.md), which contains only explicit definitions. +The expressions listed above *extend* their enclosing scope by adding new definitions, or replacing existing ones with the same name. +An explicit definition can replace a definition of any type; an implicit definition can only replace another implicit definition. + +Each of the above expressions defines which of its subexpressions are enclosed by the extended scope. +In all other cases, the same scope that encloses an expression is the enclosing scope for its subexpressions. + +The Nix language is [statically scoped](https://en.wikipedia.org/wiki/Scope_(computer_science)#Lexical_scope); +the value of a variable is determined only by the variable's enclosing scope, and not by the dynamic context in which the variable is evaluated. + +> **Note** +> +> Expressions entered into the [Nix REPL](@docroot@/command-ref/new-cli/nix3-repl.md) are enclosed by a scope that can be extended by command line arguments or previous REPL commands. +> These ways of extending scope are not, strictly speaking, part of the Nix language. diff --git a/doc/manual/src/language/string-context.md b/doc/manual/src/language/string-context.md new file mode 100644 index 000000000..6a3482cfd --- /dev/null +++ b/doc/manual/src/language/string-context.md @@ -0,0 +1,134 @@ +# String context + +> **Note** +> +> This is an advanced topic. +> The Nix language is designed to be used without the programmer consciously dealing with string contexts or even knowing what they are. + +A string in the Nix language is not just a sequence of characters like strings in other languages. +It is actually a pair of a sequence of characters and a *string context*. +The string context is an (unordered) set of *string context elements*. + +The purpose of string contexts is to collect non-string values attached to strings via +[string concatenation](./operators.md#string-concatenation), +[string interpolation](./string-interpolation.md), +and similar operations. +The idea is that a user can combine together values to create a build instructions for derivations without manually keeping track of where they come from. +Then the Nix language implicitly does that bookkeeping to efficiently obtain the closure of derivation inputs. + +> **Note** +> +> String contexts are *not* explicitly manipulated in idiomatic Nix language code. + +String context elements come in different forms: + +- [deriving path]{#string-context-element-derived-path} + + A string context element of this type is a [deriving path](@docroot@/glossary.md#gloss-deriving-path). + They can be either of type [constant](#string-context-constant) or [output](#string-context-output), which correspond to the types of deriving paths. + + - [Constant string context elements]{#string-context-constant} + + > **Example** + > + > [`builtins.storePath`] creates a string with a single constant string context element: + > + > ```nix + > builtins.getContext (builtins.storePath "/nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10") + > ``` + > evaluates to + > ```nix + > { + > "/nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10" = { + > path = true; + > }; + > } + > ``` + + [deriving path]: @docroot@/glossary.md#gloss-deriving-path + [store path]: @docroot@/glossary.md#gloss-store-path + [`builtins.storePath`]: ./builtins.md#builtins-storePath + + - [Output string context elements]{#string-context-output} + + > **Example** + > + > The behavior of string contexts are best demonstrated with a built-in function that is still experimental: [`builtins.outputOf`]. + > This example will *not* work with stable Nix! + > + > ```nix + > builtins.getContext + > (builtins.outputOf + > (builtins.storePath "/nix/store/fvchh9cvcr7kdla6n860hshchsba305w-hello-2.12.drv") + > "out") + > ``` + > evaluates to + > ```nix + > { + > "/nix/store/fvchh9cvcr7kdla6n860hshchsba305w-hello-2.12.drv" = { + > outputs = [ "out" ]; + > }; + > } + > ``` + + [`builtins.outputOf`]: ./builtins.md#builtins-outputOf + +- [*derivation deep*]{#string-context-element-derivation-deep} + + *derivation deep* is an advanced feature intended to be used with the + [`exportReferencesGraph` derivation attribute](./advanced-attributes.html#adv-attr-exportReferencesGraph). + A *derivation deep* string context element is a derivation path, and refers to both its outputs and the entire build closure of that derivation: + all its outputs, all the other derivations the given derivation depends on, and all the outputs of those. + + > **Example** + > + > The best way to illustrate *derivation deep* string contexts is with [`builtins.addDrvOutputDependencies`]. + > Take a regular constant string context element pointing to a derivation, and transform it into a "Derivation deep" string context element. + > + > ```nix + > builtins.getContext + > (builtins.addDrvOutputDependencies + > (builtins.storePath "/nix/store/fvchh9cvcr7kdla6n860hshchsba305w-hello-2.12.drv")) + > ``` + > evaluates to + > ```nix + > { + > "/nix/store/fvchh9cvcr7kdla6n860hshchsba305w-hello-2.12.drv" = { + > allOutputs = true; + > }; + > } + > ``` + + [`builtins.addDrvOutputDependencies`]: ./builtins.md#builtins-addDrvOutputDependencies + [`builtins.unsafeDiscardOutputDependency`]: ./builtins.md#builtins-unsafeDiscardOutputDependency + +## Inspecting string contexts + +Most basically, [`builtins.hasContext`] will tell whether a string has a non-empty context. + +When more granular information is needed, [`builtins.getContext`] can be used. +It creates an [attribute set] representing the string context, which can be inspected as usual. + +[`builtins.hasContext`]: ./builtins.md#builtins-hasContext +[`builtins.getContext`]: ./builtins.md#builtins-getContext +[attribute set]: ./types.md#attribute-set + +## Clearing string contexts + +[`buitins.unsafeDiscardStringContext`](./builtins.md#builtins-unsafeDiscardStringContext) will make a copy of a string, but with an empty string context. +The returned string can be used in more ways, e.g. by operators that require the string context to be empty. +The requirement to explicitly discard the string context in such use cases helps ensure that string context elements are not lost by mistake. +The "unsafe" marker is only there to remind that Nix normally guarantees that dependencies are tracked, whereas the returned string has lost them. + +## Constructing string contexts + +[`builtins.appendContext`] will create a copy of a string, but with additional string context elements. +The context is specified explicitly by an [attribute set] in the format that [`builtins.hasContext`] produces. +A string with arbitrary contexts can be made like this: + +1. Create a string with the desired string context elements. + (The contents of the string do not matter.) +2. Dump its context with [`builtins.getContext`]. +3. Combine it with a base string and repeated [`builtins.appendContext`] calls. + +[`builtins.appendContext`]: ./builtins.md#builtins-appendContext diff --git a/doc/manual/src/language/string-interpolation.md b/doc/manual/src/language/string-interpolation.md index 7d81c2020..1778bdfa0 100644 --- a/doc/manual/src/language/string-interpolation.md +++ b/doc/manual/src/language/string-interpolation.md @@ -4,9 +4,9 @@ String interpolation is a language feature where a [string], [path], or [attribu Such a construct is called *interpolated string*, and the expression inside is an [interpolated expression](#interpolated-expression). -[string]: ./values.md#type-string -[path]: ./values.md#type-path -[attribute set]: ./values.md#attribute-set +[string]: ./types.md#type-string +[path]: ./types.md#type-path +[attribute set]: ./types.md#attribute-set ## Examples @@ -20,7 +20,7 @@ Rather than writing (where `freetype` is a [derivation]), you can instead write -[derivation]: ../glossary.md#gloss-derivation +[derivation]: @docroot@/glossary.md#gloss-derivation ```nix "--with-freetype2-library=${freetype}/lib" @@ -43,6 +43,47 @@ configureFlags = " Note that Nix expressions and strings can be arbitrarily nested; in this case the outer string contains various interpolated expressions that themselves contain strings (e.g., `"-thread"`), some of which in turn contain interpolated expressions (e.g., `${mesa}`). +To write a literal `${` in an regular string, escape it with a backslash (`\`). + +> **Example** +> +> ```nix +> "echo \${PATH}" +> ``` +> +> "echo ${PATH}" + +To write a literal `${` in an indented string, escape it with two single quotes (`''`). + +> **Example** +> +> ```nix +> '' +> echo ''${PATH} +> '' +> ``` +> +> "echo ${PATH}\n" + +`$${` can be written literally in any string. + +> **Example** +> +> In Make, `$` in file names or recipes is represented as `$$`, see [GNU `make`: Basics of Variable Reference](https://www.gnu.org/software/make/manual/html_node/Reference.html#Basics-of-Variable-References). +> This can be expressed directly in the Nix language strings: +> +> ```nix +> '' +> MAKEVAR = Hello +> all: +> @export BASHVAR=world; echo $(MAKEVAR) $${BASHVAR} +> '' +> ``` +> +> "MAKEVAR = Hello\nall:\n\t@export BASHVAR=world; echo $(MAKEVAR) $\${BASHVAR}\n" + +See the [documentation on strings][string] for details. + ### Path Rather than writing @@ -107,9 +148,9 @@ An expression that is interpolated must evaluate to one of the following: A string interpolates to itself. -A path in an interpolated expression is first copied into the Nix store, and the resulting string is the [store path] of the newly created [store object](../glossary.md#gloss-store-object). +A path in an interpolated expression is first copied into the Nix store, and the resulting string is the [store path] of the newly created [store object](@docroot@/store/store-object.md). -[store path]: ../glossary.md#gloss-store-path +[store path]: @docroot@/store/store-path.md > **Example** > diff --git a/doc/manual/src/language/syntax.md b/doc/manual/src/language/syntax.md new file mode 100644 index 000000000..6108bacd6 --- /dev/null +++ b/doc/manual/src/language/syntax.md @@ -0,0 +1,866 @@ +# Language Constructs + +This section covers syntax and semantics of the Nix language. + +## Basic Literals + +### String {#string-literal} + + *Strings* can be written in three ways. + + The most common way is to enclose the string between double quotes, e.g., `"foo bar"`. + Strings can span multiple lines. + The results of other expressions can be included into a string by enclosing them in `${ }`, a feature known as [string interpolation]. + + [string interpolation]: ./string-interpolation.md + + The following must be escaped to represent them within a string, by prefixing with a backslash (`\`): + + - Double quote (`"`) + + > **Example** + > + > ```nix + > "\"" + > ``` + > + > "\"" + + - Backslash (`\`) + + > **Example** + > + > ```nix + > "\\" + > ``` + > + > "\\" + + - Dollar sign followed by an opening curly bracket (`${`) – "dollar-curly" + + > **Example** + > + > ```nix + > "\${" + > ``` + > + > "\${" + + The newline, carriage return, and tab characters can be written as `\n`, `\r` and `\t`, respectively. + + A "double-dollar-curly" (`$${`) can be written literally. + + > **Example** + > + > ```nix + > "$${" + > ``` + > + > "$\${" + + String values are output on the terminal with Nix-specific escaping. + Strings written to files will contain the characters encoded by the escaping. + + The second way to write string literals is as an *indented string*, which is enclosed between pairs of *double single-quotes* (`''`), like so: + + ```nix + '' + This is the first line. + This is the second line. + This is the third line. + '' + ``` + + This kind of string literal intelligently strips indentation from + the start of each line. To be precise, it strips from each line a + number of spaces equal to the minimal indentation of the string as a + whole (disregarding the indentation of empty lines). For instance, + the first and second line are indented two spaces, while the third + line is indented four spaces. Thus, two spaces are stripped from + each line, so the resulting string is + + ```nix + "This is the first line.\nThis is the second line.\n This is the third line.\n" + ``` + + > **Note** + > + > Whitespace and newline following the opening `''` is ignored if there is no non-whitespace text on the initial line. + + > **Warning** + > + > Prefixed tab characters are not stripped. + > + > > **Example** + > > + > > The following indented string is prefixed with tabs: + > > + > > '' + > > all: + > > @echo hello + > > '' + > > + > > "\tall:\n\t\t@echo hello\n" + + Indented strings support [string interpolation]. + + The following must be escaped to represent them in an indented string: + + - `$` is escaped by prefixing it with two single quotes (`''`) + + > **Example** + > + > ```nix + > '' + > ''$ + > '' + > ``` + > + > "$\n" + + - `''` is escaped by prefixing it with one single quote (`'`) + + > **Example** + > + > ```nix + > '' + > ''' + > '' + > ``` + > + > "''\n" + + These special characters are escaped as follows: + - Linefeed (`\n`): `''\n` + - Carriage return (`\r`): `''\r` + - Tab (`\t`): `''\t` + + `''\` escapes any other character. + + A "double-dollar-curly" (`$${`) can be written literally. + + > **Example** + > + > ```nix + > '' + > $${ + > '' + > ``` + > + > "$\${\n" + + Indented strings are primarily useful in that they allow multi-line + string literals to follow the indentation of the enclosing Nix + expression, and that less escaping is typically necessary for + strings representing languages such as shell scripts and + configuration files because `''` is much less common than `"`. + Example: + + ```nix + stdenv.mkDerivation { + ... + postInstall = + '' + mkdir $out/bin $out/etc + cp foo $out/bin + echo "Hello World" > $out/etc/foo.conf + ${if enableBar then "cp bar $out/bin" else ""} + ''; + ... + } + ``` + + Finally, as a convenience, *URIs* as defined in appendix B of + [RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as + is*, without quotes. For instance, the string + `"http://example.org/foo.tar.bz2"` can also be written as + `http://example.org/foo.tar.bz2`. + +### Number {#number-literal} + + + + Numbers, which can be *integers* (like `123`) or *floating point* + (like `123.43` or `.27e13`). + + See [arithmetic] and [comparison] operators for semantics. + + [arithmetic]: ./operators.md#arithmetic + [comparison]: ./operators.md#comparison + +### Path {#path-literal} + + *Paths* can be expressed by path literals such as `./builder.sh`. + + A path literal must contain at least one slash to be recognised as such. + For instance, `builder.sh` is not a path: + it's parsed as an expression that selects the attribute `sh` from the variable `builder`. + + Path literals are resolved relative to their [base directory](@docroot@/glossary.md#gloss-base-directory). + Path literals may also refer to absolute paths by starting with a slash. + + > **Note** + > + > Absolute paths make expressions less portable. + > In the case where a function translates a path literal into an absolute path string for a configuration file, it is recommended to write a string literal instead. + > This avoids some confusion about whether files at that location will be used during evaluation. + > It also avoids unintentional situations where some function might try to copy everything at the location into the store. + + If the first component of a path is a `~`, it is interpreted such that the rest of the path were relative to the user's home directory. + For example, `~/foo` would be equivalent to `/home/edolstra/foo` for a user whose home directory is `/home/edolstra`. + Path literals that start with `~` are not allowed in [pure](@docroot@/command-ref/conf-file.md#conf-pure-eval) evaluation. + + Path literals can also include [string interpolation], besides being [interpolated into other expressions]. + + [interpolated into other expressions]: ./string-interpolation.md#interpolated-expressions + + At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path. + + `a.${foo}/b.${bar}` is a syntactically valid number division operation. + `./a.${foo}/b.${bar}` is a path. + + [Lookup path](./constructs/lookup-path.md) literals such as `` also resolve to path values. + +## List {#list-literal} + +Lists are formed by enclosing a whitespace-separated list of values +between square brackets. For example, + +```nix +[ 123 ./foo.nix "abc" (f { x = y; }) ] +``` + +defines a list of four elements, the last being the result of a call to +the function `f`. Note that function calls have to be enclosed in +parentheses. If they had been omitted, e.g., + +```nix +[ 123 ./foo.nix "abc" f { x = y; } ] +``` + +the result would be a list of five elements, the fourth one being a +function and the fifth being a set. + +Note that lists are only lazy in values, and they are strict in length. + +Elements in a list can be accessed using [`builtins.elemAt`](./builtins.md#builtins-elemAt). + +## Attribute Set {#attrs-literal} + +An attribute set is a collection of name-value-pairs called *attributes*. + +Attribute sets are written enclosed in curly brackets (`{ }`). +Attribute names and attribute values are separated by an equal sign (`=`). +Each value can be an arbitrary expression, terminated by a semicolon (`;`) + +An attribute name is a string without context, and is denoted by a [name] (an [identifier](./identifiers.md#identifiers) or [string literal](#string-literal)). + +[name]: ./identifiers.md#names + +> **Syntax** +> +> *attrset* → `{` { *name* `=` *expr* `;` } `}` + +Attributes can appear in any order. +An attribute name may only occur once in each attribute set. + +> **Example** +> +> This defines an attribute set with attributes named: +> - `x` with the value `123`, an integer +> - `text` with the value `"Hello"`, a string +> - `y` where the value is the result of applying the function `f` to the attribute set `{ bla = 456; }` +> +> ```nix +> { +> x = 123; +> text = "Hello"; +> y = f { bla = 456; }; +> } +> ``` + +Attributes in nested attribute sets can be written using *attribute paths*. + +> **Syntax** +> +> *attrset* → `{` { *attrpath* `=` *expr* `;` } `}` + +An attribute path is a dot-separated list of [names][name]. + +> **Syntax** +> +> *attrpath* = *name* { `.` *name* } + + + +> **Example** +> +> ```nix +> { a.b.c = 1; a.b.d = 2; } +> ``` +> +> { +> a = { +> b = { +> c = 1; +> d = 2; +> }; +> }; +> } + +Attribute names can also be set implicitly by using the [`inherit` keyword](#inheriting-attributes). + +> **Example** +> +> ```nix +> { inherit (builtins) true; } +> ``` +> +> { true = true; } + +Attributes can be accessed with the [`.` operator](./operators.md#attribute-selection). + +Example: + +```nix +{ a = "Foo"; b = "Bar"; }.a +``` + +This evaluates to `"Foo"`. + +It is possible to provide a default value in an attribute selection using the `or` keyword. + +Example: + +```nix +{ a = "Foo"; b = "Bar"; }.c or "Xyzzy" +``` + +```nix +{ a = "Foo"; b = "Bar"; }.c.d.e.f.g or "Xyzzy" +``` + +will both evaluate to `"Xyzzy"` because there is no `c` attribute in the set. + +You can use arbitrary double-quoted strings as attribute names: + +```nix +{ "$!@#?" = 123; }."$!@#?" +``` + +```nix +let bar = "bar"; in +{ "foo ${bar}" = 123; }."foo ${bar}" +``` + +Both will evaluate to `123`. + +Attribute names support [string interpolation]: + +```nix +let bar = "foo"; in +{ foo = 123; }.${bar} +``` + +```nix +let bar = "foo"; in +{ ${bar} = 123; }.foo +``` + +Both will evaluate to `123`. + +In the special case where an attribute name inside of a set declaration +evaluates to `null` (which is normally an error, as `null` cannot be coerced to +a string), that attribute is simply not added to the set: + +```nix +{ ${if foo then "bar" else null} = true; } +``` + +This will evaluate to `{}` if `foo` evaluates to `false`. + +A set that has a `__functor` attribute whose value is callable (i.e. is +itself a function or a set with a `__functor` attribute whose value is +callable) can be applied as if it were a function, with the set itself +passed in first , e.g., + +```nix +let add = { __functor = self: x: x + self.x; }; + inc = add // { x = 1; }; +in inc 1 +``` + +evaluates to `2`. This can be used to attach metadata to a function +without the caller needing to treat it specially, or to implement a form +of object-oriented programming, for example. + +## Recursive sets + +Recursive sets are like normal [attribute sets](./types.md#attribute-set), but the attributes can refer to each other. + +> *rec-attrset* = `rec {` [ *name* `=` *expr* `;` `]`... `}` + +Example: + +```nix +rec { + x = y; + y = 123; +}.x +``` + +This evaluates to `123`. + +Note that without `rec` the binding `x = y;` would +refer to the variable `y` in the surrounding scope, if one exists, and +would be invalid if no such variable exists. That is, in a normal +(non-recursive) set, attributes are not added to the lexical scope; in a +recursive set, they are. + +Recursive sets of course introduce the danger of infinite recursion. For +example, the expression + +```nix +rec { + x = y; + y = x; +}.x +``` + +will crash with an `infinite recursion encountered` error message. + +## Let-expressions + +A let-expression allows you to define local variables for an expression. + +> *let-in* = `let` [ *identifier* = *expr* ]... `in` *expr* + +Example: + +```nix +let + x = "foo"; + y = "bar"; +in x + y +``` + +This evaluates to `"foobar"`. + +## Inheriting attributes + +When defining an [attribute set](./types.md#attribute-set) or in a [let-expression](#let-expressions) it is often convenient to copy variables from the surrounding lexical scope (e.g., when you want to propagate attributes). +This can be shortened using the `inherit` keyword. + +Example: + +```nix +let x = 123; in +{ + inherit x; + y = 456; +} +``` + +is equivalent to + +```nix +let x = 123; in +{ + x = x; + y = 456; +} +``` + +and both evaluate to `{ x = 123; y = 456; }`. + +> **Note** +> +> This works because `x` is added to the lexical scope by the `let` construct. + +It is also possible to inherit attributes from another attribute set. + +Example: + +In this fragment from `all-packages.nix`, + +```nix +graphviz = (import ../tools/graphics/graphviz) { + inherit fetchurl stdenv libpng libjpeg expat x11 yacc; + inherit (xorg) libXaw; +}; + +xorg = { + libX11 = ...; + libXaw = ...; + ... +} + +libpng = ...; +libjpg = ...; +... +``` + +the set used in the function call to the function defined in +`../tools/graphics/graphviz` inherits a number of variables from the +surrounding scope (`fetchurl` ... `yacc`), but also inherits `libXaw` +(the X Athena Widgets) from the `xorg` set. + +Summarizing the fragment + +```nix +... +inherit x y z; +inherit (src-set) a b c; +... +``` + +is equivalent to + +```nix +... +x = x; y = y; z = z; +a = src-set.a; b = src-set.b; c = src-set.c; +... +``` + +when used while defining local variables in a let-expression or while +defining a set. + +In a `let` expression, `inherit` can be used to selectively bring specific attributes of a set into scope. For example + + +```nix +let + x = { a = 1; b = 2; }; + inherit (builtins) attrNames; +in +{ + names = attrNames x; +} +``` + +is equivalent to + +```nix +let + x = { a = 1; b = 2; }; +in +{ + names = builtins.attrNames x; +} +``` + +both evaluate to `{ names = [ "a" "b" ]; }`. + +## Functions + +Functions have the following form: + +```nix +pattern: body +``` + +The pattern specifies what the argument of the function must look like, +and binds variables in the body to (parts of) the argument. There are +three kinds of patterns: + + - If a pattern is a single identifier, then the function matches any + argument. Example: + + ```nix + let negate = x: !x; + concat = x: y: x + y; + in if negate true then concat "foo" "bar" else "" + ``` + + Note that `concat` is a function that takes one argument and returns + a function that takes another argument. This allows partial + parameterisation (i.e., only filling some of the arguments of a + function); e.g., + + ```nix + map (concat "foo") [ "bar" "bla" "abc" ] + ``` + + evaluates to `[ "foobar" "foobla" "fooabc" ]`. + + - A *set pattern* of the form `{ name1, name2, …, nameN }` matches a + set containing the listed attributes, and binds the values of those + attributes to variables in the function body. For example, the + function + + ```nix + { x, y, z }: z + y + x + ``` + + can only be called with a set containing exactly the attributes `x`, + `y` and `z`. No other attributes are allowed. If you want to allow + additional arguments, you can use an ellipsis (`...`): + + ```nix + { x, y, z, ... }: z + y + x + ``` + + This works on any set that contains at least the three named + attributes. + + It is possible to provide *default values* for attributes, in + which case they are allowed to be missing. A default value is + specified by writing `name ? e`, where *e* is an arbitrary + expression. For example, + + ```nix + { x, y ? "foo", z ? "bar" }: z + y + x + ``` + + specifies a function that only requires an attribute named `x`, but + optionally accepts `y` and `z`. + + - An `@`-pattern provides a means of referring to the whole value + being matched: + + ```nix + args@{ x, y, z, ... }: z + y + x + args.a + ``` + + but can also be written as: + + ```nix + { x, y, z, ... } @ args: z + y + x + args.a + ``` + + Here `args` is bound to the argument *as passed*, which is further + matched against the pattern `{ x, y, z, ... }`. + The `@`-pattern makes mainly sense with an ellipsis(`...`) as + you can access attribute names as `a`, using `args.a`, which was + given as an additional attribute to the function. + + > **Warning** + > + > `args@` binds the name `args` to the attribute set that is passed to the function. + > In particular, `args` does *not* include any default values specified with `?` in the function's set pattern. + > + > For instance + > + > ```nix + > let + > f = args@{ a ? 23, ... }: [ a args ]; + > in + > f {} + > ``` + > + > is equivalent to + > + > ```nix + > let + > f = args @ { ... }: [ (args.a or 23) args ]; + > in + > f {} + > ``` + > + > and both expressions will evaluate to: + > + > ```nix + > [ 23 {} ] + > ``` + +Note that functions do not have names. If you want to give them a name, +you can bind them to an attribute, e.g., + +```nix +let concat = { x, y }: x + y; +in concat { x = "foo"; y = "bar"; } +``` + +## Conditionals + +Conditionals look like this: + +```nix +if e1 then e2 else e3 +``` + +where *e1* is an expression that should evaluate to a Boolean value +(`true` or `false`). + +## Assertions + +Assertions are generally used to check that certain requirements on or +between features and dependencies hold. They look like this: + +```nix +assert e1; e2 +``` + +where *e1* is an expression that should evaluate to a Boolean value. If +it evaluates to `true`, *e2* is returned; otherwise expression +evaluation is aborted and a backtrace is printed. + +Here is a Nix expression for the Subversion package that shows how +assertions can be used:. + +```nix +{ localServer ? false +, httpServer ? false +, sslSupport ? false +, pythonBindings ? false +, javaSwigBindings ? false +, javahlBindings ? false +, stdenv, fetchurl +, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null +}: + +assert localServer -> db4 != null; â‘ +assert httpServer -> httpd != null && httpd.expat == expat; â‘¡ +assert sslSupport -> openssl != null && (httpServer -> httpd.openssl == openssl); â‘¢ +assert pythonBindings -> swig != null && swig.pythonSupport; +assert javaSwigBindings -> swig != null && swig.javaSupport; +assert javahlBindings -> j2sdk != null; + +stdenv.mkDerivation { + name = "subversion-1.1.1"; + ... + openssl = if sslSupport then openssl else null; â‘£ + ... +} +``` + +The points of interest are: + +1. This assertion states that if Subversion is to have support for + local repositories, then Berkeley DB is needed. So if the Subversion + function is called with the `localServer` argument set to `true` but + the `db4` argument set to `null`, then the evaluation fails. + + Note that `->` is the [logical + implication](https://en.wikipedia.org/wiki/Truth_table#Logical_implication) + Boolean operation. + +2. This is a more subtle condition: if Subversion is built with Apache + (`httpServer`) support, then the Expat library (an XML library) used + by Subversion should be same as the one used by Apache. This is + because in this configuration Subversion code ends up being linked + with Apache code, and if the Expat libraries do not match, a build- + or runtime link error or incompatibility might occur. + +3. This assertion says that in order for Subversion to have SSL support + (so that it can access `https` URLs), an OpenSSL library must be + passed. Additionally, it says that *if* Apache support is enabled, + then Apache's OpenSSL should match Subversion's. (Note that if + Apache support is not enabled, we don't care about Apache's + OpenSSL.) + +4. The conditional here is not really related to assertions, but is + worth pointing out: it ensures that if SSL support is disabled, then + the Subversion derivation is not dependent on OpenSSL, even if a + non-`null` value was passed. This prevents an unnecessary rebuild of + Subversion if OpenSSL changes. + +## With-expressions + +A *with-expression*, + +```nix +with e1; e2 +``` + +introduces the set *e1* into the lexical scope of the expression *e2*. +For instance, + +```nix +let as = { x = "foo"; y = "bar"; }; +in with as; x + y +``` + +evaluates to `"foobar"` since the `with` adds the `x` and `y` attributes +of `as` to the lexical scope in the expression `x + y`. The most common +use of `with` is in conjunction with the `import` function. E.g., + +```nix +with (import ./definitions.nix); ... +``` + +makes all attributes defined in the file `definitions.nix` available as +if they were defined locally in a `let`-expression. + +The bindings introduced by `with` do not shadow bindings introduced by +other means, e.g. + +```nix +let a = 3; in with { a = 1; }; let a = 4; in with { a = 2; }; ... +``` + +establishes the same scope as + +```nix +let a = 1; in let a = 2; in let a = 3; in let a = 4; in ... +``` + +Variables coming from outer `with` expressions *are* shadowed: + +```nix +with { a = "outer"; }; +with { a = "inner"; }; +a +``` + +Does evaluate to `"inner"`. + +## Comments + +- Inline comments start with `#` and run until the end of the line. + + > **Example** + > + > ```nix + > # A number + > 2 # Equals 1 + 1 + > ``` + > + > ```console + > 2 + > ``` + +- Block comments start with `/*` and run until the next occurrence of `*/`. + + > **Example** + > + > ```nix + > /* + > Block comments + > can span multiple lines. + > */ "hello" + > ``` + > + > ```console + > "hello" + > ``` + + This means that block comments cannot be nested. + + > **Example** + > + > ```nix + > /* /* nope */ */ 1 + > ``` + > + > ```console + > error: syntax error, unexpected '*' + > + > at «string»:1:15: + > + > 1| /* /* nope */ * + > | ^ + > ``` + + Consider escaping nested comments and unescaping them in post-processing. + + > **Example** + > + > ```nix + > /* /* nested *\/ */ 1 + > ``` + > + > ```console + > 1 + > ``` diff --git a/doc/manual/src/language/types.md b/doc/manual/src/language/types.md new file mode 100644 index 000000000..229756e6b --- /dev/null +++ b/doc/manual/src/language/types.md @@ -0,0 +1,120 @@ +# Data Types + +Every value in the Nix language has one of the following types: + +* [Integer](#type-int) +* [Float](#type-float) +* [Boolean](#type-bool) +* [String](#type-string) +* [Path](#type-path) +* [Null](#type-null) +* [Attribute set](#type-attrs) +* [List](#type-list) +* [Function](#type-function) +* [External](#type-external) + +## Primitives + +### Integer {#type-int} + +An _integer_ in the Nix language is a signed 64-bit integer. + +Non-negative integers can be expressed as [integer literals](syntax.md#number-literal). +Negative integers are created with the [arithmetic negation operator](./operators.md#arithmetic). +The function [`builtins.isInt`](builtins.md#builtins-isInt) can be used to determine if a value is an integer. + +### Float {#type-float} + +A _float_ in the Nix language is a 64-bit [IEEE 754](https://en.wikipedia.org/wiki/IEEE_754) floating-point number. + +Most non-negative floats can be expressed as [float literals](syntax.md#number-literal). +Negative floats are created with the [arithmetic negation operator](./operators.md#arithmetic). +The function [`builtins.isFloat`](builtins.md#builtins-isFloat) can be used to determine if a value is a float. + +### Boolean {#type-bool} + +A _boolean_ in the Nix language is one of _true_ or _false_. + + + +These values are available as attributes of [`builtins`](builtins.md#builtins-builtins) as [`builtins.true`](builtins.md#builtins-true) and [`builtins.false`](builtins.md#builtins-false). +The function [`builtins.isBool`](builtins.md#builtins-isBool) can be used to determine if a value is a boolean. + +### String {#type-string} + +A _string_ in the Nix language is an immutable, finite-length sequence of bytes, along with a [string context](string-context.md). +Nix does not assume or support working natively with character encodings. + +String values without string context can be expressed as [string literals](syntax.md#string-literal). +The function [`builtins.isString`](builtins.md#builtins-isString) can be used to determine if a value is a string. + +### Path {#type-path} + +A _path_ in the Nix language is an immutable, finite-length sequence of bytes starting with `/`, representing a POSIX-style, canonical file system path. +Path values are distinct from string values, even if they contain the same sequence of bytes. +Operations that produce paths will simplify the result as the standard C function [`realpath`] would, except that there is no symbolic link resolution. + +[`realpath`]: https://pubs.opengroup.org/onlinepubs/9699919799/functions/realpath.html + +Paths are suitable for referring to local files, and are often preferable over strings. +- Path values do not contain trailing or duplicate slashes, `.`, or `..`. +- Relative path literals are automatically resolved relative to their [base directory]. +- Tooling can recognize path literals and provide additional features, such as autocompletion, refactoring automation and jump-to-file. + +[base directory]: @docroot@/glossary.md#gloss-base-directory + +A file is not required to exist at a given path in order for that path value to be valid, but a path that is converted to a string with [string interpolation] or [string-and-path concatenation] must resolve to a readable file or directory which will be copied into the Nix store. +For instance, evaluating `"${./foo.txt}"` will cause `foo.txt` from the same directory to be copied into the Nix store and result in the string `"/nix/store/-foo.txt"`. +Operations such as [`import`] can also expect a path to resolve to a readable file or directory. + +[string interpolation]: string-interpolation.md#interpolated-expression +[string-and-path concatenation]: operators.md#string-and-path-concatenation +[`import`]: builtins.md#builtins-import + +> **Note** +> +> The Nix language assumes that all input files will remain _unchanged_ while evaluating a Nix expression. +> For example, assume you used a file path in an interpolated string during a `nix repl` session. +> Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new [store path], since Nix might not re-read the file contents. +> Use `:r` to reset the repl as needed. + +[store path]: @docroot@/store/store-path.md + +Path values can be expressed as [path literals](syntax.md#path-literal). +The function [`builtins.isPath`](builtins.md#builtins-isPath) can be used to determine if a value is a path. + +### Null {#type-null} + +There is a single value of type _null_ in the Nix language. + + + +This value is available as an attribute on the [`builtins`](builtins.md#builtins-builtins) attribute set as [`builtins.null`](builtins.md#builtins-null). + +## Compound values + +### Attribute set {#type-attrs} + + + +An attribute set can be constructed with an [attribute set literal](syntax.md#attrs-literal). +The function [`builtins.isAttrs`](builtins.md#builtins-isAttrs) can be used to determine if a value is an attribute set. + +### List {#type-list} + + + +A list can be constructed with a [list literal](syntax.md#list-literal). +The function [`builtins.isList`](builtins.md#builtins-isList) can be used to determine if a value is a list. + +## Function {#type-function} + + + +A function can be constructed with a [function expression](syntax.md#functions). +The function [`builtins.isFunction`](builtins.md#builtins-isFunction) can be used to determine if a value is a function. + +## External {#type-external} + +An _external_ value is an opaque value created by a Nix [plugin](../command-ref/conf-file.md#conf-plugin-files). +Such a value can be substituted in Nix expressions but only created and used by plugin code. diff --git a/doc/manual/src/language/values.md b/doc/manual/src/language/values.md deleted file mode 100644 index 74ffc7070..000000000 --- a/doc/manual/src/language/values.md +++ /dev/null @@ -1,269 +0,0 @@ -# Data Types - -## Primitives - -- String - - *Strings* can be written in three ways. - - The most common way is to enclose the string between double quotes, - e.g., `"foo bar"`. Strings can span multiple lines. The special - characters `"` and `\` and the character sequence `${` must be - escaped by prefixing them with a backslash (`\`). Newlines, carriage - returns and tabs can be written as `\n`, `\r` and `\t`, - respectively. - - You can include the results of other expressions into a string by enclosing them in `${ }`, a feature known as [string interpolation]. - - [string interpolation]: ./string-interpolation.md - - The second way to write string literals is as an *indented string*, - which is enclosed between pairs of *double single-quotes*, like so: - - ```nix - '' - This is the first line. - This is the second line. - This is the third line. - '' - ``` - - This kind of string literal intelligently strips indentation from - the start of each line. To be precise, it strips from each line a - number of spaces equal to the minimal indentation of the string as a - whole (disregarding the indentation of empty lines). For instance, - the first and second line are indented two spaces, while the third - line is indented four spaces. Thus, two spaces are stripped from - each line, so the resulting string is - - ```nix - "This is the first line.\nThis is the second line.\n This is the third line.\n" - ``` - - Note that the whitespace and newline following the opening `''` is - ignored if there is no non-whitespace text on the initial line. - - Indented strings support [string interpolation]. - - Since `${` and `''` have special meaning in indented strings, you - need a way to quote them. `$` can be escaped by prefixing it with - `''` (that is, two single quotes), i.e., `''$`. `''` can be escaped - by prefixing it with `'`, i.e., `'''`. `$` removes any special - meaning from the following `$`. Linefeed, carriage-return and tab - characters can be written as `''\n`, `''\r`, `''\t`, and `''\` - escapes any other character. - - Indented strings are primarily useful in that they allow multi-line - string literals to follow the indentation of the enclosing Nix - expression, and that less escaping is typically necessary for - strings representing languages such as shell scripts and - configuration files because `''` is much less common than `"`. - Example: - - ```nix - stdenv.mkDerivation { - ... - postInstall = - '' - mkdir $out/bin $out/etc - cp foo $out/bin - echo "Hello World" > $out/etc/foo.conf - ${if enableBar then "cp bar $out/bin" else ""} - ''; - ... - } - ``` - - Finally, as a convenience, *URIs* as defined in appendix B of - [RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as - is*, without quotes. For instance, the string - `"http://example.org/foo.tar.bz2"` can also be written as - `http://example.org/foo.tar.bz2`. - -- Number - - Numbers, which can be *integers* (like `123`) or *floating point* - (like `123.43` or `.27e13`). - - See [arithmetic] and [comparison] operators for semantics. - - [arithmetic]: ./operators.md#arithmetic - [comparison]: ./operators.md#comparison - -- Path - - *Paths*, e.g., `/bin/sh` or `./builder.sh`. A path must contain at - least one slash to be recognised as such. For instance, `builder.sh` - is not a path: it's parsed as an expression that selects the - attribute `sh` from the variable `builder`. If the file name is - relative, i.e., if it does not begin with a slash, it is made - absolute at parse time relative to the directory of the Nix - expression that contained it. For instance, if a Nix expression in - `/foo/bar/bla.nix` refers to `../xyzzy/fnord.nix`, the absolute path - is `/foo/xyzzy/fnord.nix`. - - If the first component of a path is a `~`, it is interpreted as if - the rest of the path were relative to the user's home directory. - e.g. `~/foo` would be equivalent to `/home/edolstra/foo` for a user - whose home directory is `/home/edolstra`. - - For instance, evaluating `"${./foo.txt}"` will cause `foo.txt` in the current directory to be copied into the Nix store and result in the string `"/nix/store/-foo.txt"`. - - Note that the Nix language assumes that all input files will remain _unchanged_ while evaluating a Nix expression. - For example, assume you used a file path in an interpolated string during a `nix repl` session. - Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new [store path], since Nix might not re-read the file contents. - - [store path]: ../glossary.md#gloss-store-path - - Paths can include [string interpolation] and can themselves be [interpolated in other expressions]. - - [interpolated in other expressions]: ./string-interpolation.md#interpolated-expressions - - At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path. - - `a.${foo}/b.${bar}` is a syntactically valid division operation. - `./a.${foo}/b.${bar}` is a path. - - [Lookup paths](./constructs/lookup-path.md) such as `` resolve to path values. - -- Boolean - - *Booleans* with values `true` and `false`. - -- Null - - The null value, denoted as `null`. - -## List - -Lists are formed by enclosing a whitespace-separated list of values -between square brackets. For example, - -```nix -[ 123 ./foo.nix "abc" (f { x = y; }) ] -``` - -defines a list of four elements, the last being the result of a call to -the function `f`. Note that function calls have to be enclosed in -parentheses. If they had been omitted, e.g., - -```nix -[ 123 ./foo.nix "abc" f { x = y; } ] -``` - -the result would be a list of five elements, the fourth one being a -function and the fifth being a set. - -Note that lists are only lazy in values, and they are strict in length. - -Elements in a list can be accessed using [`builtins.elemAt`](./builtins.md#builtins-elemAt). - -## Attribute Set - -An attribute set is a collection of name-value-pairs (called *attributes*) enclosed in curly brackets (`{ }`). - -An attribute name can be an identifier or a [string](#string). -An identifier must start with a letter (`a-z`, `A-Z`) or underscore (`_`), and can otherwise contain letters (`a-z`, `A-Z`), numbers (`0-9`), underscores (`_`), apostrophes (`'`), or dashes (`-`). - -> **Syntax** -> -> *name* = *identifier* | *string* \ -> *identifier* ~ `[a-zA-Z_][a-zA-Z0-9_'-]*` - -Names and values are separated by an equal sign (`=`). -Each value is an arbitrary expression terminated by a semicolon (`;`). - -> **Syntax** -> -> *attrset* = `{` [ *name* `=` *expr* `;` ]... `}` - -Attributes can appear in any order. -An attribute name may only occur once. - -Example: - -```nix -{ - x = 123; - text = "Hello"; - y = f { bla = 456; }; -} -``` - -This defines a set with attributes named `x`, `text`, `y`. - -Attributes can be accessed with the [`.` operator](./operators.md#attribute-selection). - -Example: - -```nix -{ a = "Foo"; b = "Bar"; }.a -``` - -This evaluates to `"Foo"`. - -It is possible to provide a default value in an attribute selection using the `or` keyword. - -Example: - -```nix -{ a = "Foo"; b = "Bar"; }.c or "Xyzzy" -``` - -```nix -{ a = "Foo"; b = "Bar"; }.c.d.e.f.g or "Xyzzy" -``` - -will both evaluate to `"Xyzzy"` because there is no `c` attribute in the set. - -You can use arbitrary double-quoted strings as attribute names: - -```nix -{ "$!@#?" = 123; }."$!@#?" -``` - -```nix -let bar = "bar"; in -{ "foo ${bar}" = 123; }."foo ${bar}" -``` - -Both will evaluate to `123`. - -Attribute names support [string interpolation]: - -```nix -let bar = "foo"; in -{ foo = 123; }.${bar} -``` - -```nix -let bar = "foo"; in -{ ${bar} = 123; }.foo -``` - -Both will evaluate to `123`. - -In the special case where an attribute name inside of a set declaration -evaluates to `null` (which is normally an error, as `null` cannot be coerced to -a string), that attribute is simply not added to the set: - -```nix -{ ${if foo then "bar" else null} = true; } -``` - -This will evaluate to `{}` if `foo` evaluates to `false`. - -A set that has a `__functor` attribute whose value is callable (i.e. is -itself a function or a set with a `__functor` attribute whose value is -callable) can be applied as if it were a function, with the set itself -passed in first , e.g., - -```nix -let add = { __functor = self: x: x + self.x; }; - inc = add // { x = 1; }; -in inc 1 -``` - -evaluates to `2`. This can be used to attach metadata to a function -without the caller needing to treat it specially, or to implement a form -of object-oriented programming, for example. diff --git a/doc/manual/src/language/variables.md b/doc/manual/src/language/variables.md new file mode 100644 index 000000000..af6aff8a2 --- /dev/null +++ b/doc/manual/src/language/variables.md @@ -0,0 +1,10 @@ +# Variables + +A *variable* is an [identifier](identifiers.md) used as an expression. + +> **Syntax** +> +> *expression* → *identifier* + +A variable must have the same name as a definition in the [scope](./scope.md) that encloses it. +The value of a variable is the value of the corresponding expression in the enclosing scope. diff --git a/doc/manual/src/package-management/copy-closure.md b/doc/manual/src/package-management/copy-closure.md deleted file mode 100644 index 14326298b..000000000 --- a/doc/manual/src/package-management/copy-closure.md +++ /dev/null @@ -1,34 +0,0 @@ -# Copying Closures via SSH - -The command `nix-copy-closure` copies a Nix store path along with all -its dependencies to or from another machine via the SSH protocol. It -doesn’t copy store paths that are already present on the target machine. -For example, the following command copies Firefox with all its -dependencies: - - $ nix-copy-closure --to alice@itchy.example.org $(type -p firefox) - -See the [manpage for `nix-copy-closure`](../command-ref/nix-copy-closure.md) for details. - -With `nix-store ---export` and `nix-store --import` you can write the closure of a store -path (that is, the path and all its dependencies) to a file, and then -unpack that file into another Nix store. For example, - - $ nix-store --export $(nix-store --query --requisites $(type -p firefox)) > firefox.closure - -writes the closure of Firefox to a file. You can then copy this file to -another machine and install the closure: - - $ nix-store --import < firefox.closure - -Any store paths in the closure that are already present in the target -store are ignored. It is also possible to pipe the export into another -command, e.g. to copy and install a closure directly to/on another -machine: - - $ nix-store --export $(nix-store --query --requisites $(type -p firefox)) | bzip2 | \ - ssh alice@itchy.example.org "bunzip2 | nix-store --import" - -However, `nix-copy-closure` is generally more efficient because it only -copies paths that are not already present in the target Nix store. diff --git a/doc/manual/src/protocols/derivation-aterm.md b/doc/manual/src/protocols/derivation-aterm.md index e58b602a3..1ba757ae0 100644 --- a/doc/manual/src/protocols/derivation-aterm.md +++ b/doc/manual/src/protocols/derivation-aterm.md @@ -14,6 +14,6 @@ Derivations are serialised in one of the following formats: DrvWithVersion(, ...) ``` - The only `version-string`s that are in use today are for [experimental features](@docroot@/contributing/experimental-features.md): + The only `version-string`s that are in use today are for [experimental features](@docroot@/development/experimental-features.md): - - `"xp-dyn-drv"` for the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. + - `"xp-dyn-drv"` for the [`dynamic-derivations`](@docroot@/development/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. diff --git a/doc/manual/src/protocols/json/derivation.md b/doc/manual/src/protocols/json/derivation.md index 649d543cc..2f85340d6 100644 --- a/doc/manual/src/protocols/json/derivation.md +++ b/doc/manual/src/protocols/json/derivation.md @@ -3,7 +3,7 @@ > **Warning** > > This JSON format is currently -> [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) +> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) > and subject to change. The JSON serialization of a @@ -18,10 +18,30 @@ is a JSON object with the following fields: Information about the output paths of the derivation. This is a JSON object with one member per output, where the key is the output name and the value is a JSON object with these fields: - * `path`: The output path. + * `path`: + The output path, if it is known in advanced. + Otherwise, `null`. + + + * `method`: + For an output which will be [content addresed], a string representing the [method](@docroot@/store/store-object/content-address.md) of content addressing that is chosen. + Valid method strings are: + + - [`flat`](@docroot@/store/store-object/content-address.md#method-flat) + - [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive) + - [`text`](@docroot@/store/store-object/content-address.md#method-text) + - [`git`](@docroot@/store/store-object/content-address.md#method-git) + + Otherwise, `null`. * `hashAlgo`: - For fixed-output derivations, the hashing algorithm (e.g. `sha256`), optionally prefixed by `r:` if `hash` denotes a NAR hash rather than a flat file hash. + For an output which will be [content addresed], the name of the hash algorithm used. + Valid algorithm strings are: + + - `md5` + - `sha1` + - `sha256` + - `sha512` * `hash`: For fixed-output derivations, the expected content hash in base-16. @@ -32,7 +52,8 @@ is a JSON object with the following fields: > "outputs": { > "out": { > "path": "/nix/store/2543j7c6jn75blc3drf4g5vhb1rhdq29-source", - > "hashAlgo": "r:sha256", + > "method": "nar", + > "hashAlgo": "sha256", > "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62" > } > } diff --git a/doc/manual/src/protocols/json/store-object-info.md b/doc/manual/src/protocols/json/store-object-info.md index ba4ab098f..6b4f48437 100644 --- a/doc/manual/src/protocols/json/store-object-info.md +++ b/doc/manual/src/protocols/json/store-object-info.md @@ -3,7 +3,7 @@ > **Warning** > > This JSON format is currently -> [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) +> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) > and subject to change. Info about a [store object]. @@ -24,41 +24,45 @@ Info about a [store object]. An array of [store paths][store path], possibly including this one. -* `ca` (optional): +* `ca`: - Content address of this store object's file system object, used to compute its store path. + If the store object is [content-addressed], + this is the content address of this store object's file system object, used to compute its store path. + Otherwise (i.e. if it is [input-addressed]), this is `null`. -[store path]: @docroot@/glossary.md#gloss-store-path +[store path]: @docroot@/store/store-path.md [file system object]: @docroot@/store/file-system-object.md -[Nix Archive]: @docroot@/glossary.md#gloss-nar +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive ## Impure fields These are not intrinsic properties of the store object. In other words, the same store object residing in different store could have different values for these properties. -* `deriver` (optional): +* `deriver`: - The path to the [derivation] from which this store object is produced. + If known, the path to the [derivation] from which this store object was produced. + Otherwise `null`. [derivation]: @docroot@/glossary.md#gloss-store-derivation * `registrationTime` (optional): - When this derivation was added to the store. + If known, when this derivation was added to the store. + Otherwise `null`. -* `ultimate` (optional): +* `ultimate`: Whether this store object is trusted because we built it ourselves, rather than substituted a build product from elsewhere. -* `signatures` (optional): +* `signatures`: Signatures claiming that this store object is what it claims to be. Not relevant for [content-addressed] store objects, but useful for [input-addressed] store objects. - [content-addressed]: @docroot@/glossary.md#gloss-content-addressed-store-object - [input-addressed]: @docroot@/glossary.md#gloss-input-addressed-store-object +[content-addressed]: @docroot@/store/store-object/content-address.md +[input-addressed]: @docroot@/glossary.md#gloss-input-addressed-store-object ### `.narinfo` extra fields @@ -83,7 +87,7 @@ This information is not intrinsic to the store object, but about how it is store ## Computed closure fields -These fields are not stored at all, but computed by traverising the other other fields across all the store objects in a [closure]. +These fields are not stored at all, but computed by traversing the other fields across all the store objects in a [closure]. * `closureSize`: diff --git a/doc/manual/src/protocols/nix-archive.md b/doc/manual/src/protocols/nix-archive.md new file mode 100644 index 000000000..640b527f1 --- /dev/null +++ b/doc/manual/src/protocols/nix-archive.md @@ -0,0 +1,43 @@ +# Nix Archive (NAR) format + +This is the complete specification of the [Nix Archive] format. +The Nix Archive format closely follows the abstract specification of a [file system object] tree, +because it is designed to serialize exactly that data structure. + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#nix-archive +[file system object]: @docroot@/store/file-system-object.md + +The format of this specification is close to [Extended Backus–Naur form](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form), with the exception of the `str(..)` function / parameterized rule, which length-prefixes and pads strings. +This makes the resulting binary format easier to parse. + +Regular users do *not* need to know this information. +But for those interested in exactly how Nix works, e.g. if they are reimplementing it, this information can be useful. + +```ebnf +nar = str("nix-archive-1"), nar-obj; + +nar-obj = str("("), nar-obj-inner, str(")"); + +nar-obj-inner + = str("type"), str("regular") regular + | str("type"), str("symlink") symlink + | str("type"), str("directory") directory + ; + +regular = [ str("executable"), str("") ], str("contents"), str(contents); + +symlink = str("target"), str(target); + +(* side condition: directory entries must be ordered by their names *) +directory = { directory-entry }; + +directory-entry = str("entry"), str("("), str("name"), str(name), str("node"), nar-obj, str(")"); +``` + +The `str` function / parameterized rule is defined as follows: + +- `str(s)` = `int(|s|), pad(s);` + +- `int(n)` = the 64-bit little endian representation of the number `n` + +- `pad(s)` = the byte sequence `s`, padded with 0s to a multiple of 8 byte diff --git a/doc/manual/src/protocols/store-path.md b/doc/manual/src/protocols/store-path.md index 565c4fa75..52352d358 100644 --- a/doc/manual/src/protocols/store-path.md +++ b/doc/manual/src/protocols/store-path.md @@ -1,12 +1,14 @@ # Complete Store Path Calculation -This is the complete specification for how store paths are calculated. +This is the complete specification for how [store path]s are calculated. The format of this specification is close to [Extended Backus–Naur form](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form), but must deviate for a few things such as hash functions which we treat as bidirectional for specification purposes. Regular users do *not* need to know this information --- store paths can be treated as black boxes computed from the properties of the store objects they refer to. But for those interested in exactly how Nix works, e.g. if they are reimplementing it, this information can be useful. +[store path](@docroot@/store/store-path.md) + ## Store path proper ```ebnf @@ -34,18 +36,23 @@ where - `type` = one of: - ```ebnf - | "text" ( ":" store-path )* + | "text" { ":" store-path } ``` - for encoded derivations written to the store. + This is for the + ["Text"](@docroot@/store/store-object/content-address.md#method-text) + method of content addressing store objects. The optional trailing store paths are the references of the store object. - ```ebnf - | "source" ( ":" store-path )* + | "source" { ":" store-path } [ ":self" ] ``` - For paths copied to the store and hashed via a [Nix Archive (NAR)] and [SHA-256][sha-256]. - Just like in the text case, we can have the store objects referenced by their paths. + This is for the + ["Nix Archive"](@docroot@/store/store-object/content-address.md#method-nix-archive) + method of content addressing store objects, + if the hash algorithm is [SHA-256]. + Just like in the "Text" case, we can have the store objects referenced by their paths. Additionally, we can have an optional `:self` label to denote self reference. - ```ebnf @@ -53,8 +60,12 @@ where ``` For either the outputs built from derivations, - paths copied to the store hashed that area single file hashed directly, or the via a hash algorithm other than [SHA-256][sha-256]. - (in that case "source" is used; this is only necessary for compatibility). + or content-addressed store objects that are not using one of the two above cases. + To be explicit about the latter, that is currently these methods: + + - ["Flat"](@docroot@/store/store-object/content-address.md#method-flat) + - ["Git"](@docroot@/store/store-object/content-address.md#method-git) + - ["Nix Archive"](@docroot@/store/store-object/content-address.md#method-nix-archive) if the hash algorithm is not [SHA-256]. `id` is the name of the output (usually, "out"). For content-addressed store objects, `id`, is always "out". @@ -113,8 +124,8 @@ where Note that `id` = `"out"`, regardless of the name part of the store path. Also note that NAR + SHA-256 must not use this case, and instead must use the `type` = `"source:" ...` case. -[Nix Archive (NAR)]: @docroot@/glossary.md#gloss-NAR -[sha-256]: https://en.m.wikipedia.org/wiki/SHA-256 +[Nix Archive (NAR)]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive +[SHA-256]: https://en.m.wikipedia.org/wiki/SHA-256 ### Historical Note diff --git a/doc/manual/src/protocols/tarball-fetcher.md b/doc/manual/src/protocols/tarball-fetcher.md index 274fa6d63..5cff05d66 100644 --- a/doc/manual/src/protocols/tarball-fetcher.md +++ b/doc/manual/src/protocols/tarball-fetcher.md @@ -22,7 +22,7 @@ Link: ; rel="immutable" *flakeref* must be a tarball flakeref. It can contain the tarball flake attributes `narHash`, `rev`, `revCount` and `lastModified`. If `narHash` is included, its -value must be the NAR hash of the unpacked tarball (as computed via +value must be the [NAR hash][Nix Archive] of the unpacked tarball (as computed via `nix hash path`). Nix checks the contents of the returned tarball against the `narHash` attribute. The `rev` and `revCount` attributes are useful when the tarball flake is a mirror of a fetcher type that @@ -40,3 +40,31 @@ Link: ///archive/.tar.gz +``` + +> **Example** +> +> +> ```nix +> # flake.nix +> { +> inputs = { +> foo.url = "https://gitea.example.org/some-person/some-flake/archive/main.tar.gz"; +> bar.url = "https://gitea.example.org/some-other-person/other-flake/archive/442793d9ec0584f6a6e82fa253850c8085bb150a.tar.gz"; +> qux = { +> url = "https://forgejo.example.org/another-person/some-non-flake-repo/archive/development.tar.gz"; +> flake = false; +> }; +> }; +> outputs = { foo, bar, qux }: { /* ... */ }; +> } +``` + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive diff --git a/doc/manual/src/quick-start.md b/doc/manual/src/quick-start.md index 75853ced7..9eb7a3265 100644 --- a/doc/manual/src/quick-start.md +++ b/doc/manual/src/quick-start.md @@ -34,7 +34,7 @@ For more in-depth information you are kindly referred to subsequent chapters. lolcat: command not found ``` -1. Search for more packages on to try them out. +1. Search for more packages on [search.nixos.org](https://search.nixos.org/) to try them out. 1. Free up storage space: diff --git a/doc/manual/src/release-notes/index.md b/doc/manual/src/release-notes/index.md index cc805e631..d4e6292a6 100644 --- a/doc/manual/src/release-notes/index.md +++ b/doc/manual/src/release-notes/index.md @@ -1,12 +1,13 @@ # Nix Release Notes +The Nix release cycle is calendar-based as follows: + Nix has a release cycle of roughly 6 weeks. Notable changes and additions are announced in the release notes for each version. -Bugfixes can be backported on request to previous Nix releases. -We typically backport only as far back as the Nix version used in the latest NixOS release, which is announced in the [NixOS release notes](https://nixos.org/manual/nixos/stable/release-notes.html#ch-release-notes). - -Backports never skip releases. -If a feature is backported to version `x.y`, it must also be available in version `x.(y+1)`. -This ensures that upgrading from an older version with backports is still safe and no backported functionality will go missing. +The supported Nix versions are: +- The latest release +- The version used in the stable NixOS release, which is announced in the [NixOS release notes](https://nixos.org/manual/nixos/stable/release-notes.html#ch-release-notes). +Bugfixes and security issues are backported to every supported version. +Patch releases are published as needed. diff --git a/doc/manual/src/release-notes/rl-2.15.md b/doc/manual/src/release-notes/rl-2.15.md index 133121999..e7e52631b 100644 --- a/doc/manual/src/release-notes/rl-2.15.md +++ b/doc/manual/src/release-notes/rl-2.15.md @@ -11,7 +11,7 @@ As the choice of hash formats is no longer binary, the `--base16` flag is also added to explicitly specify the Base16 format, which is still the default. -* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](../glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents. +* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](@docroot@/glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents. The new `^` syntax for store paths introduced in Nix 2.13 allows explicitly referencing output paths of a derivation. Using this is better and more clear than relying on the now-removed `.drv` special handling. diff --git a/doc/manual/src/release-notes/rl-2.18.md b/doc/manual/src/release-notes/rl-2.18.md index 4bbc52b50..eb26fc9e7 100644 --- a/doc/manual/src/release-notes/rl-2.18.md +++ b/doc/manual/src/release-notes/rl-2.18.md @@ -13,7 +13,7 @@ - The `discard-references` feature has been stabilized. This means that the - [unsafeDiscardReferences](@docroot@/contributing/experimental-features.md#xp-feature-discard-references) + [unsafeDiscardReferences](@docroot@/development/experimental-features.md#xp-feature-discard-references) attribute is no longer guarded by an experimental flag and can be used freely. @@ -21,7 +21,7 @@ This only affects `nix-build --json` when "building" non-derivation things like fetched sources, which is a no-op. - A new builtin [`outputOf`](@docroot@/language/builtins.md#builtins-outputOf) has been added. - It is part of the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. + It is part of the [`dynamic-derivations`](@docroot@/development/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. - Flake follow paths at depths greater than 2 are now handled correctly, preventing "follows a non-existent input" errors. diff --git a/doc/manual/src/release-notes/rl-2.19.md b/doc/manual/src/release-notes/rl-2.19.md index ba6eb9c64..e2e2f85cc 100644 --- a/doc/manual/src/release-notes/rl-2.19.md +++ b/doc/manual/src/release-notes/rl-2.19.md @@ -17,8 +17,8 @@ - `nix-shell` shebang lines now support single-quoted arguments. -- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). - This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). +- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/development/experimental-features.md#xp-fetch-tree). + This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/development/experimental-features.md#xp-fetch-tree). - The interface for creating and updating lock files has been overhauled: @@ -33,7 +33,7 @@ - The flake-specific flags `--recreate-lock-file` and `--update-input` have been removed from all commands operating on installables. They are superceded by `nix flake update`. -- Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). +- Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches). - [`nix path-info --json`](@docroot@/command-ref/new-cli/nix3-path-info.md) (experimental) now returns a JSON map rather than JSON list. diff --git a/doc/manual/src/release-notes/rl-2.20.md b/doc/manual/src/release-notes/rl-2.20.md index 8ede168a4..eb724f600 100644 --- a/doc/manual/src/release-notes/rl-2.20.md +++ b/doc/manual/src/release-notes/rl-2.20.md @@ -200,3 +200,9 @@ while performing various operations (including `nix develop`, `nix flake update`, and so on). With several fixes to Nix's signal handlers, Nix commands will now exit quickly after Ctrl-C is pressed. + +- `nix copy` to a `ssh-ng` store now needs `--substitute-on-destination` (a.k.a. `-s`) + in order to substitute paths on the remote store instead of copying them. + The behavior is consistent with `nix copy` to a different kind of remote store. + Previously this behavior was controlled by the + `builders-use-substitutes` setting and `--substitute-on-destination` was ignored. diff --git a/doc/manual/src/release-notes/rl-2.21.md b/doc/manual/src/release-notes/rl-2.21.md new file mode 100644 index 000000000..75114f117 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.21.md @@ -0,0 +1,302 @@ +# Release 2.21.0 (2024-03-11) + +- Fix a fixed-output derivation sandbox escape (CVE-2024-27297) + + Cooperating Nix derivations could send file descriptors to files in the Nix + store to each other via Unix domain sockets in the abstract namespace. This + allowed one derivation to modify the output of the other derivation, after Nix + has registered the path as "valid" and immutable in the Nix database. + In particular, this allowed the output of fixed-output derivations to be + modified from their expected content. + + This isn't the case any more. + +- CLI options `--arg-from-file` and `--arg-from-stdin` [#10122](https://github.com/NixOS/nix/pull/10122) + + The new CLI option `--arg-from-file` *name* *path* passes the contents + of file *path* as a string value via the function argument *name* to a + Nix expression. Similarly, the new option `--arg-from-stdin` *name* + reads the contents of the string from standard input. + +- Concise error printing in `nix repl` [#9928](https://github.com/NixOS/nix/pull/9928) + + Previously, if an element of a list or attribute set threw an error while + evaluating, `nix repl` would print the entire error (including source location + information) inline. This output was clumsy and difficult to parse: + + ``` + nix-repl> { err = builtins.throw "uh oh!"; } + { err = «error: + … while calling the 'throw' builtin + at «string»:1:9: + 1| { err = builtins.throw "uh oh!"; } + | ^ + + error: uh oh!»; } + ``` + + Now, only the error message is displayed, making the output much more readable. + ``` + nix-repl> { err = builtins.throw "uh oh!"; } + { err = «error: uh oh!»; } + ``` + + However, if the whole expression being evaluated throws an error, source + locations and (if applicable) a stack trace are printed, just like you'd expect: + + ``` + nix-repl> builtins.throw "uh oh!" + error: + … while calling the 'throw' builtin + at «string»:1:1: + 1| builtins.throw "uh oh!" + | ^ + + error: uh oh! + ``` + +- `--debugger` can now access bindings from `let` expressions [#8827](https://github.com/NixOS/nix/issues/8827) [#9918](https://github.com/NixOS/nix/pull/9918) + + Breakpoints and errors in the bindings of a `let` expression can now access + those bindings in the debugger. Previously, only the body of `let` expressions + could access those bindings. + +- Enter the `--debugger` when `builtins.trace` is called if `debugger-on-trace` is set [#9914](https://github.com/NixOS/nix/pull/9914) + + If the `debugger-on-trace` option is set and `--debugger` is given, + `builtins.trace` calls will behave similarly to `builtins.break` and will enter + the debug REPL. This is useful for determining where warnings are being emitted + from. + +- Debugger prints source position information [#9913](https://github.com/NixOS/nix/pull/9913) + + The `--debugger` now prints source location information, instead of the + pointers of source location information. Before: + + ``` + nix-repl> :bt + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + 0x600001522598 + ``` + + After: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + /nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27 + + 131| + 132| bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs; + | ^ + 133| in + ``` + +- The `--debugger` will start more reliably in `let` expressions and function calls [#6649](https://github.com/NixOS/nix/issues/6649) [#9917](https://github.com/NixOS/nix/pull/9917) + + Previously, if you attempted to evaluate this file with the debugger: + + ```nix + let + a = builtins.trace "before inner break" ( + builtins.break "hello" + ); + b = builtins.trace "before outer break" ( + builtins.break a + ); + in + b + ``` + + Nix would correctly enter the debugger at `builtins.break a`, but if you asked + it to `:continue`, it would skip over the `builtins.break "hello"` expression + entirely. + + Now, Nix will correctly enter the debugger at both breakpoints. + +- Nested debuggers are no longer supported [#9920](https://github.com/NixOS/nix/pull/9920) + + Previously, evaluating an expression that throws an error in the debugger would + enter a second, nested debugger: + + ``` + nix-repl> builtins.throw "what" + error: what + + + Starting REPL to allow you to inspect the current state of the evaluator. + + Welcome to Nix 2.18.1. Type :? for help. + + nix-repl> + ``` + + Now, it just prints the error message like `nix repl`: + + ``` + nix-repl> builtins.throw "what" + error: + … while calling the 'throw' builtin + at «string»:1:1: + 1| builtins.throw "what" + | ^ + + error: what + ``` + +- Consistent order of function arguments in printed expressions [#9874](https://github.com/NixOS/nix/pull/9874) + + Function arguments are now printed in lexicographic order rather than the internal, creation-time based symbol order. + +- Fix duplicate attribute error positions for `inherit` [#9874](https://github.com/NixOS/nix/pull/9874) + + When an `inherit` caused a duplicate attribute error the position of the error was not reported correctly, placing the error with the inherit itself or at the start of the bindings block instead of the offending attribute name. + +- `inherit (x) ...` evaluates `x` only once [#9847](https://github.com/NixOS/nix/pull/9847) + + `inherit (x) a b ...` now evaluates the expression `x` only once for all inherited attributes rather than once for each inherited attribute. + This does not usually have a measurable impact, but side-effects (such as `builtins.trace`) would be duplicated and expensive expressions (such as derivations) could cause a measurable slowdown. + +- Store paths are allowed to start with `.` [#912](https://github.com/NixOS/nix/issues/912) [#9091](https://github.com/NixOS/nix/pull/9091) [#9095](https://github.com/NixOS/nix/pull/9095) [#9120](https://github.com/NixOS/nix/pull/9120) [#9121](https://github.com/NixOS/nix/pull/9121) [#9122](https://github.com/NixOS/nix/pull/9122) [#9130](https://github.com/NixOS/nix/pull/9130) [#9219](https://github.com/NixOS/nix/pull/9219) [#9224](https://github.com/NixOS/nix/pull/9224) [#9867](https://github.com/NixOS/nix/pull/9867) + + Leading periods were allowed by accident in Nix 2.4. The Nix team has considered this to be a bug, but this behavior has since been relied on by users, leading to unnecessary difficulties. + From now on, leading periods are supported. The names `.` and `..` are disallowed, as well as those starting with `.-` or `..-`. + + Nix versions that denied leading periods are documented [in the issue](https://github.com/NixOS/nix/issues/912#issuecomment-1919583286). + +- `nix repl` pretty-prints values [#9931](https://github.com/NixOS/nix/pull/9931) + + `nix repl` will now pretty-print values: + + ``` + { + attrs = { + a = { + b = { + c = { }; + }; + }; + }; + list = [ 1 ]; + list' = [ + 1 + 2 + 3 + ]; + } + ``` + +- Introduction of `--regex` and `--all` in `nix profile remove` and `nix profile upgrade` [#10166](https://github.com/NixOS/nix/pull/10166) + + Previously the command-line arguments for `nix profile remove` and `nix profile upgrade` matched the package entries using regular expression. + For instance: + + ``` + nix profile remove '.*vim.*' + ``` + + This would remove all packages that contain `vim` in their name. + + In most cases, only singular package names were used to remove and upgrade packages. Mixing this with regular expressions sometimes lead to unintended behavior. For instance, `python3.1` could match `python311`. + + To avoid unintended behavior, the arguments are now only matching exact names. + + Matching using regular expressions is still possible by using the new `--regex` flag: + + ``` + nix profile remove --regex '.*vim.*' + ``` + + One of the most useful cases for using regular expressions was to upgrade all packages. This was previously accomplished by: + + ``` + nix profile upgrade '.*' + ``` + + With the introduction of the `--all` flag, this now becomes more straightforward: + + ``` + nix profile upgrade --all + ``` + +- Visual clutter in `--debugger` is reduced [#9919](https://github.com/NixOS/nix/pull/9919) + + Before: + ``` + info: breakpoint reached + + + Starting REPL to allow you to inspect the current state of the evaluator. + + Welcome to Nix 2.20.0pre20231222_dirty. Type :? for help. + + nix-repl> :continue + error: uh oh + + + Starting REPL to allow you to inspect the current state of the evaluator. + + Welcome to Nix 2.20.0pre20231222_dirty. Type :? for help. + + nix-repl> + ``` + + After: + + ``` + info: breakpoint reached + + Nix 2.20.0pre20231222_dirty debugger + Type :? for help. + nix-repl> :continue + error: uh oh + + nix-repl> + ``` + +- Cycle detection in `nix repl` is simpler and more reliable [#8672](https://github.com/NixOS/nix/issues/8672) [#9926](https://github.com/NixOS/nix/pull/9926) + + The cycle detection in `nix repl`, `nix eval`, `builtins.trace`, and everywhere + else values are printed is now simpler and matches the cycle detection in + `nix-instantiate --eval` output. + + Before: + + ``` + nix eval --expr 'let self = { inherit self; }; in self' + { self = { self = «repeated»; }; } + ``` + + After: + + ``` + { self = «repeated»; } + ``` + +- In the debugger, `while evaluating the attribute` errors now include position information [#9915](https://github.com/NixOS/nix/pull/9915) + + Before: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + 0x600001522598 + ``` + + After: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + /nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27 + + 131| + 132| bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs; + | ^ + 133| in + ``` + +- Stack size is increased on macOS [#9860](https://github.com/NixOS/nix/pull/9860) + + Previously, Nix would set the stack size to 64MiB on Linux, but would leave the + stack size set to the default (approximately 8KiB) on macOS. Now, the stack + size is correctly set to 64MiB on macOS as well, which should reduce stack + overflow segfaults in deeply-recursive Nix expressions. + diff --git a/doc/manual/src/release-notes/rl-2.22.md b/doc/manual/src/release-notes/rl-2.22.md new file mode 100644 index 000000000..c78d3d692 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.22.md @@ -0,0 +1,21 @@ +# Release 2.22.0 (2024-04-23) + +### Significant changes + +- Remove experimental repl-flake [#10103](https://github.com/NixOS/nix/issues/10103) [#10299](https://github.com/NixOS/nix/pull/10299) + + The `repl-flake` experimental feature has been removed. The `nix repl` command now works like the rest of the new CLI in that `nix repl {path}` now tries to load a flake at `{path}` (or fails if the `flakes` experimental feature isn't enabled). + +### Other changes + +- `nix eval` prints derivations as `.drv` paths [#10200](https://github.com/NixOS/nix/pull/10200) + + `nix eval` will now print derivations as their `.drv` paths, rather than as + attribute sets. This makes commands like `nix eval nixpkgs#bash` terminate + instead of infinitely looping into recursive self-referential attributes: + + ```ShellSession + $ nix eval nixpkgs#bash + «derivation /nix/store/m32cbgbd598f4w299g0hwyv7gbw6rqcg-bash-5.2p26.drv» + ``` + diff --git a/doc/manual/src/release-notes/rl-2.23.md b/doc/manual/src/release-notes/rl-2.23.md new file mode 100644 index 000000000..ac842fdc0 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.23.md @@ -0,0 +1,102 @@ +# Release 2.23.0 (2024-06-03) + +- New builtin: `builtins.warn` [#306026](https://github.com/NixOS/nix/issues/306026) [#10592](https://github.com/NixOS/nix/pull/10592) + + `builtins.warn` behaves like `builtins.trace "warning: ${msg}"`, has an accurate log level, and is controlled by the options + [`debugger-on-trace`](@docroot@/command-ref/conf-file.md#conf-debugger-on-trace), + [`debugger-on-warn`](@docroot@/command-ref/conf-file.md#conf-debugger-on-warn) and + [`abort-on-warn`](@docroot@/command-ref/conf-file.md#conf-abort-on-warn). + +- Make `nix build --keep-going` consistent with `nix-build --keep-going` + + This means that if e.g. multiple fixed-output derivations fail to + build, all hash mismatches are displayed. + +- Modify `nix derivation {add,show}` JSON format [#9866](https://github.com/NixOS/nix/issues/9866) [#10722](https://github.com/NixOS/nix/pull/10722) + + The JSON format for derivations has been slightly revised to better conform to our [JSON guidelines](@docroot@/development/cli-guideline.md#returning-future-proof-json). + In particular, the hash algorithm and content addressing method of content-addresed derivation outputs are now separated into two fields `hashAlgo` and `method`, + rather than one field with an arcane `:`-separated format. + + This JSON format is only used by the experimental `nix derivation` family of commands, at this time. + Future revisions are expected as the JSON format is still not entirely in compliance even after these changes. + +- Warn on unknown settings anywhere in the command line [#10701](https://github.com/NixOS/nix/pull/10701) + + All `nix` commands will now properly warn when an unknown option is specified anywhere in the command line. + + Before: + + ```console + $ nix-instantiate --option foobar baz --expr '{}' + warning: unknown setting 'foobar' + $ nix-instantiate '{}' --option foobar baz --expr + $ nix eval --expr '{}' --option foobar baz + { } + ``` + + After: + + ```console + $ nix-instantiate --option foobar baz --expr '{}' + warning: unknown setting 'foobar' + $ nix-instantiate '{}' --option foobar baz --expr + warning: unknown setting 'foobar' + $ nix eval --expr '{}' --option foobar baz + warning: unknown setting 'foobar' + { } + ``` + +- `nix env shell` is the new `nix shell`, and `nix shell` remains an accepted alias [#10504](https://github.com/NixOS/nix/issues/10504) [#10807](https://github.com/NixOS/nix/pull/10807) + + This is part of an effort to bring more structure to the CLI subcommands. + + `nix env` will be about the process environment. + Future commands may include `nix env run` and `nix env print-env`. + + It is also somewhat analogous to the [planned](https://github.com/NixOS/nix/issues/10504) `nix dev shell` (currently `nix develop`), which is less about environment variables, and more about running a development shell, which is a more powerful command, but also requires more setup. + +- Flake operations that expect derivations now print the failing value and its type [#10778](https://github.com/NixOS/nix/pull/10778) + + In errors like `flake output attribute 'nixosConfigurations.yuki.config' is not a derivation or path`, the message now includes the failing value and type. + + Before: + + ``` + error: flake output attribute 'nixosConfigurations.yuki.config' is not a derivation or path + ```` + + After: + + ``` + error: expected flake output attribute 'nixosConfigurations.yuki.config' to be a derivation or path but found a set: { appstream = «thunk»; assertions = «thunk»; boot = { bcache = «thunk»; binfmt = «thunk»; binfmtMiscRegistrations = «thunk»; blacklistedKernelModules = «thunk»; bootMount = «thunk»; bootspec = «thunk»; cleanTmpDir = «thunk»; consoleLogLevel = «thunk»; «43 attributes elided» }; «48 attributes elided» } + ``` + +- `fetchTree` now fetches Git repositories shallowly by default [#10028](https://github.com/NixOS/nix/pull/10028) + + `builtins.fetchTree` now clones Git repositories shallowly by default, which reduces network traffic and disk usage significantly in many cases. + + Previously, the default behavior was to clone the full history of a specific tag or branch (e.g. `ref`) and only afterwards extract the files of one specific revision. + + From now on, the `ref` and `allRefs` arguments will be ignored, except if shallow cloning is disabled by setting `shallow = false`. + + The defaults for `builtins.fetchGit` remain unchanged. Here, shallow cloning has to be enabled manually by passing `shallow = true`. + +- Store object info JSON format now uses `null` rather than omitting fields [#9995](https://github.com/NixOS/nix/pull/9995) + + The [store object info JSON format](@docroot@/protocols/json/store-object-info.md), used for e.g. `nix path-info`, no longer omits fields to indicate absent information, but instead includes the fields with a `null` value. + For example, `"ca": null` is used to to indicate a store object that isn't content-addressed rather than omitting the `ca` field entirely. + This makes records of this sort more self-describing, and easier to consume programmatically. + + We will follow this design principle going forward; + the [JSON guidelines](@docroot@/development/json-guideline.md) in the contributing section have been updated accordingly. + +- Large path warnings [#10661](https://github.com/NixOS/nix/pull/10661) + + Nix can now warn when evaluation of a Nix expression causes a large + path to be copied to the Nix store. The threshold for this warning can + be configured using [the `warn-large-path-threshold` + setting](@docroot@/command-ref/conf-file.md#warn-large-path-threshold), + e.g. `--warn-large-path-threshold 100M` will warn about paths larger + than 100 MiB. + diff --git a/doc/manual/src/release-notes/rl-2.24.md b/doc/manual/src/release-notes/rl-2.24.md new file mode 100644 index 000000000..5bcc1d79c --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.24.md @@ -0,0 +1,324 @@ +# Release 2.24.0 (2024-07-31) + +### Significant changes + +- Harden user sandboxing + + The build directory has been hardened against interference with the outside world by nesting it inside another directory owned by (and only readable by) the daemon user. + + This is a low severity security fix, [CVE-2024-38531](https://www.cve.org/CVERecord?id=CVE-2024-38531). + + Credit: [**@alois31**](https://github.com/alois31), [**Linus Heckemann (@lheckemann)**](https://github.com/lheckemann) + Co-authors: [**@edolstra**](https://github.com/edolstra) + +- `nix-shell ` looks for `shell.nix` [#496](https://github.com/NixOS/nix/issues/496) [#2279](https://github.com/NixOS/nix/issues/2279) [#4529](https://github.com/NixOS/nix/issues/4529) [#5431](https://github.com/NixOS/nix/issues/5431) [#11053](https://github.com/NixOS/nix/issues/11053) [#11057](https://github.com/NixOS/nix/pull/11057) + + `nix-shell $x` now looks for `$x/shell.nix` when `$x` resolves to a directory. + + Although this might be seen as a breaking change, its primarily interactive usage makes it a minor issue. + This adjustment addresses a commonly reported problem. + + This also applies to `nix-shell` shebang scripts. Consider the following example: + + ```shell + #!/usr/bin/env nix-shell + #!nix-shell -i bash + ``` + + This will now load `shell.nix` from the script's directory, if it exists; `default.nix` otherwise. + + The old behavior can be opted into by setting the option [`nix-shell-always-looks-for-shell-nix`](@docroot@/command-ref/conf-file.md#conf-nix-shell-always-looks-for-shell-nix) to `false`. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + +- `nix-repl`'s `:doc` shows documentation comments [#3904](https://github.com/NixOS/nix/issues/3904) [#10771](https://github.com/NixOS/nix/issues/10771) [#1652](https://github.com/NixOS/nix/pull/1652) [#9054](https://github.com/NixOS/nix/pull/9054) [#11072](https://github.com/NixOS/nix/pull/11072) + + `nix repl` has a `:doc` command that previously only rendered documentation for internally defined functions. + This feature has been extended to also render function documentation comments, in accordance with [RFC 145]. + + Example: + + ``` + nix-repl> :doc lib.toFunction + Function toFunction + … defined at /home/user/h/nixpkgs/lib/trivial.nix:1072:5 + + Turns any non-callable values into constant functions. Returns + callable values as is. + + Inputs + + v + + : Any value + + Examples + + :::{.example} + + ## lib.trivial.toFunction usage example + + | nix-repl> lib.toFunction 1 2 + | 1 + | + | nix-repl> lib.toFunction (x: x + 1) 2 + | 3 + + ::: + ``` + + Known limitations: + - It does not render documentation for "formals", such as `{ /** the value to return */ x, ... }: x`. + - Some extensions to markdown are not yet supported, as you can see in the example above. + + We'd like to acknowledge [Yingchi Long (@inclyc)](https://github.com/inclyc) for proposing a proof of concept for this functionality in [#9054](https://github.com/NixOS/nix/pull/9054), as well as [@sternenseemann](https://github.com/sternenseemann) and [Johannes Kirschbauer (@hsjobeki)](https://github.com/hsjobeki) for their contributions, proposals, and their work on [RFC 145]. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + + [RFC 145]: https://github.com/NixOS/rfcs/pull/145 + +### Other changes + +- Solve `cached failure of attribute X` [#9165](https://github.com/NixOS/nix/issues/9165) [#10513](https://github.com/NixOS/nix/issues/10513) [#10564](https://github.com/NixOS/nix/pull/10564) + + This eliminates all "cached failure of attribute X" messages by forcing evaluation of the original value when needed to show the exception to the user. This enhancement improves error reporting by providing the underlying message and stack trace. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Run the flake regressions test suite [#10603](https://github.com/NixOS/nix/pull/10603) + + This update introduces a GitHub action to run a subset of the [flake regressions test suite](https://github.com/NixOS/flake-regressions), which includes 259 flakes with their expected evaluation results. Currently, the action runs the first 25 flakes due to the full test suite's extensive runtime. A manually triggered action may be implemented later to run the entire test suite. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Support unit prefixes in configuration settings [#10668](https://github.com/NixOS/nix/pull/10668) + + Configuration settings in Nix now support unit prefixes, allowing for more intuitive and readable configurations. For example, you can now specify [`--min-free 1G`](@docroot@/command-ref/opt-common.md#opt-min-free) to set the minimum free space to 1 gigabyte. + + This enhancement was extracted from [#7851](https://github.com/NixOS/nix/pull/7851) and is also useful for PR [#10661](https://github.com/NixOS/nix/pull/10661). + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- `nix build`: show all FOD errors with `--keep-going` [#10734](https://github.com/NixOS/nix/pull/10734) + + The [`nix build`](@docroot@/command-ref/new-cli/nix3-build.md) command has been updated to improve the behavior of the [`--keep-going`] flag. Now, when `--keep-going` is used, all hash-mismatch errors of failing fixed-output derivations (FODs) are displayed, similar to the behavior for other build failures. This enhancement ensures that all relevant build errors are shown, making it easier for users to update multiple derivations at once or to diagnose and fix issues. + + Author: [**Jörg Thalheim (@Mic92)**](https://github.com/Mic92), [**Maximilian Bosch (@Ma27)**](https://github.com/Ma27) + + [`--keep-going`](@docroot@/command-ref/opt-common.md#opt-keep-going) + +- Build with Meson [#2503](https://github.com/NixOS/nix/issues/2503) [#10378](https://github.com/NixOS/nix/pull/10378) [#10855](https://github.com/NixOS/nix/pull/10855) [#10904](https://github.com/NixOS/nix/pull/10904) [#10908](https://github.com/NixOS/nix/pull/10908) [#10914](https://github.com/NixOS/nix/pull/10914) [#10933](https://github.com/NixOS/nix/pull/10933) [#10936](https://github.com/NixOS/nix/pull/10936) [#10954](https://github.com/NixOS/nix/pull/10954) [#10955](https://github.com/NixOS/nix/pull/10955) [#10963](https://github.com/NixOS/nix/pull/10963) [#10967](https://github.com/NixOS/nix/pull/10967) [#10973](https://github.com/NixOS/nix/pull/10973) [#11034](https://github.com/NixOS/nix/pull/11034) [#11054](https://github.com/NixOS/nix/pull/11054) [#11055](https://github.com/NixOS/nix/pull/11055) [#11060](https://github.com/NixOS/nix/pull/11060) [#11064](https://github.com/NixOS/nix/pull/11064) [#11155](https://github.com/NixOS/nix/pull/11155) + + These changes aim to replace the use of autotools and `make` with Meson for building various components of Nix. Additionally, each library is built in its own derivation, leveraging Meson's "subprojects" feature to allow a single development shell for building all libraries while also supporting separate builds. This approach aims to improve productivity and build modularity, compared to both make and a monolithic Meson-based derivation. + + Special thanks to everyone who has contributed to the Meson port, particularly [**@p01arst0rm**](https://github.com/p01arst0rm) and [**@Qyriad**](https://github.com/Qyriad). + + Authors: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314), [**Tom Bereknyei**](https://github.com/tomberek), [**Théophane Hufschmitt (@thufschmitt)**](https://github.com/thufschmitt), [**Valentin Gagarin (@fricklerhandwerk)**](https://github.com/fricklerhandwerk), [**Robert Hensing (@roberth)**](https://github.com/roberth) + Co-authors: [**@p01arst0rm**](https://github.com/p01arst0rm), [**@Qyriad**](https://github.com/Qyriad) + +- Evaluation cache: fix cache regressions [#10570](https://github.com/NixOS/nix/issues/10570) [#11086](https://github.com/NixOS/nix/pull/11086) + + This update addresses two bugs in the evaluation cache system: + + 1. Regression in #10570: The evaluation cache was not being persisted in `nix develop`. + 2. Nix could sometimes try to commit the evaluation cache SQLite transaction without there being an active transaction, resulting in non-error errors being printed. + + Author: [**Lexi Mattick (@kognise)**](https://github.com/kognise) + +- Introduce `libnixflake` [#9063](https://github.com/NixOS/nix/pull/9063) + + A new library, `libnixflake`, has been introduced to better separate the Flakes layer within Nix. This change refactors the codebase to encapsulate Flakes-specific functionality within its own library. + + See the commits in the pull request for detailed changes, with the only significant code modifications happening in the initial commit. + + This change was alluded to in [RFC 134](https://github.com/nixos/rfcs/blob/master/rfcs/0134-nix-store-layer.md) and is a step towards a more modular and maintainable codebase. + + Author: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314) + +- CLI options `--arg-from-file` and `--arg-from-stdin` [#9913](https://github.com/NixOS/nix/pull/9913) + +- The `--debugger` now prints source location information, instead of the + pointers of source location information. Before: + + ``` + nix-repl> :bt + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + 0x600001522598 + ``` + + After: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + /nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27 + + 131| + 132| bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs; + | ^ + 133| in + ``` + +- Stop vendoring `toml11` + + We don't apply any patches to it, and vendoring it locks users into + bugs (it hasn't been updated since its introduction in late 2021). + + Author: [**Winter (@winterqt)**](https://github.com/winterqt) + +- Rename hash format `base32` to `nix32` [#8678](https://github.com/NixOS/nix/pull/8678) + + Hash format `base32` was renamed to `nix32` since it used a special nix-specific character set for + [Base32](https://en.wikipedia.org/wiki/Base32). + + **Deprecation**: Use `nix32` instead of `base32` as `toHashFormat` + + For the builtin `convertHash`, the `toHashFormat` parameter now accepts the same hash formats as the `--to`/`--from` + parameters of the `nix hash conert` command: `"base16"`, `"nix32"`, `"base64"`, and `"sri"`. The former `"base32"` value + remains as a deprecated alias for `"nix32"`. Please convert your code from: + + ```nix + builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base32";} + ``` + + to + + ```nix + builtins.convertHash { inherit hash hashAlgo; toHashFormat = "nix32";} + ``` + +- Add `pipe-operators` experimental feature [#11131](https://github.com/NixOS/nix/pull/11131) + + This is a draft implementation of [RFC 0148](https://github.com/NixOS/rfcs/pull/148). + + The `pipe-operators` experimental feature adds [`<|` and `|>` operators][pipe operators] to the Nix language. + *a* `|>` *b* is equivalent to the function application *b* *a*, and + *a* `<|` *b* is equivalent to the function application *a* *b*. + + For example: + + ``` + nix-repl> 1 |> builtins.add 2 |> builtins.mul 3 + 9 + + nix-repl> builtins.add 1 <| builtins.mul 2 <| 3 + 7 + ``` + + `<|` and `|>` are right and left associative, respectively, and have lower precedence than any other operator. + These properties may change in future releases. + + See [the RFC](https://github.com/NixOS/rfcs/pull/148) for more examples and rationale. + + [pipe operators]: @docroot@/language/operators.md#pipe-operators + +- `nix-shell` shebang uses relative path [#4232](https://github.com/NixOS/nix/issues/4232) [#5088](https://github.com/NixOS/nix/pull/5088) [#11058](https://github.com/NixOS/nix/pull/11058) + + + Relative [path](@docroot@/language/types.md#type-path) literals in `nix-shell` shebang scripts' options are now resolved relative to the [script's location](@docroot@/glossary.md?highlight=base%20directory#gloss-base-directory). + Previously they were resolved relative to the current working directory. + + For example, consider the following script in `~/myproject/say-hi`: + + ```shell + #!/usr/bin/env nix-shell + #!nix-shell --expr 'import ./shell.nix' + #!nix-shell --arg toolset './greeting-tools.nix' + #!nix-shell -i bash + hello + ``` + + Older versions of `nix-shell` would resolve `shell.nix` relative to the current working directory, such as the user's home directory in this example: + + ```console + [hostname:~]$ ./myproject/say-hi + error: + … while calling the 'import' builtin + at «string»:1:2: + 1| (import ./shell.nix) + | ^ + + error: path '/home/user/shell.nix' does not exist + ``` + + Since this release, `nix-shell` resolves `shell.nix` relative to the script's location, and `~/myproject/shell.nix` is used. + + ```console + $ ./myproject/say-hi + Hello, world! + ``` + + **Opt-out** + + This is technically a breaking change, so we have added an option so you can adapt independently of your Nix update. + The old behavior can be opted into by setting the option [`nix-shell-shebang-arguments-relative-to-script`](@docroot@/command-ref/conf-file.md#conf-nix-shell-shebang-arguments-relative-to-script) to `false`. + This option will be removed in a future release. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + +- Improve handling of tarballs that don't consist of a single top-level directory [#11195](https://github.com/NixOS/nix/pull/11195) + + In previous Nix releases, the tarball fetcher (used by `builtins.fetchTarball`) erroneously merged top-level directories into a single directory, and silently discarded top-level files that are not directories. This is no longer the case. The new behaviour is that *only* if the tarball consists of a single directory, the top-level path component of the files in the tarball is removed (similar to `tar`'s `--strip-components=1`). + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Improve handling of tarballs that don't consist of a single top-level directory [#11195](https://github.com/NixOS/nix/pull/11195) + + In previous Nix releases, the tarball fetcher (used by `builtins.fetchTarball`) erroneously merged top-level directories into a single directory, and silently discarded top-level files that are not directories. This is no longer the case. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Setting to warn about large paths [#10778](https://github.com/NixOS/nix/pull/10778) + + Nix can now warn when evaluation of a Nix expression causes a large + path to be copied to the Nix store. The threshold for this warning can + be configured using the `warn-large-path-threshold` setting, + e.g. `--warn-large-path-threshold 100M`. + + +# Contributors + +This release was made possible by the following 43 contributors: + +- Andreas Rammhold [**(@andir)**](https://github.com/andir) +- Andrew Marshall [**(@amarshall)**](https://github.com/amarshall) +- Brian McKenna [**(@puffnfresh)**](https://github.com/puffnfresh) +- Cameron [**(@SkamDart)**](https://github.com/SkamDart) +- Cole Helbling [**(@cole-h)**](https://github.com/cole-h) +- Corbin Simpson [**(@MostAwesomeDude)**](https://github.com/MostAwesomeDude) +- Eelco Dolstra [**(@edolstra)**](https://github.com/edolstra) +- Emily [**(@emilazy)**](https://github.com/emilazy) +- Enno Richter [**(@elohmeier)**](https://github.com/elohmeier) +- Farid Zakaria [**(@fzakaria)**](https://github.com/fzakaria) +- HaeNoe [**(@haenoe)**](https://github.com/haenoe) +- Hamir Mahal [**(@hamirmahal)**](https://github.com/hamirmahal) +- Harmen [**(@alicebob)**](https://github.com/alicebob) +- Ivan Trubach [**(@tie)**](https://github.com/tie) +- Jared Baur [**(@jmbaur)**](https://github.com/jmbaur) +- John Ericson [**(@Ericson2314)**](https://github.com/Ericson2314) +- Jonathan De Troye [**(@detroyejr)**](https://github.com/detroyejr) +- Jörg Thalheim [**(@Mic92)**](https://github.com/Mic92) +- Klemens Nanni [**(@klemensn)**](https://github.com/klemensn) +- Las Safin [**(@L-as)**](https://github.com/L-as) +- Lexi Mattick [**(@kognise)**](https://github.com/kognise) +- Matthew Bauer [**(@matthewbauer)**](https://github.com/matthewbauer) +- Max “Goldstein†Siling [**(@GoldsteinE)**](https://github.com/GoldsteinE) +- Mingye Wang [**(@Artoria2e5)**](https://github.com/Artoria2e5) +- Philip Taron [**(@philiptaron)**](https://github.com/philiptaron) +- Pierre Bourdon [**(@delroth)**](https://github.com/delroth) +- Pino Toscano [**(@pinotree)**](https://github.com/pinotree) +- RTUnreal [**(@RTUnreal)**](https://github.com/RTUnreal) +- Robert Hensing [**(@roberth)**](https://github.com/roberth) +- Romain Neil [**(@romain-neil)**](https://github.com/romain-neil) +- Ryan Hendrickson [**(@rhendric)**](https://github.com/rhendric) +- Sergei Trofimovich [**(@trofi)**](https://github.com/trofi) +- Shogo Takata [**(@pineapplehunter)**](https://github.com/pineapplehunter) +- Siddhant Kumar [**(@siddhantk232)**](https://github.com/siddhantk232) +- Silvan Mosberger [**(@infinisil)**](https://github.com/infinisil) +- Théophane Hufschmitt [**(@thufschmitt)**](https://github.com/thufschmitt) +- Valentin Gagarin [**(@fricklerhandwerk)**](https://github.com/fricklerhandwerk) +- Winter [**(@winterqt)**](https://github.com/winterqt) +- jade [**(@lf-)**](https://github.com/lf-) +- kirillrdy [**(@kirillrdy)**](https://github.com/kirillrdy) +- pennae [**(@pennae)**](https://github.com/pennae) +- poweredbypie [**(@poweredbypie)**](https://github.com/poweredbypie) +- tomberek [**(@tomberek)**](https://github.com/tomberek) diff --git a/doc/manual/src/release-notes/rl-2.4.md b/doc/manual/src/release-notes/rl-2.4.md index 8b566fc7b..1201e53b6 100644 --- a/doc/manual/src/release-notes/rl-2.4.md +++ b/doc/manual/src/release-notes/rl-2.4.md @@ -23,7 +23,7 @@ more than 2800 commits from 195 contributors since release 2.3. * The **`nix` command** has seen a lot of work and is now almost at feature parity with the old command-line interface (the `nix-*` commands). It aims to be [more modern, consistent and pleasant to - use](../contributing/cli-guideline.md) than the old CLI. It is still + use](../development/cli-guideline.md) than the old CLI. It is still marked as experimental but its interface should not change much anymore in future releases. diff --git a/doc/manual/src/store/file-system-object/content-address.md b/doc/manual/src/store/file-system-object/content-address.md new file mode 100644 index 000000000..410d7fb7c --- /dev/null +++ b/doc/manual/src/store/file-system-object/content-address.md @@ -0,0 +1,85 @@ +# Content-Addressing File System Objects + +For many operations, Nix needs to calculate [a content addresses](@docroot@/glossary.md#gloss-content-address) of [a file system object][file system object]. +Usually this is needed as part of +[content addressing store objects](../store-object/content-address.md), +since store objects always have a root file system object. +But some command-line utilities also just work on "raw" file system objects, not part of any store object. + +Every content addressing scheme Nix uses ultimately involves feeding data into a [hash function](https://en.wikipedia.org/wiki/Hash_function), and getting back an opaque fixed-size digest which is deemed a content address. +The various *methods* of content addressing thus differ in how abstract data (in this case, a file system object and its descendents) are fed into the hash function. + +## Serialising File System Objects { #serial } + +The simplest method is to serialise the entire file system object tree into a single binary string, and then hash that binary string, yielding the content address. +In this section we describe the currently-supported methods of serialising file system objects. + +### Flat { #serial-flat } + +A single file object can just be hashed by its contents. +This is not enough information to encode the fact that the file system object is a file, +but if we *already* know that the FSO is a single non-executable file by other means, it is sufficient. + +Because the hashed data is just the raw file, as is, this choice is good for compatibility with other systems. +For example, Unix commands like `sha256sum` or `sha1sum` will produce hashes for single files that match this. + +### Nix Archive (NAR) { #serial-nix-archive } + +For the other cases of [file system objects][file system object], especially directories with arbitrary descendents, we need a more complex serialisation format. +Examples of such serialisations are the ZIP and TAR file formats. +However, for our purposes these formats have two problems: + +- They do not have a canonical serialisation, meaning that given an FSO, there can +be many different serialisations. + For instance, TAR files can have variable amounts of padding between archive members; + and some archive formats leave the order of directory entries undefined. + This would be bad because we use serialisation to compute cryptographic hashes over file system objects, and for those hashes to be useful as a content address or for integrity checking, uniqueness is crucial. + Otherwise, correct hashes would report false mismatches, and the store would fail to find the content. + +- They store more information than we have in our notion of FSOs, such as time stamps. + This can cause FSOs that Nix should consider equal to hash to different values on different machines, just because the dates differ. + +- As a practical consideration, the TAR format is the only truly universal format in the Unix environment. + It has many problems, such as an inability to deal with long file names and files larger than 2^33 bytes. + Current implementations such as GNU Tar work around these limitations in various ways. + +For these reasons, Nix has its very own archive format—the Nix Archive (NAR) format, +which is carefully designed to avoid the problems described above. + +The exact specification of the Nix Archive format is in `protocols/nix-archive.md` + +## Content addressing File System Objects beyond a single serialisation pass + +Serialising the entire tree and then hashing that binary string is not the only option for content addressing, however. +Another technique is that of a [Merkle graph](https://en.wikipedia.org/wiki/Merkle_tree), where previously computed hashes are included in subsequent byte strings to be hashed. + +In particular, the Merkle graphs can match the original graph structure of file system objects: +we can first hash (serialised) child file system objects, and then hash parent objects using the hashes of their children in the serialisation (to be hashed) of the parent file system objects. + +Currently, there is one such Merkle DAG content addressing method supported. + +### Git ([experimental][xp-feature-git-hashing]) { #git } + +> **Warning** +> +> This method is part of the [`git-hashing`][xp-feature-git-hashing] experimental feature. + +Git's file system model is very close to Nix's, and so Git's content addressing method is a pretty good fit. +Just as with regular Git, files and symlinks are hashed as git "blobs", and directories are hashed as git "trees". + +However, one difference between Nix's and Git's file system model needs special treatment. +Plain files, executable files, and symlinks are not differentiated as distinctly addressable objects, but by their context: by the directory entry that refers to them. +That means so long as the root object is a directory, there is no problem: +every non-directory object is owned by a parent directory, and the entry that refers to it provides the missing information. +However, if the root object is not a directory, then we have no way of knowing which one of an executable file, non-executable file, or symlink it is supposed to be. + +In response to this, we have decided to treat a bare file as non-executable file. +This is similar to do what we do with [flat serialisation](#serial-flat), which also lacks this information. +To avoid an address collision, attempts to hash a bare executable file or symlink will result in an error (just as would happen for flat serialisation also). +Thus, Git can encode some, but not all of Nix's "File System Objects", and this sort of content-addressing is likewise partial. + +In the future, we may support a Git-like hash for such file system objects, or we may adopt another Merkle DAG format which is capable of representing all Nix file system objects. + +[file system object]: ../file-system-object.md +[store object]: ../store-object.md +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/doc/manual/src/store/store-object/content-address.md b/doc/manual/src/store/store-object/content-address.md new file mode 100644 index 000000000..02dce2836 --- /dev/null +++ b/doc/manual/src/store/store-object/content-address.md @@ -0,0 +1,95 @@ +# Content-Addressing Store Objects + +Just [like][fso-ca] [File System Objects][File System Object], +[Store Objects][Store Object] can also be [content-addressed](@docroot@/glossary.md#gloss-content-addressed), +unless they are [input-addressed](@docroot@/glossary.md#gloss-input-addressed-store-object). + +For store objects, the content address we produce will take the form of a [Store Path] rather than regular hash. +In particular, the content-addressing scheme will ensure that the digest of the store path is solely computed from the + +- file system object graph (the root one and its children, if it has any) +- references +- [store directory](../store-path.md#store-directory) +- name + +of the store object, and not any other information, which would not be an intrinsic property of that store object. + +For the full specification of the algorithms involved, see the [specification of store path digests][sp-spec]. + +[File System Object]: ../file-system-object.md +[Store Object]: ../store-object.md +[Store Path]: ../store-path.md + +## Content addressing each part of a store object + +### File System Objects + +With all currently supported store object content addressing methods, the file system object is always [content-addressed][fso-ca] first, and then that hash is incorporated into content address computation for the store object. + +### References + +With all currently supported store object content addressing methods, +other objects are referred to by their regular (string-encoded-) [store paths][Store Path]. + +Self-references however cannot be referred to by their path, because we are in the midst of describing how to compute that path! + +> The alternative would require finding as hash function fixed point, i.e. the solution to an equation in the form +> ``` +> digest = hash(..... || digest || ....) +> ``` +> which is computationally infeasible. +> As far as we know, this is equivalent to finding a hash collision. + +Instead we just have a "has self reference" boolean, which will end up affecting the digest. + +### Name and Store Directory + +These two items affect the digest in a way that is standard for store path digest computations and not specific to content-addressing. +Consult the [specification of store path digests][sp-spec] for further details. + +## Content addressing Methods + +For historical reasons, we don't support all features in all combinations. +Each currently supported method of content addressing chooses a single method of file system object hashing, and may offer some restrictions on references. +The names and store directories are unrestricted however. + +### Flat { #method-flat } + +This uses the corresponding [Flat](../file-system-object/content-address.md#serial-flat) method of file system object content addressing. + +References are not supported: store objects with flat hashing *and* references can not be created. + +### Text { #method-text } + +This also uses the corresponding [Flat](../file-system-object/content-address.md#serial-flat) method of file system object content addressing. + +References to other store objects are supported, but self references are not. + +This is the only store-object content-addressing method that is not named identically with a corresponding file system object method. +It is somewhat obscure, mainly used for "drv files" +(derivations serialized as store objects in their ["ATerm" file format](@docroot@/protocols/derivation-aterm.md)). +Prefer another method if possible. + +### Nix Archive { #method-nix-archive } + +This uses the corresponding [Nix Archive](../file-system-object/content-address.md#serial-nix-archive) method of file system object content addressing. + +References (to other store objects and self references alike) are supported so long as the hash algorithm is SHA-256, but not (neither kind) otherwise. + +### Git { #method-git } + +> **Warning** +> +> This method is part of the [`git-hashing`][xp-feature-git-hashing] experimental feature. + +This uses the corresponding [Git](../file-system-object/content-address.md#serial-git) method of file system object content addressing. + +References are not supported. + +Only SHA-1 is supported at this time. +If [SHA-256-based Git](https://git-scm.com/docs/hash-function-transition) +becomes more widespread, this restriction will be revisited. + +[fso-ca]: ../file-system-object/content-address.md +[sp-spec]: @docroot@/protocols/store-path.md +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/doc/manual/src/store/store-path.md b/doc/manual/src/store/store-path.md index b5ad0c654..beec2389b 100644 --- a/doc/manual/src/store/store-path.md +++ b/doc/manual/src/store/store-path.md @@ -1,5 +1,11 @@ # Store Path +> **Example** +> +> `/nix/store/a040m110amc4h71lds2jmr8qrkj2jhxd-git-2.38.1` +> +> A rendered store path + Nix implements references to [store objects](./index.md#store-object) as *store paths*. Think of a store path as an [opaque], [unique identifier]: @@ -37,6 +43,10 @@ A store path is rendered to a file system path as the concatenation of > store directory digest name > ``` +Exactly how the digest is calculated depends on the type of store path. +Store path digests are *supposed* to be opaque, and so for most operations, it is not necessary to know the details. +That said, the manual has a full [specification of store path digests](@docroot@/protocols/store-path.md). + ## Store Directory Every [Nix store](./index.md) has a store directory. @@ -46,7 +56,7 @@ But if the store has a file system representation, the store directory contains [file system objects]: ./file-system-object.md -This means a store path is not just derived from the referenced store object itself, but depends on the store the store object is in. +This means a store path is not just derived from the referenced store object itself, but depends on the store that the store object is in. > **Note** > diff --git a/flake.lock b/flake.lock index bb2e400c0..2ac413a69 100644 --- a/flake.lock +++ b/flake.lock @@ -3,11 +3,11 @@ "flake-compat": { "flake": false, "locked": { - "lastModified": 1673956053, - "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", "owner": "edolstra", "repo": "flake-compat", - "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", "type": "github" }, "original": { @@ -16,38 +16,100 @@ "type": "github" } }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1719994518, + "narHash": "sha256-pQMhCCHyQGRzdfAkdJ4cIWiw+JNuWsTX7f0ZYSyz0VY=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "9227223f6d922fee3c7b190b2cc238a99527bbb7", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "git-hooks-nix": { + "inputs": { + "flake-compat": [], + "gitignore": [], + "nixpkgs": [ + "nixpkgs" + ], + "nixpkgs-stable": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1721042469, + "narHash": "sha256-6FPUl7HVtvRHCCBQne7Ylp4p+dpP3P/OYuzjztZ4s70=", + "owner": "cachix", + "repo": "git-hooks.nix", + "rev": "f451c19376071a90d8c58ab1a953c6e9840527fd", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "git-hooks.nix", + "type": "github" + } + }, "libgit2": { "flake": false, "locked": { - "lastModified": 1697646580, - "narHash": "sha256-oX4Z3S9WtJlwvj0uH9HlYcWv+x1hqp8mhXl7HsLu2f0=", + "lastModified": 1715853528, + "narHash": "sha256-J2rCxTecyLbbDdsyBWn9w7r3pbKRMkI9E7RvRgAqBdY=", "owner": "libgit2", "repo": "libgit2", - "rev": "45fd9ed7ae1a9b74b957ef4f337bc3c8b3df01b5", + "rev": "36f7e21ad757a3dacc58cf7944329da6bc1d6e96", "type": "github" }, "original": { "owner": "libgit2", + "ref": "v1.8.1", "repo": "libgit2", "type": "github" } }, "nixpkgs": { "locked": { - "lastModified": 1709083642, - "narHash": "sha256-7kkJQd4rZ+vFrzWu8sTRtta5D1kBG0LSRYAfhtmMlSo=", + "lastModified": 1721548954, + "narHash": "sha256-7cCC8+Tdq1+3OPyc3+gVo9dzUNkNIQfwSDJ2HSi2u3o=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "b550fe4b4776908ac2a861124307045f8e717c8e", + "rev": "63d37ccd2d178d54e7fb691d7ec76000740ea24a", "type": "github" }, "original": { "owner": "NixOS", - "ref": "release-23.11", + "ref": "nixos-24.05", "repo": "nixpkgs", "type": "github" } }, + "nixpkgs-23-11": { + "locked": { + "lastModified": 1717159533, + "narHash": "sha256-oamiKNfr2MS6yH64rUn99mIZjc45nGJlj9eGth/3Xuw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446", + "type": "github" + } + }, "nixpkgs-regression": { "locked": { "lastModified": 1643052045, @@ -67,8 +129,11 @@ "root": { "inputs": { "flake-compat": "flake-compat", + "flake-parts": "flake-parts", + "git-hooks-nix": "git-hooks-nix", "libgit2": "libgit2", "nixpkgs": "nixpkgs", + "nixpkgs-23-11": "nixpkgs-23-11", "nixpkgs-regression": "nixpkgs-regression" } } diff --git a/flake.nix b/flake.nix index 42aaace67..9e8592e3a 100644 --- a/flake.nix +++ b/flake.nix @@ -1,18 +1,28 @@ { description = "The purely functional package manager"; - # TODO switch to nixos-23.11-small - # https://nixpk.gs/pr-tracker.html?pr=291954 - inputs.nixpkgs.url = "github:NixOS/nixpkgs/release-23.11"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; + inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446"; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; - inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; }; + inputs.libgit2 = { url = "github:libgit2/libgit2/v1.8.1"; flake = false; }; + + # dev tooling + inputs.flake-parts.url = "github:hercules-ci/flake-parts"; + inputs.git-hooks-nix.url = "github:cachix/git-hooks.nix"; + # work around https://github.com/NixOS/nix/issues/7730 + inputs.flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; + inputs.git-hooks-nix.inputs.nixpkgs.follows = "nixpkgs"; + inputs.git-hooks-nix.inputs.nixpkgs-stable.follows = "nixpkgs"; + # work around 7730 and https://github.com/NixOS/nix/issues/7807 + inputs.git-hooks-nix.inputs.flake-compat.follows = ""; + inputs.git-hooks-nix.inputs.gitignore.follows = ""; + + outputs = inputs@{ self, nixpkgs, nixpkgs-regression, libgit2, ... }: - outputs = { self, nixpkgs, nixpkgs-regression, libgit2, ... }: let inherit (nixpkgs) lib; - inherit (lib) fileset; officialRelease = false; @@ -31,14 +41,9 @@ crossSystems = [ "armv6l-unknown-linux-gnueabihf" "armv7l-unknown-linux-gnueabihf" - "x86_64-unknown-freebsd13" + "riscv64-unknown-linux-gnu" "x86_64-unknown-netbsd" - ]; - - # Nix doesn't yet build on this platform, so we put it in a - # separate list. We just use this for `devShells` and - # `nixpkgsFor`, which this depends on. - shellCrossSystems = crossSystems ++ [ + "x86_64-unknown-freebsd" "x86_64-w64-mingw32" ]; @@ -50,6 +55,18 @@ "stdenv" ]; + /** + `flatMapAttrs attrs f` applies `f` to each attribute in `attrs` and + merges the results into a single attribute set. + + This can be nested to form a build matrix where all the attributes + generated by the innermost `f` are returned as is. + (Provided that the names are unique.) + + See https://nixos.org/manual/nixpkgs/stable/index.html#function-library-lib.attrsets.concatMapAttrs + */ + flatMapAttrs = attrs: f: lib.concatMapAttrs f attrs; + forAllSystems = lib.genAttrs systems; forAllCrossSystems = lib.genAttrs crossSystems; @@ -63,6 +80,17 @@ }) stdenvs); + + # We don't apply flake-parts to the whole flake so that non-development attributes + # load without fetching any development inputs. + devFlake = inputs.flake-parts.lib.mkFlake { inherit inputs; } { + imports = [ ./maintainers/flake-module.nix ]; + systems = lib.subtractLists crossSystems systems; + perSystem = { system, ... }: { + _module.args.pkgs = nixpkgsFor.${system}.native; + }; + }; + # Memoize nixpkgs for different platforms for efficiency. nixpkgsFor = forAllSystems (system: let @@ -84,31 +112,9 @@ in { inherit stdenvs native; static = native.pkgsStatic; - cross = lib.genAttrs shellCrossSystems (crossSystem: make-pkgs crossSystem "stdenv"); + cross = forAllCrossSystems (crossSystem: make-pkgs crossSystem "stdenv"); }); - installScriptFor = tarballs: - nixpkgsFor.x86_64-linux.native.callPackage ./scripts/installer.nix { - inherit tarballs; - }; - - testNixVersions = pkgs: client: daemon: - pkgs.callPackage ./package.nix { - pname = - "nix-tests" - + lib.optionalString - (lib.versionAtLeast daemon.version "2.4pre20211005" && - lib.versionAtLeast client.version "2.4pre20211005") - "-${client.version}-against-${daemon.version}"; - - inherit fileset; - - test-client = client; - test-daemon = daemon; - - doBuild = false; - }; - binaryTarball = nix: pkgs: pkgs.callPackage ./scripts/binary-tarball.nix { inherit nix; }; @@ -120,244 +126,130 @@ { nixStable = prev.nix; - default-busybox-sandbox-shell = final.busybox.override { - useMusl = true; - enableStatic = true; - enableMinimal = true; - extraConfig = '' - CONFIG_FEATURE_FANCY_ECHO y - CONFIG_FEATURE_SH_MATH y - CONFIG_FEATURE_SH_MATH_64 y + # A new scope, so that we can use `callPackage` to inject our own interdependencies + # without "polluting" the top level "`pkgs`" attrset. + # This also has the benefit of providing us with a distinct set of packages + # we can iterate over. + nixComponents = lib.makeScope final.nixDependencies.newScope (import ./packaging/components.nix); - CONFIG_ASH y - CONFIG_ASH_OPTIMIZE_FOR_SIZE y - - CONFIG_ASH_ALIAS y - CONFIG_ASH_BASH_COMPAT y - CONFIG_ASH_CMDCMD y - CONFIG_ASH_ECHO y - CONFIG_ASH_GETOPTS y - CONFIG_ASH_INTERNAL_GLOB y - CONFIG_ASH_JOB_CONTROL y - CONFIG_ASH_PRINTF y - CONFIG_ASH_TEST y - ''; - }; - - libgit2-nix = final.libgit2.overrideAttrs (attrs: { - src = libgit2; - version = libgit2.lastModifiedDate; - cmakeFlags = attrs.cmakeFlags or [] - ++ [ "-DUSE_SSH=exec" ]; + # The dependencies are in their own scope, so that they don't have to be + # in Nixpkgs top level `pkgs` or `nixComponents`. + nixDependencies = lib.makeScope final.newScope (import ./packaging/dependencies.nix { + inherit inputs stdenv versionSuffix; + pkgs = final; }); - boehmgc-nix = (final.boehmgc.override { - enableLargeConfig = true; - }).overrideAttrs(o: { - patches = (o.patches or []) ++ [ - ./dep-patches/boehmgc-coroutine-sp-fallback.diff + nix = final.nixComponents.nix; - # https://github.com/ivmai/bdwgc/pull/586 - ./dep-patches/boehmgc-traceable_allocator-public.diff - ]; - }); - - changelog-d-nix = final.buildPackages.callPackage ./misc/changelog-d.nix { }; - - nix = - let - officialRelease = false; - versionSuffix = - if officialRelease - then "" - else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}"; - - in final.callPackage ./package.nix { - inherit - fileset - stdenv - versionSuffix - ; - officialRelease = false; - boehmgc = final.boehmgc-nix; - libgit2 = final.libgit2-nix; - busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell; - } // { - # this is a proper separate downstream package, but put - # here also for back compat reasons. - perl-bindings = final.nix-perl-bindings; - }; - - nix-perl-bindings = final.callPackage ./perl { - inherit fileset stdenv; + nix_noTests = final.nix.override { + doInstallCheck = false; + doCheck = false; }; + # See https://github.com/NixOS/nixpkgs/pull/214409 + # Remove when fixed in this flake's nixpkgs + pre-commit = + if prev.stdenv.hostPlatform.system == "i686-linux" + then (prev.pre-commit.override (o: { dotnet-sdk = ""; })).overridePythonAttrs (o: { doCheck = false; }) + else prev.pre-commit; + }; in { # A Nixpkgs overlay that overrides the 'nix' and - # 'nix.perl-bindings' packages. + # 'nix-perl-bindings' packages. overlays.default = overlayFor (p: p.stdenv); - hydraJobs = { - - # Binary package for various platforms. - build = forAllSystems (system: self.packages.${system}.nix); - - shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation); - - buildStatic = lib.genAttrs linux64BitSystems (system: self.packages.${system}.nix-static); - - buildCross = forAllCrossSystems (crossSystem: - lib.genAttrs ["x86_64-linux"] (system: self.packages.${system}."nix-${crossSystem}")); - - buildNoGc = forAllSystems (system: - self.packages.${system}.nix.override { enableGC = false; } - ); - - buildNoTests = forAllSystems (system: - self.packages.${system}.nix.override { - doCheck = false; - doInstallCheck = false; - installUnitTests = false; - } - ); - - # Toggles some settings for better coverage. Windows needs these - # library combinations, and Debian build Nix with GNU readline too. - buildReadlineNoMarkdown = forAllSystems (system: - self.packages.${system}.nix.override { - enableMarkdown = false; - readlineFlavor = "readline"; - } - ); - - # Perl bindings for various platforms. - perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nix.perl-bindings); - - # Binary tarball for various platforms, containing a Nix store - # with the closure of 'nix' package, and the second half of - # the installation script. - binaryTarball = forAllSystems (system: binaryTarball nixpkgsFor.${system}.native.nix nixpkgsFor.${system}.native); - - binaryTarballCross = lib.genAttrs ["x86_64-linux"] (system: - forAllCrossSystems (crossSystem: - binaryTarball - self.packages.${system}."nix-${crossSystem}" - nixpkgsFor.${system}.cross.${crossSystem})); - - # The first half of the installation script. This is uploaded - # to https://nixos.org/nix/install. It downloads the binary - # tarball for the user's system and calls the second half of the - # installation script. - installerScript = installScriptFor [ - # Native - self.hydraJobs.binaryTarball."x86_64-linux" - self.hydraJobs.binaryTarball."i686-linux" - self.hydraJobs.binaryTarball."aarch64-linux" - self.hydraJobs.binaryTarball."x86_64-darwin" - self.hydraJobs.binaryTarball."aarch64-darwin" - # Cross - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" - ]; - installerScriptForGHA = installScriptFor [ - # Native - self.hydraJobs.binaryTarball."x86_64-linux" - self.hydraJobs.binaryTarball."x86_64-darwin" - # Cross - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" - ]; - - # docker image with Nix inside - dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); - - # Line coverage analysis. - coverage = nixpkgsFor.x86_64-linux.native.nix.override { - pname = "nix-coverage"; - withCoverageChecks = true; - }; - - # API docs for Nix's unstable internal C++ interfaces. - internal-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ./package.nix { - inherit fileset; - doBuild = false; - enableInternalAPIDocs = true; - }; - - # System tests. - tests = import ./tests/nixos { inherit lib nixpkgs nixpkgsFor; } // { - - # Make sure that nix-env still produces the exact same result - # on a particular version of Nixpkgs. - evalNixpkgs = - let - inherit (nixpkgsFor.x86_64-linux.native) runCommand nix; - in - runCommand "eval-nixos" { buildInputs = [ nix ]; } - '' - type -p nix-env - # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593. - time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages - [[ $(sha1sum < packages | cut -c1-40) = ff451c521e61e4fe72bdbe2d0ca5d1809affa733 ]] - mkdir $out - ''; - - nixpkgsLibTests = - forAllSystems (system: - import (nixpkgs + "/lib/tests/release.nix") - { pkgs = nixpkgsFor.${system}.native; - nixVersions = [ self.packages.${system}.nix ]; - } - ); - }; - - metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" { - pkgs = nixpkgsFor.x86_64-linux.native; - nixpkgs = nixpkgs-regression; - }; - - installTests = forAllSystems (system: - let pkgs = nixpkgsFor.${system}.native; in - pkgs.runCommand "install-tests" { - againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix; - againstCurrentUnstable = - # FIXME: temporarily disable this on macOS because of #3605. - if system == "x86_64-linux" - then testNixVersions pkgs pkgs.nix pkgs.nixUnstable - else null; - # Disabled because the latest stable version doesn't handle - # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work - # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable; - } "touch $out"); - - installerTests = import ./tests/installer { - binaryTarballs = self.hydraJobs.binaryTarball; - inherit nixpkgsFor; - }; - + hydraJobs = import ./packaging/hydra.nix { + inherit + inputs + binaryTarball + forAllCrossSystems + forAllSystems + lib + linux64BitSystems + nixpkgsFor + self + ; }; checks = forAllSystems (system: { binaryTarball = self.hydraJobs.binaryTarball.${system}; - perlBindings = self.hydraJobs.perlBindings.${system}; installTests = self.hydraJobs.installTests.${system}; nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system}; rl-next = let pkgs = nixpkgsFor.${system}.native; in pkgs.buildPackages.runCommand "test-rl-next-release-notes" { } '' - LANG=C.UTF-8 ${pkgs.changelog-d-nix}/bin/changelog-d ${./doc/manual/rl-next} >$out + LANG=C.UTF-8 ${pkgs.changelog-d}/bin/changelog-d ${./doc/manual/rl-next} >$out ''; + repl-completion = nixpkgsFor.${system}.native.callPackage ./tests/repl-completion.nix { }; } // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) { dockerImage = self.hydraJobs.dockerImage.${system}; - }); + } // (lib.optionalAttrs (!(builtins.elem system linux32BitSystems))) { + # Some perl dependencies are broken on i686-linux. + # Since the support is only best-effort there, disable the perl + # bindings - packages = forAllSystems (system: rec { - inherit (nixpkgsFor.${system}.native) nix changelog-d-nix; - default = nix; - } // (lib.optionalAttrs (builtins.elem system linux64BitSystems) { - nix-static = nixpkgsFor.${system}.static.nix; + # Temporarily disabled because GitHub Actions OOM issues. Once + # the old build system is gone and we are back to one build + # system, we should reenable this. + #perlBindings = self.hydraJobs.perlBindings.${system}; + } + # Add "passthru" tests + // flatMapAttrs ({ + "" = nixpkgsFor.${system}.native; + } // lib.optionalAttrs (! nixpkgsFor.${system}.native.stdenv.hostPlatform.isDarwin) { + # TODO: enable static builds for darwin, blocked on: + # https://github.com/NixOS/nixpkgs/issues/320448 + "static-" = nixpkgsFor.${system}.static; + }) + (nixpkgsPrefix: nixpkgs: + flatMapAttrs nixpkgs.nixComponents + (pkgName: pkg: + flatMapAttrs pkg.tests or {} + (testName: test: { + "${nixpkgsPrefix}${pkgName}-${testName}" = test; + }) + ) + ) + // devFlake.checks.${system} or {} + ); + + packages = forAllSystems (system: + { # Here we put attributes that map 1:1 into packages., ie + # for which we don't apply the full build matrix such as cross or static. + inherit (nixpkgsFor.${system}.native) + changelog-d; + default = self.packages.${system}.nix; + nix-internal-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-internal-api-docs; + nix-external-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-external-api-docs; + } + # We need to flatten recursive attribute sets of derivations to pass `flake check`. + // flatMapAttrs + { # Components we'll iterate over in the upcoming lambda + "nix" = { }; + # Temporarily disabled because GitHub Actions OOM issues. Once + # the old build system is gone and we are back to one build + # system, we should reenable these. + #"nix-util" = { }; + #"nix-store" = { }; + #"nix-fetchers" = { }; + } + (pkgName: {}: { + # These attributes go right into `packages.`. + "${pkgName}" = nixpkgsFor.${system}.native.nixComponents.${pkgName}; + "${pkgName}-static" = nixpkgsFor.${system}.static.nixComponents.${pkgName}; + } + // flatMapAttrs (lib.genAttrs crossSystems (_: { })) (crossSystem: {}: { + # These attributes go right into `packages.`. + "${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName}; + }) + // flatMapAttrs (lib.genAttrs stdenvs (_: { })) (stdenvName: {}: { + # These attributes go right into `packages.`. + "${pkgName}-${stdenvName}" = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nixComponents.${pkgName}; + }) + ) + // lib.optionalAttrs (builtins.elem system linux64BitSystems) { dockerImage = let pkgs = nixpkgsFor.${system}.native; @@ -372,21 +264,27 @@ ln -s ${image} $image echo "file binary-dist $image" >> $out/nix-support/hydra-build-products ''; - } // builtins.listToAttrs (map - (crossSystem: { - name = "nix-${crossSystem}"; - value = nixpkgsFor.${system}.cross.${crossSystem}.nix; - }) - crossSystems) - // builtins.listToAttrs (map - (stdenvName: { - name = "nix-${stdenvName}"; - value = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nix; - }) - stdenvs))); + }); devShells = let - makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; forDevShell = true; }).overrideAttrs (attrs: { + makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; forDevShell = true; }).overrideAttrs (attrs: + let + modular = devFlake.getSystem stdenv.buildPlatform.system; + transformFlag = prefix: flag: + assert builtins.isString flag; + let + rest = builtins.substring 2 (builtins.stringLength flag) flag; + in + "-D${prefix}:${rest}"; + havePerl = stdenv.buildPlatform == stdenv.hostPlatform && stdenv.hostPlatform.isUnix; + ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags; + in { + pname = "shell-for-" + attrs.pname; + + # Remove the version suffix to avoid unnecessary attempts to substitute in nix develop + version = lib.fileContents ./.version; + name = attrs.pname; + installFlags = "sysconfdir=$(out)/etc"; shellHook = '' PATH=$prefix/bin:$PATH @@ -397,11 +295,62 @@ XDG_DATA_DIRS+=:$out/share ''; + # We use this shell with the local checkout, not unpackPhase. + src = null; + + env = { + # Needed for Meson to find Boost. + # https://github.com/NixOS/nixpkgs/issues/86131. + BOOST_INCLUDEDIR = "${lib.getDev pkgs.nixDependencies.boost}/include"; + BOOST_LIBRARYDIR = "${lib.getLib pkgs.nixDependencies.boost}/lib"; + # For `make format`, to work without installing pre-commit + _NIX_PRE_COMMIT_HOOKS_CONFIG = + "${(pkgs.formats.yaml { }).generate "pre-commit-config.yaml" modular.pre-commit.settings.rawConfig}"; + }; + + mesonFlags = + map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents.nix-util.mesonFlags) + ++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents.nix-store.mesonFlags) + ++ map (transformFlag "libfetchers") (ignoreCrossFile pkgs.nixComponents.nix-fetchers.mesonFlags) + ++ lib.optionals havePerl (map (transformFlag "perl") (ignoreCrossFile pkgs.nixComponents.nix-perl-bindings.mesonFlags)) + ++ map (transformFlag "libexpr") (ignoreCrossFile pkgs.nixComponents.nix-expr.mesonFlags) + ++ map (transformFlag "libcmd") (ignoreCrossFile pkgs.nixComponents.nix-cmd.mesonFlags) + ; + nativeBuildInputs = attrs.nativeBuildInputs or [] + ++ pkgs.nixComponents.nix-util.nativeBuildInputs + ++ pkgs.nixComponents.nix-store.nativeBuildInputs + ++ pkgs.nixComponents.nix-fetchers.nativeBuildInputs + ++ lib.optionals havePerl pkgs.nixComponents.nix-perl-bindings.nativeBuildInputs + ++ pkgs.nixComponents.nix-internal-api-docs.nativeBuildInputs + ++ pkgs.nixComponents.nix-external-api-docs.nativeBuildInputs + ++ lib.optional + (!stdenv.buildPlatform.canExecute stdenv.hostPlatform + # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479 + && !(stdenv.hostPlatform.isWindows && stdenv.buildPlatform.isDarwin) + && stdenv.hostPlatform.emulatorAvailable pkgs.buildPackages + && lib.meta.availableOn stdenv.buildPlatform (stdenv.hostPlatform.emulator pkgs.buildPackages)) + pkgs.buildPackages.mesonEmulatorHook + ++ [ + pkgs.buildPackages.cmake + pkgs.buildPackages.shellcheck + pkgs.buildPackages.changelog-d + modular.pre-commit.settings.package + (pkgs.writeScriptBin "pre-commit-hooks-install" + modular.pre-commit.settings.installationScript) + ] # TODO: Remove the darwin check once # https://github.com/NixOS/nixpkgs/pull/291814 is available ++ lib.optional (stdenv.cc.isClang && !stdenv.buildPlatform.isDarwin) pkgs.buildPackages.bear ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) pkgs.buildPackages.clang-tools; + + buildInputs = attrs.buildInputs or [] + ++ [ + pkgs.gtest + pkgs.rapidcheck + ] + ++ lib.optional havePerl pkgs.perl + ; }); in forAllSystems (system: @@ -413,8 +362,8 @@ in (makeShells "native" nixpkgsFor.${system}.native) // (lib.optionalAttrs (!nixpkgsFor.${system}.native.stdenv.isDarwin) - (makeShells "static" nixpkgsFor.${system}.static)) // - (lib.genAttrs shellCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv)) // + (makeShells "static" nixpkgsFor.${system}.static) // + (forAllCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv))) // { default = self.devShells.${system}.native-stdenvPackages; } diff --git a/local.mk b/local.mk index 3f3abb9f0..b27c7031e 100644 --- a/local.mk +++ b/local.mk @@ -2,9 +2,14 @@ GLOBAL_CXXFLAGS += -Wno-deprecated-declarations -Werror=switch # Allow switch-enum to be overridden for files that do not support it, usually because of dependency headers. ERROR_SWITCH_ENUM = -Werror=switch-enum -$(foreach i, config.h $(wildcard src/lib*/*.hh), \ +$(foreach i, config.h $(wildcard src/lib*/*.hh) $(filter-out %_internal.h, $(wildcard src/lib*c/*.h)), \ $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) +ifdef HOST_UNIX + $(foreach i, $(wildcard src/lib*/unix/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) +endif + $(GCH): src/libutil/util.hh config.h -GCH_CXXFLAGS = -I src/libutil +GCH_CXXFLAGS = $(INCLUDE_libutil) diff --git a/m4/gcc_bug_80431.m4 b/m4/gcc_bug_80431.m4 index e42f01956..cdc4ddb40 100644 --- a/m4/gcc_bug_80431.m4 +++ b/m4/gcc_bug_80431.m4 @@ -46,11 +46,13 @@ AC_DEFUN([ENSURE_NO_GCC_BUG_80431], ]])], [status_80431=0], [status_80431=$?], - [ - # Assume we're bug-free when cross-compiling - ]) + [status_80431='']) AC_LANG_POP(C++) AS_CASE([$status_80431], + [''],[ + AC_MSG_RESULT(cannot check because cross compiling) + AC_MSG_NOTICE(assume we are bug free) + ], [0],[ AC_MSG_RESULT(yes) ], diff --git a/maintainers/README.md b/maintainers/README.md index fa321c7c0..b92833497 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -30,17 +30,18 @@ We aim to achieve this by improving the contributor experience and attracting mo ## Members - Eelco Dolstra (@edolstra) – Team lead -- Théophane Hufschmitt (@thufschmitt) - Valentin Gagarin (@fricklerhandwerk) - Thomas Bereknyei (@tomberek) - Robert Hensing (@roberth) - John Ericson (@Ericson2314) +The team is on Github as [@NixOS/nix-team](https://github.com/orgs/NixOS/teams/nix-team). + ## Meeting protocol -The team meets twice a week: +The team meets twice a week (times are denoted in the [Europe/Amsterdam](https://en.m.wikipedia.org/wiki/Time_in_the_Netherlands) time zone): -- Discussion meeting: [Fridays 13:00-14:00 CET](https://calendar.google.com/calendar/event?eid=MHNtOGVuNWtrZXNpZHR2bW1sM3QyN2ZjaGNfMjAyMjExMjVUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) +- Discussion meeting: [Wednesday 21:00-22:00 Europe/Amsterdam](https://www.google.com/calendar/event?eid=ZG5rZzNyajRjajducGV2NGY5aGkzYWIwdnJfMjAyNDA1MDhUMTkwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) 1. Triage issues and pull requests from the [No Status](#no-status) column (30 min) 2. Discuss issues and pull requests from the [To discuss](#to-discuss) column (30 min). @@ -49,15 +50,19 @@ The team meets twice a week: - mark it as draft if it is blocked on the contributor - escalate it back to the team by moving it to To discuss, and leaving a comment as to why the issue needs to be discussed again. -- Work meeting: [Mondays 13:00-15:00 CET](https://calendar.google.com/calendar/event?eid=NTM1MG1wNGJnOGpmOTZhYms3bTB1bnY5cWxfMjAyMjExMjFUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) +- Work meeting: [Mondays 14:00-16:00 Europe/Amsterdam](https://www.google.com/calendar/event?eid=Ym52NDdzYnRic2NzcDcybjZiNDhpNzhpa3NfMjAyNDA1MTNUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) 1. Code review on pull requests from [In review](#in-review). 2. Other chores and tasks. Meeting notes are collected on a [collaborative scratchpad](https://pad.lassul.us/Cv7FpYx-Ri-4VjUykQOLAw). -Notes on issues and pull requests are posted as comments and linked from the meeting notes, so they are easy to find from both places. +Notes on issues and pull requests are posted as comments and linked from the meeting notes, so they are can be found from both places. [All meeting notes](https://discourse.nixos.org/search?expanded=true&q=Nix%20team%20meeting%20minutes%20%23%20%23dev%3Anix%20in%3Atitle%20order%3Alatest_topic) are published on Discourse under the [Nix category](https://discourse.nixos.org/c/dev/nix/50). +Team meetings are generally open to anyone interested. +We can make exceptions to discuss sensitive issues, such as security incidents or people matters. +Contact any team member to get a calendar invite for reminders and updates. + ## Project board protocol The team uses a [GitHub project board](https://github.com/orgs/NixOS/projects/19/views/1) for tracking its work. diff --git a/maintainers/data/release-credits-email-to-handle.json b/maintainers/data/release-credits-email-to-handle.json new file mode 100644 index 000000000..cddc1a6e7 --- /dev/null +++ b/maintainers/data/release-credits-email-to-handle.json @@ -0,0 +1,52 @@ +{ + "bogus": "bogus", + "edolstra@gmail.com": "edolstra", + "roberth@users.noreply.github.com": "roberth", + "toscano.pino@tiscali.it": "pinotree", + "valentin@gagarin.work": "fricklerhandwerk", + "mr.trubach@icloud.com": "tie", + "robert@roberthensing.nl": "roberth", + "lix@jade.fyi": "lf-", + "cole.e.helbling@outlook.com": "cole-h", + "joerg@thalheim.io": "Mic92", + "John.Ericson@Obsidian.Systems": "Ericson2314", + "ryan.hendrickson@alum.mit.edu": "rhendric", + "67135060+poweredbypie@users.noreply.github.com": "poweredbypie", + "detroyejr@outlook.com": "detroyejr", + "silvan.mosberger@tweag.io": "infinisil", + "vcs@emily.moe": "emilazy", + "farid.m.zakaria@gmail.com": "fzakaria", + "22859658+RTUnreal@users.noreply.github.com": "RTUnreal", + "me@las.rs": "L-as", + "philip.taron@gmail.com": "philiptaron", + "root@goldstein.rs": "GoldsteinE", + "tomberek@users.noreply.github.com": "tomberek", + "lexi.mattick@neuralink.com": "kognise", + "andrew@johnandrewmarshall.com": "amarshall", + "contact@romain-neil.fr": "romain-neil", + "Mic92@users.noreply.github.com": "Mic92", + "valentin.gagarin@tweag.io": "fricklerhandwerk", + "siddhantk232@gmail.com": "siddhantk232", + "kn@openbsd.org": "klemensn", + "slyich@gmail.com": "trofi", + "theophane.hufschmitt@tweag.io": "thufschmitt", + "alicebob@lijzij.de": "alicebob", + "winter@winter.cafe": "winterqt", + "brian@brianmckenna.org": "puffnfresh", + "git@haenoe.party": "haenoe", + "peshogo@gmail.com": "pineapplehunter", + "poweredbypie@users.noreply.github.com": "poweredbypie", + "arthur200126@gmail.com": "Artoria2e5", + "tomberek@gmail.com": "tomberek", + "jaredbaur@fastmail.com": "jmbaur", + "andreas@rammhold.de": "andir", + "hamirmahal@gmail.com": "hamirmahal", + "git@JohnEricson.me": "Ericson2314", + "8763518+SkamDart@users.noreply.github.com": "SkamDart", + "kirillrdy@gmail.com": "kirillrdy", + "pennae@lix.systems": "pennae", + "delroth@gmail.com": "delroth", + "enno@nerdworks.de": "elohmeier", + "mjbauer95@gmail.com": "matthewbauer", + "MostAwesomeDude@gmail.com": "MostAwesomeDude" +} \ No newline at end of file diff --git a/maintainers/data/release-credits-handle-to-name.json b/maintainers/data/release-credits-handle-to-name.json new file mode 100644 index 000000000..abf9ed05b --- /dev/null +++ b/maintainers/data/release-credits-handle-to-name.json @@ -0,0 +1,45 @@ +{ + "fzakaria": "Farid Zakaria", + "kognise": "Lexi Mattick", + "L-as": "Las Safin", + "haenoe": "HaeNoe", + "andir": "Andreas Rammhold", + "matthewbauer": "Matthew Bauer", + "emilazy": "Emily", + "pineapplehunter": "Shogo Takata", + "RTUnreal": null, + "jmbaur": "Jared Baur", + "Ericson2314": "John Ericson", + "pinotree": "Pino Toscano", + "tie": "Ivan Trubach", + "poweredbypie": null, + "fricklerhandwerk": "Valentin Gagarin", + "Mic92": "J\u00f6rg Thalheim", + "alicebob": "Harmen", + "elohmeier": "Enno Richter", + "delroth": "Pierre Bourdon", + "kirillrdy": null, + "thufschmitt": "Th\u00e9ophane Hufschmitt", + "detroyejr": "Jonathan De Troye", + "klemensn": "Klemens Nanni", + "tomberek": null, + "rhendric": "Ryan Hendrickson", + "philiptaron": "Philip Taron", + "puffnfresh": "Brian McKenna", + "lf-": "jade", + "romain-neil": "Romain Neil", + "hamirmahal": "Hamir Mahal", + "edolstra": "Eelco Dolstra", + "Artoria2e5": "Mingye Wang", + "SkamDart": "Cameron", + "roberth": "Robert Hensing", + "amarshall": "Andrew Marshall", + "trofi": "Sergei Trofimovich", + "cole-h": "Cole Helbling", + "infinisil": "Silvan Mosberger", + "siddhantk232": "Siddhant Kumar", + "winterqt": "Winter", + "GoldsteinE": "Max \u201cGoldstein\u201d Siling", + "pennae": null, + "MostAwesomeDude": "Corbin Simpson" +} \ No newline at end of file diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix new file mode 100644 index 000000000..be91df536 --- /dev/null +++ b/maintainers/flake-module.nix @@ -0,0 +1,677 @@ +{ lib, getSystem, inputs, ... }: + +{ + imports = [ + inputs.git-hooks-nix.flakeModule + ]; + + perSystem = { config, pkgs, ... }: { + + # https://flake.parts/options/pre-commit-hooks-nix.html#options + pre-commit.settings = { + hooks = { + clang-format = { + enable = true; + excludes = [ + # We don't want to format test data + # ''tests/(?!nixos/).*\.nix'' + ''^tests/unit/[^/]*/data/.*$'' + + # Don't format vendored code + ''^doc/manual/redirects\.js$'' + ''^doc/manual/theme/highlight\.js$'' + + # We haven't applied formatting to these files yet + ''^doc/manual/redirects\.js$'' + ''^doc/manual/theme/highlight\.js$'' + ''^precompiled-headers\.h$'' + ''^src/build-remote/build-remote\.cc$'' + ''^src/libcmd/built-path\.cc$'' + ''^src/libcmd/built-path\.hh$'' + ''^src/libcmd/command\.cc$'' + ''^src/libcmd/command\.hh$'' + ''^src/libcmd/common-eval-args\.cc$'' + ''^src/libcmd/common-eval-args\.hh$'' + ''^src/libcmd/editor-for\.cc$'' + ''^src/libcmd/installable-attr-path\.cc$'' + ''^src/libcmd/installable-attr-path\.hh$'' + ''^src/libcmd/installable-derived-path\.cc$'' + ''^src/libcmd/installable-derived-path\.hh$'' + ''^src/libcmd/installable-flake\.cc$'' + ''^src/libcmd/installable-flake\.hh$'' + ''^src/libcmd/installable-value\.cc$'' + ''^src/libcmd/installable-value\.hh$'' + ''^src/libcmd/installables\.cc$'' + ''^src/libcmd/installables\.hh$'' + ''^src/libcmd/legacy\.hh$'' + ''^src/libcmd/markdown\.cc$'' + ''^src/libcmd/misc-store-flags\.cc$'' + ''^src/libcmd/repl-interacter\.cc$'' + ''^src/libcmd/repl-interacter\.hh$'' + ''^src/libcmd/repl\.cc$'' + ''^src/libcmd/repl\.hh$'' + ''^src/libexpr-c/nix_api_expr\.cc$'' + ''^src/libexpr-c/nix_api_external\.cc$'' + ''^src/libexpr/attr-path\.cc$'' + ''^src/libexpr/attr-path\.hh$'' + ''^src/libexpr/attr-set\.cc$'' + ''^src/libexpr/attr-set\.hh$'' + ''^src/libexpr/eval-cache\.cc$'' + ''^src/libexpr/eval-cache\.hh$'' + ''^src/libexpr/eval-error\.cc$'' + ''^src/libexpr/eval-inline\.hh$'' + ''^src/libexpr/eval-settings\.cc$'' + ''^src/libexpr/eval-settings\.hh$'' + ''^src/libexpr/eval\.cc$'' + ''^src/libexpr/eval\.hh$'' + ''^src/libexpr/function-trace\.cc$'' + ''^src/libexpr/gc-small-vector\.hh$'' + ''^src/libexpr/get-drvs\.cc$'' + ''^src/libexpr/get-drvs\.hh$'' + ''^src/libexpr/json-to-value\.cc$'' + ''^src/libexpr/nixexpr\.cc$'' + ''^src/libexpr/nixexpr\.hh$'' + ''^src/libexpr/parser-state\.hh$'' + ''^src/libexpr/pos-table\.hh$'' + ''^src/libexpr/primops\.cc$'' + ''^src/libexpr/primops\.hh$'' + ''^src/libexpr/primops/context\.cc$'' + ''^src/libexpr/primops/fetchClosure\.cc$'' + ''^src/libexpr/primops/fetchMercurial\.cc$'' + ''^src/libexpr/primops/fetchTree\.cc$'' + ''^src/libexpr/primops/fromTOML\.cc$'' + ''^src/libexpr/print-ambiguous\.cc$'' + ''^src/libexpr/print-ambiguous\.hh$'' + ''^src/libexpr/print-options\.hh$'' + ''^src/libexpr/print\.cc$'' + ''^src/libexpr/print\.hh$'' + ''^src/libexpr/search-path\.cc$'' + ''^src/libexpr/symbol-table\.hh$'' + ''^src/libexpr/value-to-json\.cc$'' + ''^src/libexpr/value-to-json\.hh$'' + ''^src/libexpr/value-to-xml\.cc$'' + ''^src/libexpr/value-to-xml\.hh$'' + ''^src/libexpr/value\.hh$'' + ''^src/libexpr/value/context\.cc$'' + ''^src/libexpr/value/context\.hh$'' + ''^src/libfetchers/attrs\.cc$'' + ''^src/libfetchers/cache\.cc$'' + ''^src/libfetchers/cache\.hh$'' + ''^src/libfetchers/fetch-settings\.cc$'' + ''^src/libfetchers/fetch-settings\.hh$'' + ''^src/libfetchers/fetch-to-store\.cc$'' + ''^src/libfetchers/fetchers\.cc$'' + ''^src/libfetchers/fetchers\.hh$'' + ''^src/libfetchers/filtering-source-accessor\.cc$'' + ''^src/libfetchers/filtering-source-accessor\.hh$'' + ''^src/libfetchers/fs-source-accessor\.cc$'' + ''^src/libfetchers/fs-source-accessor\.hh$'' + ''^src/libfetchers/git-utils\.cc$'' + ''^src/libfetchers/git-utils\.hh$'' + ''^src/libfetchers/github\.cc$'' + ''^src/libfetchers/indirect\.cc$'' + ''^src/libfetchers/memory-source-accessor\.cc$'' + ''^src/libfetchers/path\.cc$'' + ''^src/libfetchers/registry\.cc$'' + ''^src/libfetchers/registry\.hh$'' + ''^src/libfetchers/tarball\.cc$'' + ''^src/libfetchers/tarball\.hh$'' + ''^src/libfetchers/git\.cc$'' + ''^src/libfetchers/mercurial\.cc$'' + ''^src/libflake/flake/config\.cc$'' + ''^src/libflake/flake/flake\.cc$'' + ''^src/libflake/flake/flake\.hh$'' + ''^src/libflake/flake/flakeref\.cc$'' + ''^src/libflake/flake/flakeref\.hh$'' + ''^src/libflake/flake/lockfile\.cc$'' + ''^src/libflake/flake/lockfile\.hh$'' + ''^src/libflake/flake/url-name\.cc$'' + ''^src/libmain/common-args\.cc$'' + ''^src/libmain/common-args\.hh$'' + ''^src/libmain/loggers\.cc$'' + ''^src/libmain/loggers\.hh$'' + ''^src/libmain/progress-bar\.cc$'' + ''^src/libmain/shared\.cc$'' + ''^src/libmain/shared\.hh$'' + ''^src/libmain/unix/stack\.cc$'' + ''^src/libstore/binary-cache-store\.cc$'' + ''^src/libstore/binary-cache-store\.hh$'' + ''^src/libstore/build-result\.hh$'' + ''^src/libstore/builtins\.hh$'' + ''^src/libstore/builtins/buildenv\.cc$'' + ''^src/libstore/builtins/buildenv\.hh$'' + ''^src/libstore/common-protocol-impl\.hh$'' + ''^src/libstore/common-protocol\.cc$'' + ''^src/libstore/common-protocol\.hh$'' + ''^src/libstore/common-ssh-store-config\.hh$'' + ''^src/libstore/content-address\.cc$'' + ''^src/libstore/content-address\.hh$'' + ''^src/libstore/daemon\.cc$'' + ''^src/libstore/daemon\.hh$'' + ''^src/libstore/derivations\.cc$'' + ''^src/libstore/derivations\.hh$'' + ''^src/libstore/derived-path-map\.cc$'' + ''^src/libstore/derived-path-map\.hh$'' + ''^src/libstore/derived-path\.cc$'' + ''^src/libstore/derived-path\.hh$'' + ''^src/libstore/downstream-placeholder\.cc$'' + ''^src/libstore/downstream-placeholder\.hh$'' + ''^src/libstore/dummy-store\.cc$'' + ''^src/libstore/export-import\.cc$'' + ''^src/libstore/filetransfer\.cc$'' + ''^src/libstore/filetransfer\.hh$'' + ''^src/libstore/gc-store\.hh$'' + ''^src/libstore/globals\.cc$'' + ''^src/libstore/globals\.hh$'' + ''^src/libstore/http-binary-cache-store\.cc$'' + ''^src/libstore/legacy-ssh-store\.cc$'' + ''^src/libstore/legacy-ssh-store\.hh$'' + ''^src/libstore/length-prefixed-protocol-helper\.hh$'' + ''^src/libstore/linux/personality\.cc$'' + ''^src/libstore/linux/personality\.hh$'' + ''^src/libstore/local-binary-cache-store\.cc$'' + ''^src/libstore/local-fs-store\.cc$'' + ''^src/libstore/local-fs-store\.hh$'' + ''^src/libstore/log-store\.cc$'' + ''^src/libstore/log-store\.hh$'' + ''^src/libstore/machines\.cc$'' + ''^src/libstore/machines\.hh$'' + ''^src/libstore/make-content-addressed\.cc$'' + ''^src/libstore/make-content-addressed\.hh$'' + ''^src/libstore/misc\.cc$'' + ''^src/libstore/names\.cc$'' + ''^src/libstore/names\.hh$'' + ''^src/libstore/nar-accessor\.cc$'' + ''^src/libstore/nar-accessor\.hh$'' + ''^src/libstore/nar-info-disk-cache\.cc$'' + ''^src/libstore/nar-info-disk-cache\.hh$'' + ''^src/libstore/nar-info\.cc$'' + ''^src/libstore/nar-info\.hh$'' + ''^src/libstore/outputs-spec\.cc$'' + ''^src/libstore/outputs-spec\.hh$'' + ''^src/libstore/parsed-derivations\.cc$'' + ''^src/libstore/path-info\.cc$'' + ''^src/libstore/path-info\.hh$'' + ''^src/libstore/path-references\.cc$'' + ''^src/libstore/path-regex\.hh$'' + ''^src/libstore/path-with-outputs\.cc$'' + ''^src/libstore/path\.cc$'' + ''^src/libstore/path\.hh$'' + ''^src/libstore/pathlocks\.cc$'' + ''^src/libstore/pathlocks\.hh$'' + ''^src/libstore/profiles\.cc$'' + ''^src/libstore/profiles\.hh$'' + ''^src/libstore/realisation\.cc$'' + ''^src/libstore/realisation\.hh$'' + ''^src/libstore/remote-fs-accessor\.cc$'' + ''^src/libstore/remote-fs-accessor\.hh$'' + ''^src/libstore/remote-store-connection\.hh$'' + ''^src/libstore/remote-store\.cc$'' + ''^src/libstore/remote-store\.hh$'' + ''^src/libstore/s3-binary-cache-store\.cc$'' + ''^src/libstore/s3\.hh$'' + ''^src/libstore/serve-protocol-impl\.cc$'' + ''^src/libstore/serve-protocol-impl\.hh$'' + ''^src/libstore/serve-protocol\.cc$'' + ''^src/libstore/serve-protocol\.hh$'' + ''^src/libstore/sqlite\.cc$'' + ''^src/libstore/sqlite\.hh$'' + ''^src/libstore/ssh-store\.cc$'' + ''^src/libstore/ssh\.cc$'' + ''^src/libstore/ssh\.hh$'' + ''^src/libstore/store-api\.cc$'' + ''^src/libstore/store-api\.hh$'' + ''^src/libstore/store-dir-config\.hh$'' + ''^src/libstore/build/derivation-goal\.cc$'' + ''^src/libstore/build/derivation-goal\.hh$'' + ''^src/libstore/build/drv-output-substitution-goal\.cc$'' + ''^src/libstore/build/drv-output-substitution-goal\.hh$'' + ''^src/libstore/build/entry-points\.cc$'' + ''^src/libstore/build/goal\.cc$'' + ''^src/libstore/build/goal\.hh$'' + ''^src/libstore/unix/build/hook-instance\.cc$'' + ''^src/libstore/unix/build/local-derivation-goal\.cc$'' + ''^src/libstore/unix/build/local-derivation-goal\.hh$'' + ''^src/libstore/build/substitution-goal\.cc$'' + ''^src/libstore/build/substitution-goal\.hh$'' + ''^src/libstore/build/worker\.cc$'' + ''^src/libstore/build/worker\.hh$'' + ''^src/libstore/builtins/fetchurl\.cc$'' + ''^src/libstore/builtins/unpack-channel\.cc$'' + ''^src/libstore/gc\.cc$'' + ''^src/libstore/local-overlay-store\.cc$'' + ''^src/libstore/local-overlay-store\.hh$'' + ''^src/libstore/local-store\.cc$'' + ''^src/libstore/local-store\.hh$'' + ''^src/libstore/unix/user-lock\.cc$'' + ''^src/libstore/unix/user-lock\.hh$'' + ''^src/libstore/optimise-store\.cc$'' + ''^src/libstore/unix/pathlocks\.cc$'' + ''^src/libstore/posix-fs-canonicalise\.cc$'' + ''^src/libstore/posix-fs-canonicalise\.hh$'' + ''^src/libstore/uds-remote-store\.cc$'' + ''^src/libstore/uds-remote-store\.hh$'' + ''^src/libstore/windows/build\.cc$'' + ''^src/libstore/worker-protocol-impl\.hh$'' + ''^src/libstore/worker-protocol\.cc$'' + ''^src/libstore/worker-protocol\.hh$'' + ''^src/libutil-c/nix_api_util_internal\.h$'' + ''^src/libutil/archive\.cc$'' + ''^src/libutil/archive\.hh$'' + ''^src/libutil/args\.cc$'' + ''^src/libutil/args\.hh$'' + ''^src/libutil/args/root\.hh$'' + ''^src/libutil/callback\.hh$'' + ''^src/libutil/canon-path\.cc$'' + ''^src/libutil/canon-path\.hh$'' + ''^src/libutil/chunked-vector\.hh$'' + ''^src/libutil/closure\.hh$'' + ''^src/libutil/comparator\.hh$'' + ''^src/libutil/compute-levels\.cc$'' + ''^src/libutil/config-impl\.hh$'' + ''^src/libutil/config\.cc$'' + ''^src/libutil/config\.hh$'' + ''^src/libutil/current-process\.cc$'' + ''^src/libutil/current-process\.hh$'' + ''^src/libutil/english\.cc$'' + ''^src/libutil/english\.hh$'' + ''^src/libutil/environment-variables\.cc$'' + ''^src/libutil/error\.cc$'' + ''^src/libutil/error\.hh$'' + ''^src/libutil/exit\.hh$'' + ''^src/libutil/experimental-features\.cc$'' + ''^src/libutil/experimental-features\.hh$'' + ''^src/libutil/file-content-address\.cc$'' + ''^src/libutil/file-content-address\.hh$'' + ''^src/libutil/file-descriptor\.cc$'' + ''^src/libutil/file-descriptor\.hh$'' + ''^src/libutil/file-path-impl\.hh$'' + ''^src/libutil/file-path\.hh$'' + ''^src/libutil/file-system\.cc$'' + ''^src/libutil/file-system\.hh$'' + ''^src/libutil/finally\.hh$'' + ''^src/libutil/fmt\.hh$'' + ''^src/libutil/fs-sink\.cc$'' + ''^src/libutil/fs-sink\.hh$'' + ''^src/libutil/git\.cc$'' + ''^src/libutil/git\.hh$'' + ''^src/libutil/hash\.cc$'' + ''^src/libutil/hash\.hh$'' + ''^src/libutil/hilite\.cc$'' + ''^src/libutil/hilite\.hh$'' + ''^src/libutil/source-accessor\.hh$'' + ''^src/libutil/json-impls\.hh$'' + ''^src/libutil/json-utils\.cc$'' + ''^src/libutil/json-utils\.hh$'' + ''^src/libutil/linux/cgroup\.cc$'' + ''^src/libutil/linux/namespaces\.cc$'' + ''^src/libutil/logging\.cc$'' + ''^src/libutil/logging\.hh$'' + ''^src/libutil/lru-cache\.hh$'' + ''^src/libutil/memory-source-accessor\.cc$'' + ''^src/libutil/memory-source-accessor\.hh$'' + ''^src/libutil/pool\.hh$'' + ''^src/libutil/position\.cc$'' + ''^src/libutil/position\.hh$'' + ''^src/libutil/posix-source-accessor\.cc$'' + ''^src/libutil/posix-source-accessor\.hh$'' + ''^src/libutil/processes\.hh$'' + ''^src/libutil/ref\.hh$'' + ''^src/libutil/references\.cc$'' + ''^src/libutil/references\.hh$'' + ''^src/libutil/regex-combinators\.hh$'' + ''^src/libutil/serialise\.cc$'' + ''^src/libutil/serialise\.hh$'' + ''^src/libutil/signals\.hh$'' + ''^src/libutil/signature/local-keys\.cc$'' + ''^src/libutil/signature/local-keys\.hh$'' + ''^src/libutil/signature/signer\.cc$'' + ''^src/libutil/signature/signer\.hh$'' + ''^src/libutil/source-accessor\.cc$'' + ''^src/libutil/source-accessor\.hh$'' + ''^src/libutil/source-path\.cc$'' + ''^src/libutil/source-path\.hh$'' + ''^src/libutil/split\.hh$'' + ''^src/libutil/suggestions\.cc$'' + ''^src/libutil/suggestions\.hh$'' + ''^src/libutil/sync\.hh$'' + ''^src/libutil/terminal\.cc$'' + ''^src/libutil/terminal\.hh$'' + ''^src/libutil/thread-pool\.cc$'' + ''^src/libutil/thread-pool\.hh$'' + ''^src/libutil/topo-sort\.hh$'' + ''^src/libutil/types\.hh$'' + ''^src/libutil/unix/file-descriptor\.cc$'' + ''^src/libutil/unix/file-path\.cc$'' + ''^src/libutil/unix/monitor-fd\.hh$'' + ''^src/libutil/unix/processes\.cc$'' + ''^src/libutil/unix/signals-impl\.hh$'' + ''^src/libutil/unix/signals\.cc$'' + ''^src/libutil/unix-domain-socket\.cc$'' + ''^src/libutil/unix/users\.cc$'' + ''^src/libutil/url-parts\.hh$'' + ''^src/libutil/url\.cc$'' + ''^src/libutil/url\.hh$'' + ''^src/libutil/users\.cc$'' + ''^src/libutil/users\.hh$'' + ''^src/libutil/util\.cc$'' + ''^src/libutil/util\.hh$'' + ''^src/libutil/variant-wrapper\.hh$'' + ''^src/libutil/windows/environment-variables\.cc$'' + ''^src/libutil/windows/file-descriptor\.cc$'' + ''^src/libutil/windows/file-path\.cc$'' + ''^src/libutil/windows/processes\.cc$'' + ''^src/libutil/windows/users\.cc$'' + ''^src/libutil/windows/windows-error\.cc$'' + ''^src/libutil/windows/windows-error\.hh$'' + ''^src/libutil/xml-writer\.cc$'' + ''^src/libutil/xml-writer\.hh$'' + ''^src/nix-build/nix-build\.cc$'' + ''^src/nix-channel/nix-channel\.cc$'' + ''^src/nix-collect-garbage/nix-collect-garbage\.cc$'' + ''^src/nix-env/buildenv.nix$'' + ''^src/nix-env/nix-env\.cc$'' + ''^src/nix-env/user-env\.cc$'' + ''^src/nix-env/user-env\.hh$'' + ''^src/nix-instantiate/nix-instantiate\.cc$'' + ''^src/nix-store/dotgraph\.cc$'' + ''^src/nix-store/graphml\.cc$'' + ''^src/nix-store/nix-store\.cc$'' + ''^src/nix/add-to-store\.cc$'' + ''^src/nix/app\.cc$'' + ''^src/nix/build\.cc$'' + ''^src/nix/bundle\.cc$'' + ''^src/nix/cat\.cc$'' + ''^src/nix/config-check\.cc$'' + ''^src/nix/config\.cc$'' + ''^src/nix/copy\.cc$'' + ''^src/nix/derivation-add\.cc$'' + ''^src/nix/derivation-show\.cc$'' + ''^src/nix/derivation\.cc$'' + ''^src/nix/develop\.cc$'' + ''^src/nix/diff-closures\.cc$'' + ''^src/nix/dump-path\.cc$'' + ''^src/nix/edit\.cc$'' + ''^src/nix/eval\.cc$'' + ''^src/nix/flake\.cc$'' + ''^src/nix/fmt\.cc$'' + ''^src/nix/hash\.cc$'' + ''^src/nix/log\.cc$'' + ''^src/nix/ls\.cc$'' + ''^src/nix/main\.cc$'' + ''^src/nix/make-content-addressed\.cc$'' + ''^src/nix/nar\.cc$'' + ''^src/nix/optimise-store\.cc$'' + ''^src/nix/path-from-hash-part\.cc$'' + ''^src/nix/path-info\.cc$'' + ''^src/nix/prefetch\.cc$'' + ''^src/nix/profile\.cc$'' + ''^src/nix/realisation\.cc$'' + ''^src/nix/registry\.cc$'' + ''^src/nix/repl\.cc$'' + ''^src/nix/run\.cc$'' + ''^src/nix/run\.hh$'' + ''^src/nix/search\.cc$'' + ''^src/nix/sigs\.cc$'' + ''^src/nix/store-copy-log\.cc$'' + ''^src/nix/store-delete\.cc$'' + ''^src/nix/store-gc\.cc$'' + ''^src/nix/store-info\.cc$'' + ''^src/nix/store-repair\.cc$'' + ''^src/nix/store\.cc$'' + ''^src/nix/unix/daemon\.cc$'' + ''^src/nix/upgrade-nix\.cc$'' + ''^src/nix/verify\.cc$'' + ''^src/nix/why-depends\.cc$'' + + ''^tests/functional/plugins/plugintest\.cc'' + ''^tests/functional/test-libstoreconsumer/main\.cc'' + ''^tests/nixos/ca-fd-leak/sender\.c'' + ''^tests/nixos/ca-fd-leak/smuggler\.c'' + ''^tests/nixos/user-sandboxing/attacker\.c'' + ''^tests/unit/libexpr-support/tests/libexpr\.hh'' + ''^tests/unit/libexpr-support/tests/value/context\.cc'' + ''^tests/unit/libexpr-support/tests/value/context\.hh'' + ''^tests/unit/libexpr/derived-path\.cc'' + ''^tests/unit/libexpr/error_traces\.cc'' + ''^tests/unit/libexpr/eval\.cc'' + ''^tests/unit/libexpr/json\.cc'' + ''^tests/unit/libexpr/main\.cc'' + ''^tests/unit/libexpr/primops\.cc'' + ''^tests/unit/libexpr/search-path\.cc'' + ''^tests/unit/libexpr/trivial\.cc'' + ''^tests/unit/libexpr/value/context\.cc'' + ''^tests/unit/libexpr/value/print\.cc'' + ''^tests/unit/libfetchers/public-key\.cc'' + ''^tests/unit/libflake/flakeref\.cc'' + ''^tests/unit/libflake/url-name\.cc'' + ''^tests/unit/libstore-support/tests/derived-path\.cc'' + ''^tests/unit/libstore-support/tests/derived-path\.hh'' + ''^tests/unit/libstore-support/tests/nix_api_store\.hh'' + ''^tests/unit/libstore-support/tests/outputs-spec\.cc'' + ''^tests/unit/libstore-support/tests/outputs-spec\.hh'' + ''^tests/unit/libstore-support/tests/path\.cc'' + ''^tests/unit/libstore-support/tests/path\.hh'' + ''^tests/unit/libstore-support/tests/protocol\.hh'' + ''^tests/unit/libstore/common-protocol\.cc'' + ''^tests/unit/libstore/content-address\.cc'' + ''^tests/unit/libstore/derivation\.cc'' + ''^tests/unit/libstore/derived-path\.cc'' + ''^tests/unit/libstore/downstream-placeholder\.cc'' + ''^tests/unit/libstore/machines\.cc'' + ''^tests/unit/libstore/nar-info-disk-cache\.cc'' + ''^tests/unit/libstore/nar-info\.cc'' + ''^tests/unit/libstore/outputs-spec\.cc'' + ''^tests/unit/libstore/path-info\.cc'' + ''^tests/unit/libstore/path\.cc'' + ''^tests/unit/libstore/serve-protocol\.cc'' + ''^tests/unit/libstore/worker-protocol\.cc'' + ''^tests/unit/libutil-support/tests/characterization\.hh'' + ''^tests/unit/libutil-support/tests/hash\.cc'' + ''^tests/unit/libutil-support/tests/hash\.hh'' + ''^tests/unit/libutil/args\.cc'' + ''^tests/unit/libutil/canon-path\.cc'' + ''^tests/unit/libutil/chunked-vector\.cc'' + ''^tests/unit/libutil/closure\.cc'' + ''^tests/unit/libutil/compression\.cc'' + ''^tests/unit/libutil/config\.cc'' + ''^tests/unit/libutil/file-content-address\.cc'' + ''^tests/unit/libutil/git\.cc'' + ''^tests/unit/libutil/hash\.cc'' + ''^tests/unit/libutil/hilite\.cc'' + ''^tests/unit/libutil/json-utils\.cc'' + ''^tests/unit/libutil/logging\.cc'' + ''^tests/unit/libutil/lru-cache\.cc'' + ''^tests/unit/libutil/pool\.cc'' + ''^tests/unit/libutil/references\.cc'' + ''^tests/unit/libutil/suggestions\.cc'' + ''^tests/unit/libutil/tests\.cc'' + ''^tests/unit/libutil/url\.cc'' + ''^tests/unit/libutil/xml-writer\.cc'' + ]; + }; + shellcheck = { + enable = true; + excludes = [ + # We haven't linted these files yet + ''^config/install-sh$'' + ''^misc/bash/completion\.sh$'' + ''^misc/fish/completion\.fish$'' + ''^misc/zsh/completion\.zsh$'' + ''^scripts/create-darwin-volume\.sh$'' + ''^scripts/install-darwin-multi-user\.sh$'' + ''^scripts/install-multi-user\.sh$'' + ''^scripts/install-nix-from-closure\.sh$'' + ''^scripts/install-systemd-multi-user\.sh$'' + ''^src/nix/get-env\.sh$'' + ''^tests/functional/build\.sh$'' + ''^tests/functional/ca/build-dry\.sh$'' + ''^tests/functional/ca/build-with-garbage-path\.sh$'' + ''^tests/functional/ca/common\.sh$'' + ''^tests/functional/ca/concurrent-builds\.sh$'' + ''^tests/functional/ca/eval-store\.sh$'' + ''^tests/functional/ca/gc\.sh$'' + ''^tests/functional/ca/import-derivation\.sh$'' + ''^tests/functional/ca/new-build-cmd\.sh$'' + ''^tests/functional/ca/nix-shell\.sh$'' + ''^tests/functional/ca/post-hook\.sh$'' + ''^tests/functional/ca/recursive\.sh$'' + ''^tests/functional/ca/repl\.sh$'' + ''^tests/functional/ca/selfref-gc\.sh$'' + ''^tests/functional/ca/why-depends\.sh$'' + ''^tests/functional/characterisation-test-infra\.sh$'' + ''^tests/functional/check\.sh$'' + ''^tests/functional/common/vars-and-functions\.sh$'' + ''^tests/functional/completions\.sh$'' + ''^tests/functional/compute-levels\.sh$'' + ''^tests/functional/config\.sh$'' + ''^tests/functional/db-migration\.sh$'' + ''^tests/functional/debugger\.sh$'' + ''^tests/functional/dependencies\.builder0\.sh$'' + ''^tests/functional/dependencies\.sh$'' + ''^tests/functional/dump-db\.sh$'' + ''^tests/functional/dyn-drv/build-built-drv\.sh$'' + ''^tests/functional/dyn-drv/common\.sh$'' + ''^tests/functional/dyn-drv/dep-built-drv\.sh$'' + ''^tests/functional/dyn-drv/eval-outputOf\.sh$'' + ''^tests/functional/dyn-drv/old-daemon-error-hack\.sh$'' + ''^tests/functional/dyn-drv/recursive-mod-json\.sh$'' + ''^tests/functional/eval-store\.sh$'' + ''^tests/functional/eval\.sh$'' + ''^tests/functional/export-graph\.sh$'' + ''^tests/functional/export\.sh$'' + ''^tests/functional/extra-sandbox-profile\.sh$'' + ''^tests/functional/fetchClosure\.sh$'' + ''^tests/functional/fetchGit\.sh$'' + ''^tests/functional/fetchGitRefs\.sh$'' + ''^tests/functional/fetchGitSubmodules\.sh$'' + ''^tests/functional/fetchGitVerification\.sh$'' + ''^tests/functional/fetchMercurial\.sh$'' + ''^tests/functional/fetchurl\.sh$'' + ''^tests/functional/fixed\.builder1\.sh$'' + ''^tests/functional/fixed\.builder2\.sh$'' + ''^tests/functional/fixed\.sh$'' + ''^tests/functional/flakes/absolute-paths\.sh$'' + ''^tests/functional/flakes/check\.sh$'' + ''^tests/functional/flakes/common\.sh$'' + ''^tests/functional/flakes/config\.sh$'' + ''^tests/functional/flakes/develop\.sh$'' + ''^tests/functional/flakes/flakes\.sh$'' + ''^tests/functional/flakes/follow-paths\.sh$'' + ''^tests/functional/flakes/prefetch\.sh$'' + ''^tests/functional/flakes/run\.sh$'' + ''^tests/functional/flakes/show\.sh$'' + ''^tests/functional/fmt\.sh$'' + ''^tests/functional/fmt\.simple\.sh$'' + ''^tests/functional/gc-auto\.sh$'' + ''^tests/functional/gc-concurrent\.builder\.sh$'' + ''^tests/functional/gc-concurrent\.sh$'' + ''^tests/functional/gc-concurrent2\.builder\.sh$'' + ''^tests/functional/gc-non-blocking\.sh$'' + ''^tests/functional/gc\.sh$'' + ''^tests/functional/git-hashing/common\.sh$'' + ''^tests/functional/git-hashing/simple\.sh$'' + ''^tests/functional/hash-convert\.sh$'' + ''^tests/functional/help\.sh$'' + ''^tests/functional/impure-derivations\.sh$'' + ''^tests/functional/impure-env\.sh$'' + ''^tests/functional/impure-eval\.sh$'' + ''^tests/functional/install-darwin\.sh$'' + ''^tests/functional/lang\.sh$'' + ''^tests/functional/legacy-ssh-store\.sh$'' + ''^tests/functional/linux-sandbox\.sh$'' + ''^tests/functional/local-overlay-store/add-lower-inner\.sh$'' + ''^tests/functional/local-overlay-store/add-lower\.sh$'' + ''^tests/functional/local-overlay-store/bad-uris\.sh$'' + ''^tests/functional/local-overlay-store/build-inner\.sh$'' + ''^tests/functional/local-overlay-store/build\.sh$'' + ''^tests/functional/local-overlay-store/check-post-init-inner\.sh$'' + ''^tests/functional/local-overlay-store/check-post-init\.sh$'' + ''^tests/functional/local-overlay-store/common\.sh$'' + ''^tests/functional/local-overlay-store/delete-duplicate-inner\.sh$'' + ''^tests/functional/local-overlay-store/delete-duplicate\.sh$'' + ''^tests/functional/local-overlay-store/delete-refs-inner\.sh$'' + ''^tests/functional/local-overlay-store/delete-refs\.sh$'' + ''^tests/functional/local-overlay-store/gc-inner\.sh$'' + ''^tests/functional/local-overlay-store/gc\.sh$'' + ''^tests/functional/local-overlay-store/optimise-inner\.sh$'' + ''^tests/functional/local-overlay-store/optimise\.sh$'' + ''^tests/functional/local-overlay-store/redundant-add-inner\.sh$'' + ''^tests/functional/local-overlay-store/redundant-add\.sh$'' + ''^tests/functional/local-overlay-store/remount\.sh$'' + ''^tests/functional/local-overlay-store/stale-file-handle-inner\.sh$'' + ''^tests/functional/local-overlay-store/stale-file-handle\.sh$'' + ''^tests/functional/local-overlay-store/verify-inner\.sh$'' + ''^tests/functional/local-overlay-store/verify\.sh$'' + ''^tests/functional/logging\.sh$'' + ''^tests/functional/misc\.sh$'' + ''^tests/functional/multiple-outputs\.sh$'' + ''^tests/functional/nar-access\.sh$'' + ''^tests/functional/nested-sandboxing\.sh$'' + ''^tests/functional/nested-sandboxing/command\.sh$'' + ''^tests/functional/nix-build\.sh$'' + ''^tests/functional/nix-channel\.sh$'' + ''^tests/functional/nix-collect-garbage-d\.sh$'' + ''^tests/functional/nix-copy-ssh-common\.sh$'' + ''^tests/functional/nix-copy-ssh-ng\.sh$'' + ''^tests/functional/nix-copy-ssh\.sh$'' + ''^tests/functional/nix-daemon-untrusting\.sh$'' + ''^tests/functional/nix-profile\.sh$'' + ''^tests/functional/nix-shell\.sh$'' + ''^tests/functional/nix_path\.sh$'' + ''^tests/functional/optimise-store\.sh$'' + ''^tests/functional/output-normalization\.sh$'' + ''^tests/functional/parallel\.builder\.sh$'' + ''^tests/functional/parallel\.sh$'' + ''^tests/functional/pass-as-file\.sh$'' + ''^tests/functional/path-from-hash-part\.sh$'' + ''^tests/functional/path-info\.sh$'' + ''^tests/functional/placeholders\.sh$'' + ''^tests/functional/plugins\.sh$'' + ''^tests/functional/post-hook\.sh$'' + ''^tests/functional/pure-eval\.sh$'' + ''^tests/functional/push-to-store-old\.sh$'' + ''^tests/functional/push-to-store\.sh$'' + ''^tests/functional/read-only-store\.sh$'' + ''^tests/functional/readfile-context\.sh$'' + ''^tests/functional/recursive\.sh$'' + ''^tests/functional/referrers\.sh$'' + ''^tests/functional/remote-store\.sh$'' + ''^tests/functional/repair\.sh$'' + ''^tests/functional/restricted\.sh$'' + ''^tests/functional/search\.sh$'' + ''^tests/functional/secure-drv-outputs\.sh$'' + ''^tests/functional/selfref-gc\.sh$'' + ''^tests/functional/shell\.sh$'' + ''^tests/functional/shell\.shebang\.sh$'' + ''^tests/functional/signing\.sh$'' + ''^tests/functional/simple\.builder\.sh$'' + ''^tests/functional/simple\.sh$'' + ''^tests/functional/ssh-relay\.sh$'' + ''^tests/functional/store-info\.sh$'' + ''^tests/functional/structured-attrs\.sh$'' + ''^tests/functional/substitute-with-invalid-ca\.sh$'' + ''^tests/functional/suggestions\.sh$'' + ''^tests/functional/supplementary-groups\.sh$'' + ''^tests/functional/tarball\.sh$'' + ''^tests/functional/test-infra\.sh$'' + ''^tests/functional/test-libstoreconsumer\.sh$'' + ''^tests/functional/timeout\.sh$'' + ''^tests/functional/toString-path\.sh$'' + ''^tests/functional/user-envs-migration\.sh$'' + ''^tests/functional/user-envs-test-case\.sh$'' + ''^tests/functional/user-envs\.builder\.sh$'' + ''^tests/functional/user-envs\.sh$'' + ''^tests/functional/why-depends\.sh$'' + ''^tests/functional/zstd\.sh$'' + ''^tests/unit/libutil/data/git/check-data\.sh$'' + ]; + }; + # TODO: nixfmt, https://github.com/NixOS/nixfmt/issues/153 + }; + }; + }; + + # We'll be pulling from this in the main flake + flake.getSystem = getSystem; +} diff --git a/maintainers/local.mk b/maintainers/local.mk new file mode 100644 index 000000000..88d594d67 --- /dev/null +++ b/maintainers/local.mk @@ -0,0 +1,15 @@ + +.PHONY: format +print-top-help += echo ' format: Format source code' + +# This uses the cached .pre-commit-hooks.yaml file +format: + @if ! type -p pre-commit &>/dev/null; then \ + echo "make format: pre-commit not found. Please use \`nix develop\`."; \ + exit 1; \ + fi; \ + if test -z "$$_NIX_PRE_COMMIT_HOOKS_CONFIG"; then \ + echo "make format: _NIX_PRE_COMMIT_HOOKS_CONFIG not set. Please use \`nix develop\`."; \ + exit 1; \ + fi; \ + pre-commit run --config $$_NIX_PRE_COMMIT_HOOKS_CONFIG --all-files diff --git a/maintainers/release-credits b/maintainers/release-credits new file mode 100755 index 000000000..7a5c87d7d --- /dev/null +++ b/maintainers/release-credits @@ -0,0 +1,184 @@ +#!/usr/bin/env nix +# vim: set filetype=python: +#!nix develop --impure --expr +#!nix `` +#!nix let flake = builtins.getFlake ("git+file://" + toString ../.); +#!nix pkgs = flake.inputs.nixpkgs.legacyPackages.${builtins.currentSystem}; +#!nix in pkgs.mkShell { nativeBuildInputs = [ +#!nix (pkgs.python3.withPackages (ps: with ps; [ requests ])) +#!nix ]; } +#!nix `` --command python3 + +# This script lists out the contributors for a given release. +# It must be run from the root of the Nix repository. + +import os +import sys +import json +import requests + +github_token = os.environ.get("GITHUB_TOKEN") +if not github_token: + print("GITHUB_TOKEN is not set. If you hit the rate limit, set it", file=sys.stderr) + # Might be ok, as we have a cache. + # raise ValueError("GITHUB_TOKEN must be set") + +# 1. Read the current version in .version +version = os.environ.get("VERSION") +if not version: + version = open(".version").read().strip() + +print(f"Generating release credits for Nix {version}", file=sys.stderr) + +# 2. Compute previous version +vcomponents = version.split(".") +if len(vcomponents) >= 2: + prev_version = f"{vcomponents[0]}.{int(vcomponents[1])-1}.0" +else: + raise ValueError(".version must have at least two components") + +# For unreleased versions +endref = "HEAD" +# For older releases +# endref = version + +# 2. Find the merge base between the current version and the previous version +mergeBase = os.popen(f"git merge-base {prev_version} {endref}").read().strip() +print(f"Merge base between {prev_version} and {endref} is {mergeBase}", file=sys.stderr) + +# 3. Find the date of the merge base +mergeBaseDate = os.popen(f"git show -s --format=%ci {mergeBase}").read().strip()[0:10] +print(f"Merge base date is {mergeBaseDate}", file=sys.stderr) + +# 4. Get the commits between the merge base and the current version + +def get_commits(): + raw = os.popen(f"git log --pretty=format:'%H\t%an\t%ae' {mergeBase}..{endref}").read().strip() + lines = raw.split("\n") + return [ { "hash": items[0], "author": items[1], "email": items[2] } + for line in lines + for items in (line.split("\t"),) + ] + +def commits_to_first_commit_by_email(commits): + by_email = dict() + for commit in commits: + email = commit["email"] + if email not in by_email: + by_email[email] = commit + return by_email + + +samples = commits_to_first_commit_by_email(get_commits()) + +# For quick testing, only pick two samples from the dict +# samples = dict(list(samples.items())[:2]) + +# Query the GitHub API to get handle +def get_github_commit(commit): + url = f"https://api.github.com/repos/NixOS/nix/commits/{commit['hash']}" + headers = {'Authorization': f'token {github_token}'} + response = requests.get(url, headers=headers) + response.raise_for_status() + return response.json() + +class Cache: + def __init__(self, filename, require = True): + self.filename = filename + try: + with open(filename, "r") as f: + self.values = json.load(f) + except FileNotFoundError: + if require: + raise + self.values = dict() + def save(self): + with open(self.filename, "w") as f: + json.dump(self.values, f, indent=4) + print(f"Saved cache to {self.filename}", file=sys.stderr) + +# The email to handle cache maps email addresses to either +# - a handle (string) +# - None (if no handle was found) +email_to_handle_cache = Cache("maintainers/data/release-credits-email-to-handle.json") + +handles = set() +emails = dict() + +for sample in samples: + s = samples[sample] + email = s["email"] + if not email in email_to_handle_cache.values: + print(f"Querying GitHub API for {s['hash']}, to get handle for {s['email']}") + ghc = get_github_commit(samples[sample]) + gha = ghc["author"] + if gha and gha["login"]: + handle = gha["login"] + print(f"Handle: {handle}") + email_to_handle_cache.values[email] = handle + else: + print(f"Found no handle for {s['email']}") + email_to_handle_cache.values[email] = None + handle = email_to_handle_cache.values[email] + if handle is not None: + handles.add(handle) + else: + emails[email] = s["author"] + +# print(email_to_handle_cache.values) + +email_to_handle_cache.save() + +handle_to_name_cache = Cache("maintainers/data/release-credits-handle-to-name.json") + +print(f"Found {len(handles)} handles", file=sys.stderr) + +for handle in handles: + if not handle in handle_to_name_cache.values: + print(f"Querying GitHub API for {handle}, to get name", file=sys.stderr) + url = f"https://api.github.com/users/{handle}" + headers = {'Authorization': f'token {github_token}'} + response = requests.get(url, headers=headers) + response.raise_for_status() + user = response.json() + name = user["name"] + print(f"Name: {name}", file=sys.stderr) + handle_to_name_cache.values[handle] = name + +handle_to_name_cache.save() + +entries = list() + +def is_bot(handle): + return handle.endswith("[bot]") + +for handle in handles: + name = handle_to_name_cache.values[handle] + if name is None: + # This way it looks more regular + name = handle + + # Note to bots: I'm only doing this because it would distract humans, and + # therefore delay your creation. I'm sorry; it's for your benefit. + if not is_bot(handle): + entries += [ f"- {name} [**(@{handle})**](https://github.com/{handle})" ] + +def shuffle(entries): + salt = os.urandom(16) + return sorted(entries, key=lambda x: hash((x, salt))) + +# Fair ordering is undecidable +entries = shuffle(entries) + +# For a sanity check, we could sort the entries by handle instead. +# entries = sorted(entries) + +print("") +print(f"This release was made possible by the following {len(entries)} contributors:") +print("") + +for entry in entries: + print(entry) + +for email in emails: + print(f"- {emails[email]}") diff --git a/maintainers/release-notes b/maintainers/release-notes index 2d84485c1..c0c4ee734 100755 --- a/maintainers/release-notes +++ b/maintainers/release-notes @@ -1,5 +1,6 @@ #!/usr/bin/env nix -#!nix shell .#changelog-d-nix --command bash +# vim: set filetype=bash: +#!nix shell .#changelog-d --command bash # --- CONFIGURATION --- @@ -151,6 +152,13 @@ section_title="Release $version_full ($DATE)" echo "# $section_title" echo changelog-d doc/manual/rl-next | sed -e 's/ *$//' + + if ! $IS_PATCH; then + echo + echo "# Contributors" + echo + VERSION=$version_full ./maintainers/release-credits + fi ) | tee -a $file log "Wrote $file" diff --git a/maintainers/release-process.md b/maintainers/release-process.md index da6886ea9..7a2b3c0a7 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -39,6 +39,10 @@ release: * Proof-read / edit / rearrange the release notes if needed. Breaking changes and highlights should go to the top. +* Run `maintainers/release-credits` to make sure the credits script works + and produces a sensible output. Some emails might not automatically map to + a GitHub handle. + * Push. ```console diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 4e2c379f0..731988568 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -11,6 +11,8 @@ use JSON::PP; use LWP::UserAgent; use Net::Amazon::S3; +delete $ENV{'shell'}; # shut up a LWP::UserAgent.pm warning + my $evalId = $ARGV[0] or die "Usage: $0 EVAL-ID\n"; my $releasesBucketName = "nix-releases"; @@ -36,11 +38,11 @@ sub fetch { my $evalUrl = "https://hydra.nixos.org/eval/$evalId"; my $evalInfo = decode_json(fetch($evalUrl, 'application/json')); #print Dumper($evalInfo); -my $flakeUrl = $evalInfo->{flake} or die; -my $flakeInfo = decode_json(`nix flake metadata --json "$flakeUrl"` or die); -my $nixRev = $flakeInfo->{revision} or die; +my $flakeUrl = $evalInfo->{flake}; +my $flakeInfo = decode_json(`nix flake metadata --json "$flakeUrl"` or die) if $flakeUrl; +my $nixRev = ($flakeInfo ? $flakeInfo->{revision} : $evalInfo->{jobsetevalinputs}->{nix}->{revision}) or die; -my $buildInfo = decode_json(fetch("$evalUrl/job/build.x86_64-linux", 'application/json')); +my $buildInfo = decode_json(fetch("$evalUrl/job/build.nix.x86_64-linux", 'application/json')); #print Dumper($buildInfo); my $releaseName = $buildInfo->{nixname}; @@ -83,12 +85,19 @@ my $channelsBucket = $s3_us->bucket($channelsBucketName) or die; sub getStorePath { my ($jobName, $output) = @_; my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json')); - return $buildInfo->{buildoutputs}->{$output or "out"}->{path} or die "cannot get store path for '$jobName'"; + return $buildInfo->{buildoutputs}->{$output or "out"}->{path} // die "cannot get store path for '$jobName'"; } sub copyManual { - my $manual = getStorePath("build.x86_64-linux", "doc"); - print "$manual\n"; + my $manual; + eval { + $manual = getStorePath("build.nix.x86_64-linux", "doc"); + }; + if ($@) { + warn "$@"; + return; + } + print "Manual: $manual\n"; my $manualNar = "$tmpDir/$releaseName-manual.nar.xz"; print "$manualNar\n"; @@ -154,19 +163,34 @@ downloadFile("binaryTarball.x86_64-linux", "1"); downloadFile("binaryTarball.aarch64-linux", "1"); downloadFile("binaryTarball.x86_64-darwin", "1"); downloadFile("binaryTarball.aarch64-darwin", "1"); -downloadFile("binaryTarballCross.x86_64-linux.armv6l-unknown-linux-gnueabihf", "1"); -downloadFile("binaryTarballCross.x86_64-linux.armv7l-unknown-linux-gnueabihf", "1"); +eval { + downloadFile("binaryTarballCross.x86_64-linux.armv6l-unknown-linux-gnueabihf", "1"); +}; +warn "$@" if $@; +eval { + downloadFile("binaryTarballCross.x86_64-linux.armv7l-unknown-linux-gnueabihf", "1"); +}; +warn "$@" if $@; +eval { + downloadFile("binaryTarballCross.x86_64-linux.riscv64-unknown-linux-gnu", "1"); +}; +warn "$@" if $@; downloadFile("installerScript", "1"); # Upload docker images to dockerhub. my $dockerManifest = ""; my $dockerManifestLatest = ""; +my $haveDocker = 0; for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) { my $system = $platforms->[0]; my $dockerPlatform = $platforms->[1]; my $fn = "nix-$version-docker-image-$dockerPlatform.tar.gz"; - downloadFile("dockerImage.$system", "1", $fn); + eval { + downloadFile("dockerImage.$system", "1", $fn); + }; + die "$@" if $@; + $haveDocker = 1; print STDERR "loading docker image for $dockerPlatform...\n"; system("docker load -i $tmpDir/$fn") == 0 or die; @@ -194,31 +218,34 @@ for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) { $dockerManifestLatest .= " --amend $latestTag" } -print STDERR "creating multi-platform docker manifest...\n"; -system("docker manifest rm nixos/nix:$version"); -system("docker manifest create nixos/nix:$version $dockerManifest") == 0 or die; -if ($isLatest) { - print STDERR "creating latest multi-platform docker manifest...\n"; - system("docker manifest rm nixos/nix:latest"); - system("docker manifest create nixos/nix:latest $dockerManifestLatest") == 0 or die; -} +if ($haveDocker) { + print STDERR "creating multi-platform docker manifest...\n"; + system("docker manifest rm nixos/nix:$version"); + system("docker manifest create nixos/nix:$version $dockerManifest") == 0 or die; + if ($isLatest) { + print STDERR "creating latest multi-platform docker manifest...\n"; + system("docker manifest rm nixos/nix:latest"); + system("docker manifest create nixos/nix:latest $dockerManifestLatest") == 0 or die; + } -print STDERR "pushing multi-platform docker manifest...\n"; -system("docker manifest push nixos/nix:$version") == 0 or die; + print STDERR "pushing multi-platform docker manifest...\n"; + system("docker manifest push nixos/nix:$version") == 0 or die; -if ($isLatest) { - print STDERR "pushing latest multi-platform docker manifest...\n"; - system("docker manifest push nixos/nix:latest") == 0 or die; + if ($isLatest) { + print STDERR "pushing latest multi-platform docker manifest...\n"; + system("docker manifest push nixos/nix:latest") == 0 or die; + } } # Upload nix-fallback-paths.nix. write_file("$tmpDir/fallback-paths.nix", "{\n" . - " x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" . - " i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" . - " aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" . - " x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" . - " aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" . + " x86_64-linux = \"" . getStorePath("build.nix.x86_64-linux") . "\";\n" . + " i686-linux = \"" . getStorePath("build.nix.i686-linux") . "\";\n" . + " aarch64-linux = \"" . getStorePath("build.nix.aarch64-linux") . "\";\n" . + " riscv64-linux = \"" . getStorePath("buildCross.nix.riscv64-unknown-linux-gnu.x86_64-linux") . "\";\n" . + " x86_64-darwin = \"" . getStorePath("build.nix.x86_64-darwin") . "\";\n" . + " aarch64-darwin = \"" . getStorePath("build.nix.aarch64-darwin") . "\";\n" . "}\n"); # Upload release files to S3. diff --git a/meson.build b/meson.build new file mode 100644 index 000000000..1554244ab --- /dev/null +++ b/meson.build @@ -0,0 +1,44 @@ +# This is just a stub project to include all the others as subprojects +# for development shell purposes + +project('nix-dev-shell', 'cpp', + version : files('.version'), + subproject_dir : 'src', +) + +# Internal Libraries +subproject('libutil') +subproject('libstore') +subproject('libfetchers') +subproject('libexpr') +subproject('libflake') +subproject('libmain') +subproject('libcmd') + +# Executables +subproject('nix') + +# Docs +subproject('internal-api-docs') +subproject('external-api-docs') + +# External C wrapper libraries +subproject('libutil-c') +subproject('libstore-c') +subproject('libexpr-c') +subproject('libmain-c') + +# Language Bindings +if not meson.is_cross_build() + subproject('perl') +endif + +# Testing +subproject('nix-util-test-support') +subproject('nix-util-tests') +subproject('nix-store-test-support') +subproject('nix-store-tests') +subproject('nix-fetchers-tests') +subproject('nix-expr-test-support') +subproject('nix-expr-tests') +subproject('nix-flake-tests') diff --git a/misc/changelog-d.cabal.nix b/misc/changelog-d.cabal.nix deleted file mode 100644 index 76f9353cd..000000000 --- a/misc/changelog-d.cabal.nix +++ /dev/null @@ -1,31 +0,0 @@ -{ mkDerivation, aeson, base, bytestring, cabal-install-parsers -, Cabal-syntax, containers, directory, filepath, frontmatter -, generic-lens-lite, lib, mtl, optparse-applicative, parsec, pretty -, regex-applicative, text, pkgs -}: -let rev = "f30f6969e9cd8b56242309639d58acea21c99d06"; -in -mkDerivation { - pname = "changelog-d"; - version = "0.1"; - src = pkgs.fetchurl { - name = "changelog-d-${rev}.tar.gz"; - url = "https://codeberg.org/roberth/changelog-d/archive/${rev}.tar.gz"; - hash = "sha256-8a2+i5u7YoszAgd5OIEW0eYUcP8yfhtoOIhLJkylYJ4="; - } // { inherit rev; }; - isLibrary = false; - isExecutable = true; - libraryHaskellDepends = [ - aeson base bytestring cabal-install-parsers Cabal-syntax containers - directory filepath frontmatter generic-lens-lite mtl parsec pretty - regex-applicative text - ]; - executableHaskellDepends = [ - base bytestring Cabal-syntax directory filepath - optparse-applicative - ]; - doHaddock = false; - description = "Concatenate changelog entries into a single one"; - license = lib.licenses.gpl3Plus; - mainProgram = "changelog-d"; -} diff --git a/misc/changelog-d.nix b/misc/changelog-d.nix deleted file mode 100644 index 1b20f4596..000000000 --- a/misc/changelog-d.nix +++ /dev/null @@ -1,31 +0,0 @@ -# Taken temporarily from -{ - callPackage, - lib, - haskell, - haskellPackages, -}: - -let - hsPkg = haskellPackages.callPackage ./changelog-d.cabal.nix { }; - - addCompletions = haskellPackages.generateOptparseApplicativeCompletions ["changelog-d"]; - - haskellModifications = - lib.flip lib.pipe [ - addCompletions - haskell.lib.justStaticExecutables - ]; - - mkDerivationOverrides = finalAttrs: oldAttrs: { - - version = oldAttrs.version + "-git-${lib.strings.substring 0 7 oldAttrs.src.rev}"; - - meta = oldAttrs.meta // { - homepage = "https://codeberg.org/roberth/changelog-d"; - maintainers = [ lib.maintainers.roberth ]; - }; - - }; -in - (haskellModifications hsPkg).overrideAttrs mkDerivationOverrides diff --git a/misc/systemv/nix-daemon b/misc/systemv/nix-daemon index fea537167..e8326f947 100755 --- a/misc/systemv/nix-daemon +++ b/misc/systemv/nix-daemon @@ -34,6 +34,7 @@ else fi # Source function library. +# shellcheck source=/dev/null . /etc/init.d/functions LOCKFILE=/var/lock/subsys/nix-daemon @@ -41,14 +42,20 @@ RUNDIR=/var/run/nix PIDFILE=${RUNDIR}/nix-daemon.pid RETVAL=0 -base=${0##*/} +# https://www.shellcheck.net/wiki/SC3004 +# Check if gettext exists +if ! type gettext > /dev/null 2>&1 +then + # If not, create a dummy function that returns the input verbatim + gettext() { printf '%s' "$1"; } +fi start() { mkdir -p ${RUNDIR} chown ${NIX_DAEMON_USER}:${NIX_DAEMON_USER} ${RUNDIR} - echo -n $"Starting nix daemon... " + printf '%s' "$(gettext 'Starting nix daemon... ')" daemonize -u $NIX_DAEMON_USER -p ${PIDFILE} $NIX_DAEMON_BIN $NIX_DAEMON_OPTS RETVAL=$? @@ -58,7 +65,7 @@ start() { } stop() { - echo -n $"Shutting down nix daemon: " + printf '%s' "$(gettext 'Shutting down nix daemon: ')" killproc -p ${PIDFILE} $NIX_DAEMON_BIN RETVAL=$? [ $RETVAL -eq 0 ] && rm -f ${LOCKFILE} ${PIDFILE} @@ -67,7 +74,7 @@ stop() { } reload() { - echo -n $"Reloading nix daemon... " + printf '%s' "$(gettext 'Reloading nix daemon... ')" killproc -p ${PIDFILE} $NIX_DAEMON_BIN -HUP RETVAL=$? echo @@ -105,7 +112,7 @@ case "$1" in fi ;; *) - echo $"Usage: $0 {start|stop|status|restart|condrestart}" + printf '%s' "$(gettext "Usage: $0 {start|stop|status|restart|condrestart}")" exit 2 ;; esac diff --git a/mk/common-test.sh b/mk/common-test.sh index 2783d293b..817422c40 100644 --- a/mk/common-test.sh +++ b/mk/common-test.sh @@ -1,27 +1,23 @@ +# shellcheck shell=bash + # Remove overall test dir (at most one of the two should match) and # remove file extension. -test_name=$(echo -n "$test" | sed \ - -e "s|^tests/unit/[^/]*/data/||" \ + +test_name=$(echo -n "${test?must be defined by caller (test runner)}" | sed \ + -e "s|^src/[^/]*-test/data/||" \ -e "s|^tests/functional/||" \ -e "s|\.sh$||" \ ) +# shellcheck disable=SC2016 TESTS_ENVIRONMENT=( "TEST_NAME=$test_name" 'NIX_REMOTE=' 'PS4=+(${BASH_SOURCE[0]-$0}:$LINENO) ' ) -: ${BASH:=/usr/bin/env bash} +read -r -a bash <<< "${BASH:-/usr/bin/env bash}" run () { - cd "$(dirname $1)" && env "${TESTS_ENVIRONMENT[@]}" $BASH -x -e -u -o pipefail $(basename $1) -} - -init_test () { - run "$init" 2>/dev/null > /dev/null -} - -run_test_proper () { - run "$test" + cd "$(dirname "$1")" && env "${TESTS_ENVIRONMENT[@]}" "${bash[@]}" -x -e -u -o pipefail "$(basename "$1")" } diff --git a/mk/compilation-database.mk b/mk/compilation-database.mk new file mode 100644 index 000000000..f69dc0de0 --- /dev/null +++ b/mk/compilation-database.mk @@ -0,0 +1,11 @@ +compile-commands-json-files := + +define write-compile-commands + _srcs := $$(sort $$(foreach src, $$($(1)_SOURCES), $$(src))) + + $(1)_COMPILE_COMMANDS_JSON := $$(addprefix $(buildprefix), $$(addsuffix .compile_commands.json, $$(basename $$(_srcs)))) + + compile-commands-json-files += $$($(1)_COMPILE_COMMANDS_JSON) + + clean-files += $$($(1)_COMPILE_COMMANDS_JSON) +endef diff --git a/mk/cxx-big-literal.mk b/mk/cxx-big-literal.mk index 85365df8e..d64a171c8 100644 --- a/mk/cxx-big-literal.mk +++ b/mk/cxx-big-literal.mk @@ -1,5 +1,5 @@ %.gen.hh: % - @echo 'R"foo(' >> $@.tmp + @echo 'R"__NIX_STR(' >> $@.tmp $(trace-gen) cat $< >> $@.tmp - @echo ')foo"' >> $@.tmp + @echo ')__NIX_STR"' >> $@.tmp @mv $@.tmp $@ diff --git a/mk/debug-test.sh b/mk/debug-test.sh index 52482c01e..0dd4406c3 100755 --- a/mk/debug-test.sh +++ b/mk/debug-test.sh @@ -3,12 +3,8 @@ set -eu -o pipefail test=$1 -init=${2-} dir="$(dirname "${BASH_SOURCE[0]}")" source "$dir/common-test.sh" -if [ -n "$init" ]; then - (init_test) -fi -run_test_proper +run "$test" diff --git a/mk/lib.mk b/mk/lib.mk index fe0add1c9..1e7af6ad5 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -68,6 +68,7 @@ include mk/patterns.mk include mk/templates.mk include mk/cxx-big-literal.mk include mk/tests.mk +include mk/compilation-database.mk # Include all sub-Makefiles. @@ -86,17 +87,23 @@ $(foreach script, $(bin-scripts), $(eval $(call install-program-in,$(script),$(b $(foreach script, $(bin-scripts), $(eval programs-list += $(script))) $(foreach script, $(noinst-scripts), $(eval programs-list += $(script))) $(foreach template, $(template-files), $(eval $(call instantiate-template,$(template)))) -install_test_init=tests/functional/init.sh $(foreach test, $(install-tests), \ - $(eval $(call run-test,$(test),$(install_test_init))) \ + $(eval $(call run-test,$(test))) \ $(eval installcheck: $(test).test)) $(foreach test-group, $(install-tests-groups), \ - $(eval $(call run-test-group,$(test-group),$(install_test_init))) \ + $(eval $(call run-test-group,$(test-group))) \ $(eval installcheck: $(test-group).test-group) \ $(foreach test, $($(test-group)-tests), \ - $(eval $(call run-test,$(test),$(install_test_init))) \ + $(eval $(call run-test,$(test))) \ $(eval $(test-group).test-group: $(test).test))) +# Compilation database. +$(foreach lib, $(libraries), $(eval $(call write-compile-commands,$(lib)))) +$(foreach prog, $(programs), $(eval $(call write-compile-commands,$(prog)))) + +compile_commands.json: $(compile-commands-json-files) + @jq --slurp '.' $^ >$@ + # Include makefiles requiring built programs. $(foreach mf, $(makefiles-late), $(eval $(call include-sub-makefile,$(mf)))) diff --git a/mk/patterns.mk b/mk/patterns.mk index c81150260..4caa2039e 100644 --- a/mk/patterns.mk +++ b/mk/patterns.mk @@ -1,11 +1,41 @@ + +# These are the complete command lines we use to compile C and C++ files. +# - $< is the source file. +# - $1 is the object file to create. +CC_CMD=$(CC) -o $1 -c $< $(CPPFLAGS) $(GLOBAL_CFLAGS) $(CFLAGS) $($1_CFLAGS) -MMD -MF $(call filename-to-dep,$1) -MP +CXX_CMD=$(CXX) -o $1 -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($1_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep,$1) -MP + +# We use COMPILE_COMMANDS_JSON_CMD to turn a compilation command (like CC_CMD +# or CXX_CMD above) into a comple_commands.json file. We rely on bash native +# word splitting to define the positional arguments. +# - $< is the source file being compiled. +COMPILE_COMMANDS_JSON_CMD=jq --null-input '{ directory: $$ENV.PWD, file: "$<", arguments: $$ARGS.positional }' --args -- + + $(buildprefix)%.o: %.cc @mkdir -p "$(dir $@)" - $(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cxx) $(call CXX_CMD,$@) $(buildprefix)%.o: %.cpp @mkdir -p "$(dir $@)" - $(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cxx) $(call CXX_CMD,$@) $(buildprefix)%.o: %.c @mkdir -p "$(dir $@)" - $(trace-cc) $(CC) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CFLAGS) $(CFLAGS) $($@_CFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cc) $(call CC_CMD,$@) + +# In the following we need to replace the .compile_commands.json extension in $@ with .o +# to make the object file. This is needed because CC_CMD and CXX_CMD do further expansions +# based on the object file name (i.e. *_CXXFLAGS and filename-to-dep). + +$(buildprefix)%.compile_commands.json: %.cc + @mkdir -p "$(dir $@)" + $(trace-jq) $(COMPILE_COMMANDS_JSON_CMD) $(call CXX_CMD,$(@:.compile_commands.json=.o)) > $@ + +$(buildprefix)%.compile_commands.json: %.cpp + @mkdir -p "$(dir $@)" + $(trace-jq) $(COMPILE_COMMANDS_JSON_CMD) $(call CXX_CMD,$(@:.compile_commands.json=.o)) > $@ + +$(buildprefix)%.compile_commands.json: %.c + @mkdir -p "$(dir $@)" + $(trace-jq) $(COMPILE_COMMANDS_JSON_CMD) $(call CC_CMD,$(@:.compile_commands.json=.o)) > $@ diff --git a/mk/platform.mk b/mk/platform.mk index fe960dedf..22c114a20 100644 --- a/mk/platform.mk +++ b/mk/platform.mk @@ -29,4 +29,8 @@ ifdef HOST_OS HOST_SOLARIS = 1 HOST_UNIX = 1 endif + ifeq ($(HOST_KERNEL), gnu) + HOST_HURD = 1 + HOST_UNIX = 1 + endif endif diff --git a/mk/precompiled-headers.mk b/mk/precompiled-headers.mk index cdd3daecd..f2803eb79 100644 --- a/mk/precompiled-headers.mk +++ b/mk/precompiled-headers.mk @@ -8,7 +8,7 @@ GCH = $(buildprefix)precompiled-headers.h.gch $(GCH): precompiled-headers.h @rm -f $@ @mkdir -p "$(dir $@)" - $(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS) + $(trace-gen) $(CXX) -c -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS) clean-files += $(GCH) diff --git a/mk/run-test.sh b/mk/run-test.sh index da9c5a473..7f9f1d5f8 100755 --- a/mk/run-test.sh +++ b/mk/run-test.sh @@ -8,7 +8,6 @@ yellow="" normal="" test=$1 -init=${2-} dir="$(dirname "${BASH_SOURCE[0]}")" source "$dir/common-test.sh" @@ -22,20 +21,18 @@ if [ -t 1 ]; then fi run_test () { - if [ -n "$init" ]; then - (init_test 2>/dev/null > /dev/null) - fi - log="$(run_test_proper 2>&1)" && status=0 || status=$? + log="$(run "$test" 2>&1)" && status=0 || status=$? } run_test -if [ $status -eq 0 ]; then +if [[ "$status" = 0 ]]; then echo "$post_run_msg [${green}PASS$normal]" -elif [ $status -eq 99 ]; then +elif [[ "$status" = 77 ]]; then echo "$post_run_msg [${yellow}SKIP$normal]" else echo "$post_run_msg [${red}FAIL$normal]" + # shellcheck disable=SC2001 echo "$log" | sed 's/^/ /' exit "$status" fi diff --git a/mk/tests.mk b/mk/tests.mk index bac9b704a..0a10f6d3b 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -12,8 +12,8 @@ endef define run-test - $(eval $(call run-bash,$1.test,$1 $(test-deps),mk/run-test.sh $1 $2)) - $(eval $(call run-bash,$1.test-debug,$1 $(test-deps),mk/debug-test.sh $1 $2)) + $(eval $(call run-bash,$1.test,$1 $(test-deps),mk/run-test.sh $1)) + $(eval $(call run-bash,$1.test-debug,$1 $(test-deps),mk/debug-test.sh $1)) endef diff --git a/mk/tracing.mk b/mk/tracing.mk index 1fc5573d7..09db1e617 100644 --- a/mk/tracing.mk +++ b/mk/tracing.mk @@ -10,6 +10,8 @@ ifeq ($(V), 0) trace-install = @echo " INST " $@; trace-mkdir = @echo " MKDIR " $@; trace-test = @echo " TEST " $@; + trace-sh = @echo " SH " $@; + trace-jq = @echo " JQ " $@; suppress = @ diff --git a/package.nix b/package.nix index 20796a386..a7c8923e8 100644 --- a/package.nix +++ b/package.nix @@ -13,17 +13,16 @@ , curl , editline , readline -, fileset , flex , git , gtest , jq -, doxygen , libarchive , libcpuid , libgit2 , libseccomp , libsodium +, man , lowdown , mdbook , mdbook-linkcheck @@ -33,7 +32,8 @@ , pkg-config , rapidcheck , sqlite -, util-linux +, toml11 +, unixtools , xz , busybox-sandbox-shell ? null @@ -48,10 +48,8 @@ , pname ? "nix" , versionSuffix ? "" -, officialRelease ? false -# Whether to build Nix. Useful to skip for tasks like (a) just -# generating API docs or (b) testing existing pre-built versions of Nix +# Whether to build Nix. Useful to skip for tasks like testing existing pre-built versions of Nix , doBuild ? true # Run the unit tests as part of the build. See `installUnitTests` for an @@ -74,7 +72,10 @@ # sounds so long as evaluation just takes places within short-lived # processes. (When the process exits, the memory is reclaimed; it is # only leaked *within* the process.) -, enableGC ? true +# +# Temporarily disabled on Windows because the `GC_throw_bad_alloc` +# symbol is missing during linking. +, enableGC ? !stdenv.hostPlatform.isWindows # Whether to enable Markdown rendering in the Nix binary. , enableMarkdown ? !stdenv.hostPlatform.isWindows @@ -87,10 +88,6 @@ # - readline , readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline" -# Whether to build the internal API docs, can be done separately from -# everything else. -, enableInternalAPIDocs ? false - # Whether to install unit tests. This is useful when cross compiling # since we cannot run them natively during the build, but can do so # later. @@ -113,6 +110,8 @@ }: let + inherit (lib) fileset; + version = lib.fileContents ./.version + versionSuffix; # selected attributes with defaults, will be used to define some @@ -161,6 +160,8 @@ in { ./m4 # TODO: do we really need README.md? It doesn't seem used in the build. ./README.md + # This could be put behind a conditional + ./maintainers/local.mk # For make, regardless of what we are building ./local.mk ./Makefile @@ -171,17 +172,11 @@ in { ./doc ./misc ./precompiled-headers.h - ./src + (fileset.difference ./src ./src/perl) ./COPYING ./scripts/local.mk - ] ++ lib.optionals buildUnitTests [ + ] ++ lib.optionals enableManual [ ./doc/manual - ] ++ lib.optionals enableInternalAPIDocs [ - ./doc/internal-api - # Source might not be compiled, but still must be available - # for Doxygen to gather comments. - ./src - ./tests/unit ] ++ lib.optionals buildUnitTests [ ./tests/unit ] ++ lib.optionals doInstallCheck [ @@ -195,8 +190,10 @@ in { ++ lib.optional doBuild "dev" # If we are doing just build or just docs, the one thing will use # "out". We only need additional outputs if we are doing both. - ++ lib.optional (doBuild && (enableManual || enableInternalAPIDocs)) "doc" - ++ lib.optional installUnitTests "check"; + ++ lib.optional (doBuild && enableManual) "doc" + ++ lib.optional installUnitTests "check" + ++ lib.optional doCheck "testresults" + ; nativeBuildInputs = [ autoconf-archive @@ -213,14 +210,14 @@ in { git mercurial openssh + man # for testing `nix-* --help` ] ++ lib.optionals (doInstallCheck || enableManual) [ jq # Also for custom mdBook preprocessor. - ] ++ lib.optional stdenv.hostPlatform.isLinux util-linux - ++ lib.optional enableInternalAPIDocs doxygen + ] ++ lib.optional stdenv.hostPlatform.isStatic unixtools.hexdump ; - buildInputs = lib.optionals doBuild [ - boost + buildInputs = lib.optionals doBuild ( + [ brotli bzip2 curl @@ -229,6 +226,7 @@ in { libsodium openssl sqlite + toml11 xz ({ inherit readline editline; }.${readlineFlavor}) ] ++ lib.optionals enableMarkdown [ @@ -240,46 +238,22 @@ in { ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid # There have been issues building these dependencies ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin)) - (aws-sdk-cpp.override { - apis = ["s3" "transfer"]; - customMemoryManagement = false; - }) - ; + aws-sdk-cpp + ); - propagatedBuildInputs = [ + propagatedBuildInputs = lib.optionals doBuild ([ + boost nlohmann_json - ] ++ lib.optional enableGC boehmgc; + ] ++ lib.optional enableGC boehmgc + ); dontBuild = !attrs.doBuild; doCheck = attrs.doCheck; - disallowedReferences = [ boost ]; - - preConfigure = lib.optionalString (doBuild && ! stdenv.hostPlatform.isStatic) ( - '' - # Copy libboost_context so we don't get all of Boost in our closure. - # https://github.com/NixOS/nixpkgs/issues/45462 - mkdir -p $out/lib - cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib - rm -f $out/lib/*.a - '' + lib.optionalString stdenv.hostPlatform.isLinux '' - chmod u+w $out/lib/*.so.* - patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* - '' + lib.optionalString stdenv.hostPlatform.isDarwin '' - for LIB in $out/lib/*.dylib; do - chmod u+w $LIB - install_name_tool -id $LIB $LIB - install_name_tool -delete_rpath ${boost}/lib/ $LIB || true - done - install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib - '' - ); - configureFlags = [ (lib.enableFeature doBuild "build") (lib.enableFeature buildUnitTests "unit-tests") (lib.enableFeature doInstallCheck "functional-tests") - (lib.enableFeature enableInternalAPIDocs "internal-api-docs") (lib.enableFeature enableManual "doc-gen") (lib.enableFeature enableGC "gc") (lib.enableFeature enableMarkdown "markdown") @@ -303,8 +277,11 @@ in { makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1"; - installTargets = lib.optional doBuild "install" - ++ lib.optional enableInternalAPIDocs "internal-api-html"; + preCheck = '' + mkdir $testresults + ''; + + installTargets = lib.optional doBuild "install"; installFlags = "sysconfdir=$(out)/etc"; @@ -319,18 +296,16 @@ in { lib.optionalString stdenv.hostPlatform.isStatic '' mkdir -p $out/nix-support echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products - '' + lib.optionalString stdenv.isDarwin '' - install_name_tool \ - -change ${boost}/lib/libboost_context.dylib \ - $out/lib/libboost_context.dylib \ - $out/lib/libnixutil.dylib '' ) + lib.optionalString enableManual '' mkdir -p ''${!outputDoc}/nix-support echo "doc manual ''${!outputDoc}/share/doc/nix/manual" >> ''${!outputDoc}/nix-support/hydra-build-products - '' + lib.optionalString enableInternalAPIDocs '' - mkdir -p ''${!outputDoc}/nix-support - echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products + ''; + + # So the check output gets links for DLLs in the out output. + preFixup = lib.optionalString (stdenv.hostPlatform.isWindows && builtins.elem "check" finalAttrs.outputs) '' + ln -s "$check/lib/"*.dll "$check/bin" + ln -s "$out/bin/"*.dll "$check/bin" ''; doInstallCheck = attrs.doInstallCheck; @@ -341,20 +316,26 @@ in { # Work around weird bug where it doesn't think there is a Makefile. installCheckPhase = if (!doBuild && doInstallCheck) then '' + runHook preInstallCheck mkdir -p src/nix-channel make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES '' else null; # Needed for tests if we are not doing a build, but testing existing # built Nix. - preInstallCheck = lib.optionalString (! doBuild) '' - mkdir -p src/nix-channel - ''; + preInstallCheck = + lib.optionalString (! doBuild) '' + mkdir -p src/nix-channel + '' + # See https://github.com/NixOS/nix/issues/2523 + # Occurs often in tests since https://github.com/NixOS/nix/pull/9900 + + lib.optionalString stdenv.hostPlatform.isDarwin '' + export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES + ''; separateDebugInfo = !stdenv.hostPlatform.isStatic; - # TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated - # to work with `strictDeps`. + # TODO Always true after https://github.com/NixOS/nixpkgs/issues/318564 strictDeps = !withCoverageChecks; hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; diff --git a/packaging/components.nix b/packaging/components.nix new file mode 100644 index 000000000..870e9ae61 --- /dev/null +++ b/packaging/components.nix @@ -0,0 +1,43 @@ +scope: +let + inherit (scope) callPackage; +in + +# This becomes the pkgs.nixComponents attribute set +{ + nix = callPackage ../package.nix { }; + + nix-util = callPackage ../src/libutil/package.nix { }; + nix-util-c = callPackage ../src/libutil-c/package.nix { }; + nix-util-test-support = callPackage ../tests/unit/libutil-support/package.nix { }; + nix-util-tests = callPackage ../tests/unit/libutil/package.nix { }; + + nix-store = callPackage ../src/libstore/package.nix { }; + nix-store-c = callPackage ../src/libstore-c/package.nix { }; + nix-store-test-support = callPackage ../tests/unit/libstore-support/package.nix { }; + nix-store-tests = callPackage ../tests/unit/libstore/package.nix { }; + + nix-fetchers = callPackage ../src/libfetchers/package.nix { }; + nix-fetchers-tests = callPackage ../tests/unit/libfetchers/package.nix { }; + + nix-expr = callPackage ../src/libexpr/package.nix { }; + nix-expr-c = callPackage ../src/libexpr-c/package.nix { }; + nix-expr-test-support = callPackage ../tests/unit/libexpr-support/package.nix { }; + nix-expr-tests = callPackage ../tests/unit/libexpr/package.nix { }; + + nix-flake = callPackage ../src/libflake/package.nix { }; + nix-flake-tests = callPackage ../tests/unit/libflake/package.nix { }; + + nix-main = callPackage ../src/libmain/package.nix { }; + nix-main-c = callPackage ../src/libmain-c/package.nix { }; + + nix-cmd = callPackage ../src/libcmd/package.nix { }; + + # Will replace `nix` once the old build system is gone. + nix-ng = callPackage ../src/nix/package.nix { }; + + nix-internal-api-docs = callPackage ../src/internal-api-docs/package.nix { }; + nix-external-api-docs = callPackage ../src/external-api-docs/package.nix { }; + + nix-perl-bindings = callPackage ../src/perl/package.nix { }; +} diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix new file mode 100644 index 000000000..e0737593f --- /dev/null +++ b/packaging/dependencies.nix @@ -0,0 +1,165 @@ +# These overrides are applied to the dependencies of the Nix components. + +{ + # Flake inputs; used for sources + inputs, + + # The raw Nixpkgs, not affected by this scope + pkgs, + + stdenv, + versionSuffix, +}: + +let + prevStdenv = stdenv; +in + +let + inherit (pkgs) lib; + + root = ../.; + + stdenv = if prevStdenv.isDarwin && prevStdenv.isx86_64 + then darwinStdenv + else prevStdenv; + + # Fix the following error with the default x86_64-darwin SDK: + # + # error: aligned allocation function of type 'void *(std::size_t, std::align_val_t)' is only available on macOS 10.13 or newer + # + # Despite the use of the 10.13 deployment target here, the aligned + # allocation function Clang uses with this setting actually works + # all the way back to 10.6. + darwinStdenv = pkgs.overrideSDK prevStdenv { darwinMinVersion = "10.13"; }; + + # Nixpkgs implements this by returning a subpath into the fetched Nix sources. + resolvePath = p: p; + + # Indirection for Nixpkgs to override when package.nix files are vendored + filesetToSource = lib.fileset.toSource; + + localSourceLayer = finalAttrs: prevAttrs: + let + workDirPath = + # Ideally we'd pick finalAttrs.workDir, but for now `mkDerivation` has + # the requirement that everything except passthru and meta must be + # serialized by mkDerivation, which doesn't work for this. + prevAttrs.workDir; + + workDirSubpath = lib.path.removePrefix root workDirPath; + sources = assert prevAttrs.fileset._type == "fileset"; prevAttrs.fileset; + src = lib.fileset.toSource { fileset = sources; inherit root; }; + + in + { + sourceRoot = "${src.name}/" + workDirSubpath; + inherit src; + + # Clear what `derivation` can't/shouldn't serialize; see prevAttrs.workDir. + fileset = null; + workDir = null; + }; + + # Work around weird `--as-needed` linker behavior with BSD, see + # https://github.com/mesonbuild/meson/issues/3593 + bsdNoLinkAsNeeded = finalAttrs: prevAttrs: + lib.optionalAttrs stdenv.hostPlatform.isBSD { + mesonFlags = [ (lib.mesonBool "b_asneeded" false) ] ++ prevAttrs.mesonFlags or []; + }; + + miscGoodPractice = finalAttrs: prevAttrs: + { + strictDeps = prevAttrs.strictDeps or true; + enableParallelBuilding = true; + }; + +in +scope: { + inherit stdenv versionSuffix; + version = lib.fileContents ../.version + versionSuffix; + + aws-sdk-cpp = (pkgs.aws-sdk-cpp.override { + apis = [ "s3" "transfer" ]; + customMemoryManagement = false; + }).overrideAttrs { + # only a stripped down version is built, which takes a lot less resources + # to build, so we don't need a "big-parallel" machine. + requiredSystemFeatures = [ ]; + }; + + libseccomp = pkgs.libseccomp.overrideAttrs (_: rec { + version = "2.5.5"; + src = pkgs.fetchurl { + url = "https://github.com/seccomp/libseccomp/releases/download/v${version}/libseccomp-${version}.tar.gz"; + hash = "sha256-JIosik2bmFiqa69ScSw0r+/PnJ6Ut23OAsHJqiX7M3U="; + }; + }); + + boehmgc = pkgs.boehmgc.override { + enableLargeConfig = true; + }; + + # TODO Hack until https://github.com/NixOS/nixpkgs/issues/45462 is fixed. + boost = (pkgs.boost.override { + extraB2Args = [ + "--with-container" + "--with-context" + "--with-coroutine" + ]; + }).overrideAttrs (old: { + # Need to remove `--with-*` to use `--with-libraries=...` + buildPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.buildPhase; + installPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.installPhase; + }); + + libgit2 = pkgs.libgit2.overrideAttrs (attrs: { + src = inputs.libgit2; + version = inputs.libgit2.lastModifiedDate; + cmakeFlags = attrs.cmakeFlags or [] + ++ [ "-DUSE_SSH=exec" ]; + }); + + busybox-sandbox-shell = pkgs.busybox-sandbox-shell or (pkgs.busybox.override { + useMusl = true; + enableStatic = true; + enableMinimal = true; + extraConfig = '' + CONFIG_FEATURE_FANCY_ECHO y + CONFIG_FEATURE_SH_MATH y + CONFIG_FEATURE_SH_MATH_64 y + + CONFIG_ASH y + CONFIG_ASH_OPTIMIZE_FOR_SIZE y + + CONFIG_ASH_ALIAS y + CONFIG_ASH_BASH_COMPAT y + CONFIG_ASH_CMDCMD y + CONFIG_ASH_ECHO y + CONFIG_ASH_GETOPTS y + CONFIG_ASH_INTERNAL_GLOB y + CONFIG_ASH_JOB_CONTROL y + CONFIG_ASH_PRINTF y + CONFIG_ASH_TEST y + ''; + }); + + # TODO change in Nixpkgs, Windows works fine. First commit of + # https://github.com/NixOS/nixpkgs/pull/322977 backported will fix. + toml11 = pkgs.toml11.overrideAttrs (old: { + meta.platforms = lib.platforms.all; + }); + + inherit resolvePath filesetToSource; + + mkMesonDerivation = f: let + exts = [ + miscGoodPractice + bsdNoLinkAsNeeded + localSourceLayer + ]; + in stdenv.mkDerivation + (lib.extends + (lib.foldr lib.composeExtensions (_: _: {}) exts) + f); +} diff --git a/packaging/hydra.nix b/packaging/hydra.nix new file mode 100644 index 000000000..dbe992476 --- /dev/null +++ b/packaging/hydra.nix @@ -0,0 +1,200 @@ +{ inputs +, binaryTarball +, forAllCrossSystems +, forAllSystems +, lib +, linux64BitSystems +, nixpkgsFor +, self +}: +let + inherit (inputs) nixpkgs nixpkgs-regression; + + installScriptFor = tarballs: + nixpkgsFor.x86_64-linux.native.callPackage ../scripts/installer.nix { + inherit tarballs; + }; + + testNixVersions = pkgs: client: daemon: + pkgs.callPackage ../package.nix { + pname = + "nix-tests" + + lib.optionalString + (lib.versionAtLeast daemon.version "2.4pre20211005" && + lib.versionAtLeast client.version "2.4pre20211005") + "-${client.version}-against-${daemon.version}"; + + test-client = client; + test-daemon = daemon; + + doBuild = false; + }; + + # Technically we could just return `pkgs.nixComponents`, but for Hydra it's + # convention to transpose it, and to transpose it efficiently, we need to + # enumerate them manually, so that we don't evaluate unnecessary package sets. + forAllPackages = lib.genAttrs [ + "nix" + "nix-util" + "nix-util-c" + "nix-util-test-support" + "nix-util-tests" + "nix-store" + "nix-store-c" + "nix-store-test-support" + "nix-store-tests" + "nix-fetchers" + "nix-fetchers-tests" + "nix-expr" + "nix-expr-c" + "nix-expr-test-support" + "nix-expr-tests" + "nix-flake" + "nix-flake-tests" + "nix-main" + "nix-main-c" + "nix-cmd" + "nix-ng" + ]; +in +{ + # Binary package for various platforms. + build = forAllPackages (pkgName: + forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.${pkgName})); + + shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation); + + buildStatic = forAllPackages (pkgName: + lib.genAttrs linux64BitSystems (system: nixpkgsFor.${system}.static.nixComponents.${pkgName})); + + buildCross = forAllPackages (pkgName: + forAllCrossSystems (crossSystem: + lib.genAttrs [ "x86_64-linux" ] (system: nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName}))); + + buildNoGc = forAllSystems (system: + self.packages.${system}.nix.override { enableGC = false; } + ); + + buildNoTests = forAllSystems (system: nixpkgsFor.${system}.native.nix_noTests); + + # Toggles some settings for better coverage. Windows needs these + # library combinations, and Debian build Nix with GNU readline too. + buildReadlineNoMarkdown = forAllSystems (system: + self.packages.${system}.nix.override { + enableMarkdown = false; + readlineFlavor = "readline"; + } + ); + + # Perl bindings for various platforms. + perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.nix-perl-bindings); + + # Binary tarball for various platforms, containing a Nix store + # with the closure of 'nix' package, and the second half of + # the installation script. + binaryTarball = forAllSystems (system: binaryTarball nixpkgsFor.${system}.native.nix nixpkgsFor.${system}.native); + + binaryTarballCross = lib.genAttrs [ "x86_64-linux" ] (system: + forAllCrossSystems (crossSystem: + binaryTarball + nixpkgsFor.${system}.cross.${crossSystem}.nix + nixpkgsFor.${system}.cross.${crossSystem})); + + # The first half of the installation script. This is uploaded + # to https://nixos.org/nix/install. It downloads the binary + # tarball for the user's system and calls the second half of the + # installation script. + installerScript = installScriptFor [ + # Native + self.hydraJobs.binaryTarball."x86_64-linux" + self.hydraJobs.binaryTarball."i686-linux" + self.hydraJobs.binaryTarball."aarch64-linux" + self.hydraJobs.binaryTarball."x86_64-darwin" + self.hydraJobs.binaryTarball."aarch64-darwin" + # Cross + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu" + ]; + installerScriptForGHA = installScriptFor [ + # Native + self.hydraJobs.binaryTarball."x86_64-linux" + self.hydraJobs.binaryTarball."aarch64-darwin" + # Cross + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu" + ]; + + # docker image with Nix inside + dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); + + # Line coverage analysis. + coverage = nixpkgsFor.x86_64-linux.native.nix.override { + pname = "nix-coverage"; + withCoverageChecks = true; + }; + + # API docs for Nix's unstable internal C++ interfaces. + internal-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-internal-api-docs; + + # API docs for Nix's C bindings. + external-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-external-api-docs; + + # System tests. + tests = import ../tests/nixos { inherit lib nixpkgs nixpkgsFor self; } // { + + # Make sure that nix-env still produces the exact same result + # on a particular version of Nixpkgs. + evalNixpkgs = + let + inherit (nixpkgsFor.x86_64-linux.native) runCommand nix; + in + runCommand "eval-nixos" { buildInputs = [ nix ]; } + '' + type -p nix-env + # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593. + ( + set -x + time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages + [[ $(sha1sum < packages | cut -c1-40) = e01b031fc9785a572a38be6bc473957e3b6faad7 ]] + ) + mkdir $out + ''; + + nixpkgsLibTests = + forAllSystems (system: + import (nixpkgs + "/lib/tests/test-with-nix.nix") + { + lib = nixpkgsFor.${system}.native.lib; + nix = self.packages.${system}.nix; + pkgs = nixpkgsFor.${system}.native; + } + ); + }; + + metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" { + pkgs = nixpkgsFor.x86_64-linux.native; + nixpkgs = nixpkgs-regression; + }; + + installTests = forAllSystems (system: + let pkgs = nixpkgsFor.${system}.native; in + pkgs.runCommand "install-tests" + { + againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix; + againstCurrentLatest = + # FIXME: temporarily disable this on macOS because of #3605. + if system == "x86_64-linux" + then testNixVersions pkgs pkgs.nix pkgs.nixVersions.latest + else null; + # Disabled because the latest stable version doesn't handle + # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work + # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable; + } "touch $out"); + + installerTests = import ../tests/installer { + binaryTarballs = self.hydraJobs.binaryTarball; + inherit nixpkgsFor; + }; +} diff --git a/perl/.yath.rc b/perl/.yath.rc deleted file mode 100644 index 118bf80c8..000000000 --- a/perl/.yath.rc +++ /dev/null @@ -1,2 +0,0 @@ -[test] --I=rel(lib/Nix) diff --git a/perl/Makefile b/perl/Makefile deleted file mode 100644 index 832668dd1..000000000 --- a/perl/Makefile +++ /dev/null @@ -1,21 +0,0 @@ -makefiles = local.mk - -GLOBAL_CXXFLAGS += -g -Wall -std=c++2a - -# A convenience for concurrent development of Nix and its Perl bindings. -# Not needed in a standalone build of the Perl bindings. -ifneq ("$(wildcard ../src)", "") - GLOBAL_CXXFLAGS += -I ../src -endif - --include Makefile.config - -OPTIMIZE = 1 - -ifeq ($(OPTIMIZE), 1) - GLOBAL_CXXFLAGS += -O3 -else - GLOBAL_CXXFLAGS += -O0 -endif - -include mk/lib.mk diff --git a/perl/Makefile.config.in b/perl/Makefile.config.in deleted file mode 100644 index d856de3ad..000000000 --- a/perl/Makefile.config.in +++ /dev/null @@ -1,18 +0,0 @@ -HOST_OS = @host_os@ -CC = @CC@ -CFLAGS = @CFLAGS@ -CXX = @CXX@ -CXXFLAGS = @CXXFLAGS@ -PACKAGE_NAME = @PACKAGE_NAME@ -PACKAGE_VERSION = @PACKAGE_VERSION@ -SODIUM_LIBS = @SODIUM_LIBS@ -NIX_CFLAGS = @NIX_CFLAGS@ -NIX_LIBS = @NIX_LIBS@ -nixbindir = @nixbindir@ -curl = @curl@ -nixlibexecdir = @nixlibexecdir@ -nixlocalstatedir = @nixlocalstatedir@ -perl = @perl@ -perllibdir = @perllibdir@ -nixstoredir = @nixstoredir@ -nixsysconfdir = @nixsysconfdir@ diff --git a/perl/configure.ac b/perl/configure.ac deleted file mode 100644 index a02cb06c9..000000000 --- a/perl/configure.ac +++ /dev/null @@ -1,84 +0,0 @@ -AC_INIT(nix-perl, m4_esyscmd([bash -c "echo -n $(cat ../.version)$VERSION_SUFFIX"])) -AC_CONFIG_SRCDIR(MANIFEST) -AC_CONFIG_AUX_DIR(../config) - -CFLAGS= -CXXFLAGS= -AC_PROG_CC -AC_PROG_CXX - -AC_CANONICAL_HOST - -# Use 64-bit file system calls so that we can support files > 2 GiB. -AC_SYS_LARGEFILE - -AC_DEFUN([NEED_PROG], -[ -AC_PATH_PROG($1, $2) -if test -z "$$1"; then - AC_MSG_ERROR([$2 is required]) -fi -]) - -NEED_PROG(perl, perl) -NEED_PROG(curl, curl) -NEED_PROG(bzip2, bzip2) -NEED_PROG(xz, xz) - -# Test that Perl has the open/fork feature (Perl 5.8.0 and beyond). -AC_MSG_CHECKING([whether Perl is recent enough]) -if ! $perl -e 'open(FOO, "-|", "true"); while () { print; }; close FOO or die;'; then - AC_MSG_RESULT(no) - AC_MSG_ERROR([Your Perl version is too old. Nix requires Perl 5.8.0 or newer.]) -fi -AC_MSG_RESULT(yes) - - -# Figure out where to install Perl modules. -AC_MSG_CHECKING([for the Perl installation prefix]) -perlversion=$($perl -e 'use Config; print $Config{version};') -perlarchname=$($perl -e 'use Config; print $Config{archname};') -AC_SUBST(perllibdir, [${libdir}/perl5/site_perl/$perlversion/$perlarchname]) -AC_MSG_RESULT($perllibdir) - -# Look for libsodium. -PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"]) - -# Check for the required Perl dependencies (DBI and DBD::SQLite). -perlFlags="-I$perllibdir" - -AC_ARG_WITH(dbi, AC_HELP_STRING([--with-dbi=PATH], - [prefix of the Perl DBI library]), - perlFlags="$perlFlags -I$withval") - -AC_ARG_WITH(dbd-sqlite, AC_HELP_STRING([--with-dbd-sqlite=PATH], - [prefix of the Perl DBD::SQLite library]), - perlFlags="$perlFlags -I$withval") - -AC_MSG_CHECKING([whether DBD::SQLite works]) -if ! $perl $perlFlags -e 'use DBI; use DBD::SQLite;' 2>&5; then - AC_MSG_RESULT(no) - AC_MSG_FAILURE([The Perl modules DBI and/or DBD::SQLite are missing.]) -fi -AC_MSG_RESULT(yes) - -AC_SUBST(perlFlags) - -PKG_CHECK_MODULES([NIX], [nix-store]) - -NEED_PROG([NIX], [nix]) - -# Expand all variables in config.status. -test "$prefix" = NONE && prefix=$ac_default_prefix -test "$exec_prefix" = NONE && exec_prefix='${prefix}' -for name in $ac_subst_vars; do - declare $name="$(eval echo "${!name}")" - declare $name="$(eval echo "${!name}")" - declare $name="$(eval echo "${!name}")" -done - -rm -f Makefile.config -ln -sfn ../mk mk - -AC_CONFIG_FILES([]) -AC_OUTPUT diff --git a/perl/default.nix b/perl/default.nix deleted file mode 100644 index 7103574c9..000000000 --- a/perl/default.nix +++ /dev/null @@ -1,61 +0,0 @@ -{ lib, fileset -, stdenv -, perl, perlPackages -, autoconf-archive, autoreconfHook, pkg-config -, nix, curl, bzip2, xz, boost, libsodium, darwin -}: - -perl.pkgs.toPerlModule (stdenv.mkDerivation (finalAttrs: { - name = "nix-perl-${nix.version}"; - - src = fileset.toSource { - root = ../.; - fileset = fileset.unions ([ - ../.version - ../m4 - ../mk - ./MANIFEST - ./Makefile - ./Makefile.config.in - ./configure.ac - ./lib - ./local.mk - ] ++ lib.optionals finalAttrs.doCheck [ - ./.yath.rc - ./t - ]); - }; - - nativeBuildInputs = - [ autoconf-archive - autoreconfHook - pkg-config - ]; - - buildInputs = - [ nix - curl - bzip2 - xz - perl - boost - ] - ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium - ++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security; - - # `perlPackages.Test2Harness` is marked broken for Darwin - doCheck = !stdenv.isDarwin; - - nativeCheckInputs = [ - perlPackages.Test2Harness - ]; - - configureFlags = [ - "--with-dbi=${perlPackages.DBI}/${perl.libPrefix}" - "--with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix}" - ]; - - enableParallelBuilding = true; - - postUnpack = "sourceRoot=$sourceRoot/perl"; -})) diff --git a/perl/local.mk b/perl/local.mk deleted file mode 100644 index ed4764eb9..000000000 --- a/perl/local.mk +++ /dev/null @@ -1,46 +0,0 @@ -nix_perl_sources := \ - lib/Nix/Store.pm \ - lib/Nix/Manifest.pm \ - lib/Nix/SSH.pm \ - lib/Nix/CopyClosure.pm \ - lib/Nix/Config.pm.in \ - lib/Nix/Utils.pm - -nix_perl_modules := $(nix_perl_sources:.in=) - -$(foreach x, $(nix_perl_modules), $(eval $(call install-data-in, $(x), $(perllibdir)/Nix))) - -lib/Nix/Store.cc: lib/Nix/Store.xs - $(trace-gen) xsubpp $^ -output $@ - -libraries += Store - -Store_DIR := lib/Nix - -Store_SOURCES := $(Store_DIR)/Store.cc - -Store_CXXFLAGS = \ - $(NIX_CFLAGS) \ - -I$(shell perl -e 'use Config; print $$Config{archlibexp};')/CORE \ - -D_FILE_OFFSET_BITS=64 \ - -Wno-unknown-warning-option -Wno-unused-variable -Wno-literal-suffix \ - -Wno-reserved-user-defined-literal -Wno-duplicate-decl-specifier -Wno-pointer-bool-conversion - -Store_LDFLAGS := $(SODIUM_LIBS) $(NIX_LIBS) - -ifdef HOST_CYGWIN - archlib = $(shell perl -E 'use Config; print $$Config{archlib};') - libperl = $(shell perl -E 'use Config; print $$Config{libperl};') - Store_LDFLAGS += $(shell find ${archlib} -name ${libperl}) -endif - -Store_ALLOW_UNDEFINED = 1 - -Store_FORCE_INSTALL = 1 - -Store_INSTALL_DIR = $(perllibdir)/auto/Nix/Store - -clean-files += lib/Nix/Config.pm lib/Nix/Store.cc Makefile.config - -check: all - yath test diff --git a/precompiled-headers.h b/precompiled-headers.h index f52f1cab8..e1a3f8cc0 100644 --- a/precompiled-headers.h +++ b/precompiled-headers.h @@ -42,19 +42,22 @@ #include #include #include -#include -#include -#include #include -#include -#include -#include #include #include #include -#include -#include -#include #include +#ifndef _WIN32 +# include +# include +# include +# include +# include +# include +# include +# include +# include +#endif + #include diff --git a/scripts/bigsur-nixbld-user-migration.sh b/scripts/bigsur-nixbld-user-migration.sh index f1619fd56..0eb312e07 100755 --- a/scripts/bigsur-nixbld-user-migration.sh +++ b/scripts/bigsur-nixbld-user-migration.sh @@ -3,7 +3,7 @@ ((NEW_NIX_FIRST_BUILD_UID=301)) id_available(){ - dscl . list /Users UniqueID | grep -E '\b'$1'\b' >/dev/null + dscl . list /Users UniqueID | grep -E '\b'"$1"'\b' >/dev/null } change_nixbld_names_and_ids(){ @@ -26,18 +26,18 @@ change_nixbld_names_and_ids(){ fi done - if [[ $name == _* ]]; then + if [[ "$name" == _* ]]; then echo " It looks like $name has already been renamed--skipping." else # first 3 are cleanup, it's OK if they aren't here - sudo dscl . delete /Users/$name dsAttrTypeNative:_writers_passwd &>/dev/null || true - sudo dscl . change /Users/$name NFSHomeDirectory "/private/var/empty 1" "/var/empty" &>/dev/null || true + sudo dscl . delete "/Users/$name" dsAttrTypeNative:_writers_passwd &>/dev/null || true + sudo dscl . change "/Users/$name" NFSHomeDirectory "/private/var/empty 1" "/var/empty" &>/dev/null || true # remove existing user from group - sudo dseditgroup -o edit -t user -d $name nixbld || true - sudo dscl . change /Users/$name UniqueID $uid $next_id - sudo dscl . change /Users/$name RecordName $name _$name + sudo dseditgroup -o edit -t user -d "$name" nixbld || true + sudo dscl . change "/Users/$name" UniqueID "$uid" "$next_id" + sudo dscl . change "/Users/$name" RecordName "$name" "_$name" # add renamed user to group - sudo dseditgroup -o edit -t user -a _$name nixbld + sudo dseditgroup -o edit -t user -a "_$name" nixbld echo " $name migrated to _$name (uid: $next_id)" fi done < <(dscl . list /Users UniqueID | grep nixbld | sort -n -k2) diff --git a/scripts/check-hydra-status.sh b/scripts/check-hydra-status.sh deleted file mode 100644 index e62705e94..000000000 --- a/scripts/check-hydra-status.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail -# set -x - - -# mapfile BUILDS_FOR_LATEST_EVAL < <( -# curl -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master/evals | \ -# jq -r '.evals[0].builds[] | @sh') -BUILDS_FOR_LATEST_EVAL=$( -curl -sS -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master/evals | \ - jq -r '.evals[0].builds[]') - -someBuildFailed=0 - -for buildId in $BUILDS_FOR_LATEST_EVAL; do - buildInfo=$(curl --fail -sS -H 'Accept: application/json' "https://hydra.nixos.org/build/$buildId") - - finished=$(echo "$buildInfo" | jq -r '.finished') - - if [[ $finished = 0 ]]; then - continue - fi - - buildStatus=$(echo "$buildInfo" | jq -r '.buildstatus') - - if [[ $buildStatus != 0 ]]; then - someBuildFailed=1 - echo "Job “$(echo "$buildInfo" | jq -r '.job')†failed on hydra: $buildInfo" - fi -done - -exit "$someBuildFailed" diff --git a/scripts/flake-regressions.sh b/scripts/flake-regressions.sh new file mode 100755 index 000000000..d76531134 --- /dev/null +++ b/scripts/flake-regressions.sh @@ -0,0 +1,27 @@ +#! /usr/bin/env bash + +set -e + +echo "Nix version:" +nix --version + +cd flake-regressions + +status=0 + +flakes=$(find tests -mindepth 3 -maxdepth 3 -type d -not -path '*/.*' | sort | head -n25) + +echo "Running flake tests..." + +for flake in $flakes; do + + if ! REGENERATE=0 ./eval-flake.sh "$flake"; then + status=1 + echo "⌠$flake" + else + echo "✅ $flake" + fi + +done + +exit "$status" diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index ad3ee8881..6aee073e3 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -754,7 +754,7 @@ I will: (if it does, I will tell you how to clean them up.) - create local users (see the list above for the users I'll make) - create a local group ($NIX_BUILD_GROUP_NAME) - - install Nix in to $NIX_ROOT + - install Nix in $NIX_ROOT - create a configuration file in /etc/nix - set up the "default profile" by creating some Nix-related files in $ROOT_HOME diff --git a/scripts/install-systemd-multi-user.sh b/scripts/install-systemd-multi-user.sh index 202a9bb54..a62ed7e3a 100755 --- a/scripts/install-systemd-multi-user.sh +++ b/scripts/install-systemd-multi-user.sh @@ -35,7 +35,7 @@ escape_systemd_env() { # Gather all non-empty proxy environment variables into a string create_systemd_proxy_env() { - vars="http_proxy https_proxy ftp_proxy no_proxy HTTP_PROXY HTTPS_PROXY FTP_PROXY NO_PROXY" + vars="http_proxy https_proxy ftp_proxy all_proxy no_proxy HTTP_PROXY HTTPS_PROXY FTP_PROXY ALL_PROXY NO_PROXY" for v in $vars; do if [ "x${!v:-}" != "x" ]; then echo "Environment=${v}=$(escape_systemd_env ${!v})" diff --git a/scripts/install.in b/scripts/install.in index 7d2e52b26..b4e808d8e 100755 --- a/scripts/install.in +++ b/scripts/install.in @@ -50,6 +50,11 @@ case "$(uname -s).$(uname -m)" in path=@tarballPath_armv7l-linux@ system=armv7l-linux ;; + Linux.riscv64) + hash=@tarballHash_riscv64-linux@ + path=@tarballPath_riscv64-linux@ + system=riscv64-linux + ;; Darwin.x86_64) hash=@tarballHash_x86_64-darwin@ path=@tarballPath_x86_64-darwin@ diff --git a/scripts/nix-profile-daemon.fish.in b/scripts/nix-profile-daemon.fish.in index c23aa64f0..346dce5dd 100644 --- a/scripts/nix-profile-daemon.fish.in +++ b/scripts/nix-profile-daemon.fish.in @@ -28,7 +28,7 @@ else end # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. -if test -n "$NIX_SSH_CERT_FILE" +if test -n "$NIX_SSL_CERT_FILE" : # Allow users to override the NIX_SSL_CERT_FILE else if test -e /etc/ssl/certs/ca-certificates.crt # NixOS, Ubuntu, Debian, Gentoo, Arch set --export NIX_SSL_CERT_FILE /etc/ssl/certs/ca-certificates.crt @@ -44,7 +44,7 @@ else if test -e "$NIX_LINK/etc/ca-bundle.crt" # old cacert in Nix profile set --export NIX_SSL_CERT_FILE "$NIX_LINK/etc/ca-bundle.crt" else # Fall back to what is in the nix profiles, favouring whatever is defined last. - for i in $NIX_PROFILES + for i in (string split ' ' $NIX_PROFILES) if test -e "$i/etc/ssl/certs/ca-bundle.crt" set --export NIX_SSL_CERT_FILE "$i/etc/ssl/certs/ca-bundle.crt" end diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index d256b24ed..eb124c0b5 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -1,4 +1,5 @@ # Only execute this file once per shell. +# This file is tested by tests/installer/default.nix. if [ -n "${__ETC_PROFILE_NIX_SOURCED:-}" ]; then return; fi __ETC_PROFILE_NIX_SOURCED=1 @@ -9,11 +10,9 @@ else NIX_LINK_NEW=$HOME/.local/state/nix/profile fi if [ -e "$NIX_LINK_NEW" ]; then - NIX_LINK="$NIX_LINK_NEW" -else - if [ -t 2 ] && [ -e "$NIX_LINK_NEW" ]; then + if [ -t 2 ] && [ -e "$NIX_LINK" ]; then warning="\033[1;35mwarning:\033[0m" - printf "$warning Both %s and legacy %s exist; using the latter.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2 + printf "$warning Both %s and legacy %s exist; using the former.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2 if [ "$(realpath "$NIX_LINK")" = "$(realpath "$NIX_LINK_NEW")" ]; then printf " Since the profiles match, you can safely delete either of them.\n" 1>&2 else @@ -26,6 +25,7 @@ else printf "$warning Profiles do not match. You should manually migrate from %s to %s.\n" "$NIX_LINK" "$NIX_LINK_NEW" 1>&2 fi fi + NIX_LINK="$NIX_LINK_NEW" fi export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK" @@ -69,4 +69,4 @@ else fi export PATH="$NIX_LINK/bin:@localstatedir@/nix/profiles/default/bin:$PATH" -unset NIX_LINK +unset NIX_LINK NIX_LINK_NEW diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index 44bc96e89..e868399b1 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -1,3 +1,4 @@ +# This file is tested by tests/installer/default.nix. if [ -n "$HOME" ] && [ -n "$USER" ]; then # Set up the per-user profile. @@ -9,11 +10,9 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then NIX_LINK_NEW="$HOME/.local/state/nix/profile" fi if [ -e "$NIX_LINK_NEW" ]; then - NIX_LINK="$NIX_LINK_NEW" - else - if [ -t 2 ] && [ -e "$NIX_LINK_NEW" ]; then + if [ -t 2 ] && [ -e "$NIX_LINK" ]; then warning="\033[1;35mwarning:\033[0m" - printf "$warning Both %s and legacy %s exist; using the latter.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2 + printf "$warning Both %s and legacy %s exist; using the former.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2 if [ "$(realpath "$NIX_LINK")" = "$(realpath "$NIX_LINK_NEW")" ]; then printf " Since the profiles match, you can safely delete either of them.\n" 1>&2 else @@ -26,6 +25,7 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then printf "$warning Profiles do not match. You should manually migrate from %s to %s.\n" "$NIX_LINK" "$NIX_LINK_NEW" 1>&2 fi fi + NIX_LINK="$NIX_LINK_NEW" fi # Set up environment. diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 118468477..82ad7d862 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -11,11 +11,13 @@ #include "machines.hh" #include "shared.hh" +#include "plugin.hh" #include "pathlocks.hh" #include "globals.hh" #include "serialise.hh" #include "build-result.hh" #include "store-api.hh" +#include "strings.hh" #include "derivations.hh" #include "local-store.hh" #include "legacy.hh" @@ -37,7 +39,7 @@ static std::string currentLoad; static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot) { - return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true); + return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri.render()), slot), true); } static bool allSupportedLocally(Store & store, const std::set& requiredFeatures) { @@ -135,7 +137,7 @@ static int main_build_remote(int argc, char * * argv) Machine * bestMachine = nullptr; uint64_t bestLoad = 0; for (auto & m : machines) { - debug("considering building on remote machine '%s'", m.storeUri); + debug("considering building on remote machine '%s'", m.storeUri.render()); if (m.enabled && m.systemSupported(neededSystem) && @@ -202,7 +204,7 @@ static int main_build_remote(int argc, char * * argv) else drvstr = ""; - auto error = HintFmt(errorText); + auto error = HintFmt::fromFormatString(errorText); error % drvstr % neededSystem @@ -232,17 +234,16 @@ static int main_build_remote(int argc, char * * argv) lock = -1; try { + storeUri = bestMachine->storeUri.render(); - Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri)); + Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", storeUri)); sshStore = bestMachine->openStore(); sshStore->connect(); - storeUri = bestMachine->storeUri; - } catch (std::exception & e) { auto msg = chomp(drainFD(5, false)); printError("cannot build on '%s': %s%s", - bestMachine->storeUri, e.what(), + storeUri, e.what(), msg.empty() ? "" : ": " + msg); bestMachine->enabled = false; continue; @@ -262,7 +263,20 @@ connected: auto inputs = readStrings(source); auto wantedOutputs = readStrings(source); - AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true); + AutoCloseFD uploadLock; + { + auto setUpdateLock = [&](auto && fileName){ + uploadLock = openLockFile(currentLoad + "/" + escapeUri(fileName) + ".upload-lock", true); + }; + try { + setUpdateLock(storeUri); + } catch (SysError & e) { + if (e.errNo != ENAMETOOLONG) throw; + // Try again hashing the store URL so we have a shorter path + auto h = hashString(HashAlgorithm::MD5, storeUri); + setUpdateLock(h.to_string(HashFormat::Base64, false)); + } + } { Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri)); diff --git a/doc/internal-api/.gitignore b/src/external-api-docs/.gitignore similarity index 100% rename from doc/internal-api/.gitignore rename to src/external-api-docs/.gitignore diff --git a/src/external-api-docs/.version b/src/external-api-docs/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/external-api-docs/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/external-api-docs/README.md b/src/external-api-docs/README.md new file mode 100644 index 000000000..8760ac88b --- /dev/null +++ b/src/external-api-docs/README.md @@ -0,0 +1,121 @@ +# Getting started + +> **Warning** These bindings are **experimental**, which means they can change +> at any time or be removed outright; nevertheless the plan is to provide a +> stable external C API to the Nix language and the Nix store. + +The language library allows evaluating Nix expressions and interacting with Nix +language values. The Nix store API is still rudimentary, and only allows +initialising and connecting to a store for the Nix language evaluator to +interact with. + +Currently there are two ways to interface with the Nix language evaluator +programmatically: + +1. Embedding the evaluator +2. Writing language plug-ins + +Embedding means you link the Nix C libraries in your program and use them from +there. Adding a plug-in means you make a library that gets loaded by the Nix +language evaluator, specified through a configuration option. + +Many of the components and mechanisms involved are not yet documented, therefore +please refer to the [Nix source code](https://github.com/NixOS/nix/) for +details. Additions to in-code documentation and the reference manual are highly +appreciated. + +The following examples, for simplicity, don't include error handling. See the +[Handling errors](@ref errors) section for more information. + +# Embedding the Nix Evaluator{#nix_evaluator_example} + +In this example we programmatically start the Nix language evaluator with a +dummy store (that has no store paths and cannot be written to), and evaluate the +Nix expression `builtins.nixVersion`. + +**main.c:** + +```C +#include +#include +#include +#include +#include +#include + +// NOTE: This example lacks all error handling. Production code must check for +// errors, as some return values will be undefined. + +void my_get_string_cb(const char * start, unsigned int n, void * user_data) +{ + *((char **) user_data) = strdup(start); +} + +int main() +{ + nix_libexpr_init(NULL); + + Store * store = nix_store_open(NULL, "dummy://", NULL); + EvalState * state = nix_state_create(NULL, NULL, store); // empty search path (NIX_PATH) + Value * value = nix_alloc_value(NULL, state); + + nix_expr_eval_from_string(NULL, state, "builtins.nixVersion", ".", value); + nix_value_force(NULL, state, value); + + char * version; + nix_get_string(NULL, value, my_get_string_cb, &version); + printf("Nix version: %s\n", version); + + free(version); + nix_gc_decref(NULL, value); + nix_state_free(state); + nix_store_free(store); + return 0; +} +``` + +**Usage:** + +```ShellSession +$ gcc main.c $(pkg-config nix-expr-c --libs --cflags) -o main +$ ./main +Nix version: 2.17 +``` + +# Writing a Nix language plug-in + +In this example we add a custom primitive operation (_primop_) to `builtins`. It +will increment the argument if it is an integer and throw an error otherwise. + +**plugin.c:** + +```C +#include +#include +#include + +void increment(void* user_data, nix_c_context* ctx, EvalState* state, Value** args, Value* v) { + nix_value_force(NULL, state, args[0]); + if (nix_get_type(NULL, args[0]) == NIX_TYPE_INT) { + nix_init_int(NULL, v, nix_get_int(NULL, args[0]) + 1); + } else { + nix_set_err_msg(ctx, NIX_ERR_UNKNOWN, "First argument should be an integer."); + } +} + +void nix_plugin_entry() { + const char* args[] = {"n", NULL}; + PrimOp *p = nix_alloc_primop(NULL, increment, 1, "increment", args, "Example custom built-in function: increments an integer", NULL); + nix_register_primop(NULL, p); + nix_gc_decref(NULL, p); +} +``` + +**Usage:** + +```ShellSession +$ gcc plugin.c $(pkg-config nix-expr-c --libs --cflags) -shared -o plugin.so +$ nix --plugin-files ./plugin.so repl +nix-repl> builtins.increment 1 +2 +``` diff --git a/src/external-api-docs/doxygen.cfg.in b/src/external-api-docs/doxygen.cfg.in new file mode 100644 index 000000000..1be71d895 --- /dev/null +++ b/src/external-api-docs/doxygen.cfg.in @@ -0,0 +1,58 @@ +# Doxyfile 1.9.5 + +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by +# double-quotes, unless you are using Doxywizard) that should identify the +# project for which the documentation is generated. This name is used in the +# title of most generated pages and in a few other places. +# The default value is: My Project. + +PROJECT_NAME = "Nix" + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. This +# could be handy for archiving the generated documentation or if some version +# control system is used. + +PROJECT_NUMBER = @PROJECT_NUMBER@ + +OUTPUT_DIRECTORY = @OUTPUT_DIRECTORY@ + +# Using the PROJECT_BRIEF tag one can provide an optional one line description +# for a project that appears at the top of each page and should give viewer a +# quick idea about the purpose of the project. Keep the description short. + +PROJECT_BRIEF = "Nix, the purely functional package manager: C API (experimental)" + +# If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output. +# The default value is: YES. + +GENERATE_LATEX = NO + +# The INPUT tag is used to specify the files and/or directories that contain +# documented source files. You may enter file names like myfile.cpp or +# directories like /usr/src/myproject. Separate the files or directories with +# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING +# Note: If this tag is empty the current directory is searched. + +# FIXME Make this list more maintainable somehow. We could maybe generate this +# in the Makefile, but we would need to change how `.in` files are preprocessed +# so they can expand variables despite configure variables. + +INPUT = \ + @src@/src/libutil-c \ + @src@/src/libexpr-c \ + @src@/src/libstore-c \ + @src@/doc/external-api/README.md + +FILE_PATTERNS = nix_api_*.h *.md + +# The INCLUDE_PATH tag can be used to specify one or more directories that +# contain include files that are not input files but should be processed by the +# preprocessor. Note that the INCLUDE_PATH is not recursive, so the setting of +# RECURSIVE has no effect here. +# This tag requires that the tag SEARCH_INCLUDES is set to YES. + +EXCLUDE_PATTERNS = *_internal.h +GENERATE_TREEVIEW = YES +OPTIMIZE_OUTPUT_FOR_C = YES + +USE_MDFILE_AS_MAINPAGE = doc/external-api/README.md diff --git a/src/external-api-docs/meson.build b/src/external-api-docs/meson.build new file mode 100644 index 000000000..62474ffe4 --- /dev/null +++ b/src/external-api-docs/meson.build @@ -0,0 +1,31 @@ +project('nix-external-api-docs', + version : files('.version'), + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +fs = import('fs') + +doxygen_cfg = configure_file( + input : 'doxygen.cfg.in', + output : 'doxygen.cfg', + configuration : { + 'PROJECT_NUMBER': meson.project_version(), + 'OUTPUT_DIRECTORY' : meson.current_build_dir(), + 'src' : fs.parent(fs.parent(meson.project_source_root())), + }, +) + +doxygen = find_program('doxygen', native : true, required : true) + +custom_target( + 'external-api-docs', + command : [ doxygen , doxygen_cfg ], + input : [ + doxygen_cfg, + ], + output : 'html', + install : true, + install_dir : get_option('datadir') / 'doc/nix/external-api', + build_always_stale : true, +) diff --git a/src/external-api-docs/package.nix b/src/external-api-docs/package.nix new file mode 100644 index 000000000..743b3e9b7 --- /dev/null +++ b/src/external-api-docs/package.nix @@ -0,0 +1,59 @@ +{ lib +, mkMesonDerivation + +, meson +, ninja +, doxygen + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-external-api-docs"; + inherit version; + + workDir = ./.; + fileset = + let + cpp = fileset.fileFilter (file: file.hasExt "cc" || file.hasExt "h"); + in + fileset.unions [ + ./.version + ../../.version + ./meson.build + ./doxygen.cfg.in + ./README.md + # Source is not compiled, but still must be available for Doxygen + # to gather comments. + (cpp ../libexpr-c) + (cpp ../libstore-c) + (cpp ../libutil-c) + ]; + + nativeBuildInputs = [ + meson + ninja + doxygen + ]; + + preConfigure = + '' + chmod u+w ./.version + echo ${finalAttrs.version} > ./.version + ''; + + postInstall = '' + mkdir -p ''${!outputDoc}/nix-support + echo "doc external-api-docs $out/share/doc/nix/external-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products + ''; + + meta = { + platforms = lib.platforms.all; + }; +}) diff --git a/src/internal-api-docs/.gitignore b/src/internal-api-docs/.gitignore new file mode 100644 index 000000000..dab28b6b0 --- /dev/null +++ b/src/internal-api-docs/.gitignore @@ -0,0 +1,3 @@ +/doxygen.cfg +/html +/latex diff --git a/src/internal-api-docs/.version b/src/internal-api-docs/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/internal-api-docs/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/doc/internal-api/doxygen.cfg.in b/src/internal-api-docs/doxygen.cfg.in similarity index 84% rename from doc/internal-api/doxygen.cfg.in rename to src/internal-api-docs/doxygen.cfg.in index 6c6c325bd..f1ef75b38 100644 --- a/doc/internal-api/doxygen.cfg.in +++ b/src/internal-api-docs/doxygen.cfg.in @@ -12,7 +12,9 @@ PROJECT_NAME = "Nix" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = @PACKAGE_VERSION@ +PROJECT_NUMBER = @PROJECT_NUMBER@ + +OUTPUT_DIRECTORY = @OUTPUT_DIRECTORY@ # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a @@ -36,27 +38,27 @@ GENERATE_LATEX = NO # so they can expand variables despite configure variables. INPUT = \ - src/libcmd \ - src/libexpr \ - src/libexpr/flake \ - tests/unit/libexpr \ - tests/unit/libexpr/value \ - tests/unit/libexpr/test \ - tests/unit/libexpr/test/value \ - src/libexpr/value \ - src/libfetchers \ - src/libmain \ - src/libstore \ - src/libstore/build \ - src/libstore/builtins \ - tests/unit/libstore \ - tests/unit/libstore/test \ - src/libutil \ - tests/unit/libutil \ - tests/unit/libutil/test \ - src/nix \ - src/nix-env \ - src/nix-store + @src@/libcmd \ + @src@/libexpr \ + @src@/libexpr/flake \ + @src@/nix-expr-tests \ + @src@/nix-expr-tests/value \ + @src@/nix-expr-test-support/test \ + @src@/nix-expr-test-support/test/value \ + @src@/libexpr/value \ + @src@/libfetchers \ + @src@/libmain \ + @src@/libstore \ + @src@/libstore/build \ + @src@/libstore/builtins \ + @src@/nix-store-tests \ + @src@/nix-store-test-support/test \ + @src@/libutil \ + @src@/nix-util-tests \ + @src@/nix-util-test-support/test \ + @src@/nix \ + @src@/nix-env \ + @src@/nix-store # If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names # in the source code. If set to NO, only conditional compilation will be diff --git a/src/internal-api-docs/meson.build b/src/internal-api-docs/meson.build new file mode 100644 index 000000000..54eb7e5dd --- /dev/null +++ b/src/internal-api-docs/meson.build @@ -0,0 +1,31 @@ +project('nix-internal-api-docs', + version : files('.version'), + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +fs = import('fs') + +doxygen_cfg = configure_file( + input : 'doxygen.cfg.in', + output : 'doxygen.cfg', + configuration : { + 'PROJECT_NUMBER': meson.project_version(), + 'OUTPUT_DIRECTORY' : meson.current_build_dir(), + 'src' : fs.parent(fs.parent(meson.project_source_root())) / 'src', + }, +) + +doxygen = find_program('doxygen', native : true, required : true) + +custom_target( + 'internal-api-docs', + command : [ doxygen , doxygen_cfg ], + input : [ + doxygen_cfg, + ], + output : 'html', + install : true, + install_dir : get_option('datadir') / 'doc/nix/internal-api', + build_always_stale : true, +) diff --git a/src/internal-api-docs/package.nix b/src/internal-api-docs/package.nix new file mode 100644 index 000000000..07ca6d4d9 --- /dev/null +++ b/src/internal-api-docs/package.nix @@ -0,0 +1,54 @@ +{ lib +, mkMesonDerivation + +, meson +, ninja +, doxygen + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-internal-api-docs"; + inherit version; + + workDir = ./.; + fileset = let + cpp = fileset.fileFilter (file: file.hasExt "cc" || file.hasExt "hh"); + in fileset.unions [ + ./.version + ../../.version + ./meson.build + ./doxygen.cfg.in + # Source is not compiled, but still must be available for Doxygen + # to gather comments. + (cpp ../.) + ]; + + nativeBuildInputs = [ + meson + ninja + doxygen + ]; + + preConfigure = + '' + chmod u+w ./.version + echo ${finalAttrs.version} > ./.version + ''; + + postInstall = '' + mkdir -p ''${!outputDoc}/nix-support + echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products + ''; + + meta = { + platforms = lib.platforms.all; + }; +}) diff --git a/src/libcmd/.version b/src/libcmd/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libcmd/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libcmd/build-utils-meson b/src/libcmd/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libcmd/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc index c5eb93c5d..905e70f32 100644 --- a/src/libcmd/built-path.cc +++ b/src/libcmd/built-path.cc @@ -1,6 +1,7 @@ #include "built-path.hh" #include "derivations.hh" #include "store-api.hh" +#include "comparator.hh" #include @@ -8,30 +9,24 @@ namespace nix { -#define CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, COMPARATOR) \ - bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \ - { \ - const MY_TYPE* me = this; \ - auto fields1 = std::tie(*me->drvPath, me->FIELD); \ - me = &other; \ - auto fields2 = std::tie(*me->drvPath, me->FIELD); \ - return fields1 COMPARATOR fields2; \ - } -#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, ==) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <) +// Custom implementation to avoid `ref` ptr equality +GENERATE_CMP_EXT( + , + std::strong_ordering, + SingleBuiltPathBuilt, + *me->drvPath, + me->output); -#define FIELD_TYPE std::pair -CMP(SingleBuiltPath, SingleBuiltPathBuilt, output) -#undef FIELD_TYPE +// Custom implementation to avoid `ref` ptr equality -#define FIELD_TYPE std::map -CMP(SingleBuiltPath, BuiltPathBuilt, outputs) -#undef FIELD_TYPE - -#undef CMP -#undef CMP_ONE +// TODO no `GENERATE_CMP_EXT` because no `std::set::operator<=>` on +// Darwin, per header. +GENERATE_EQUAL( + , + BuiltPathBuilt ::, + BuiltPathBuilt, + *me->drvPath, + me->outputs); StorePath SingleBuiltPath::outPath() const { diff --git a/src/libcmd/built-path.hh b/src/libcmd/built-path.hh index 99917e0ee..dc78d3e59 100644 --- a/src/libcmd/built-path.hh +++ b/src/libcmd/built-path.hh @@ -18,7 +18,8 @@ struct SingleBuiltPathBuilt { static SingleBuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view); nlohmann::json toJSON(const StoreDirConfig & store) const; - DECLARE_CMP(SingleBuiltPathBuilt); + bool operator ==(const SingleBuiltPathBuilt &) const noexcept; + std::strong_ordering operator <=>(const SingleBuiltPathBuilt &) const noexcept; }; using _SingleBuiltPathRaw = std::variant< @@ -33,6 +34,9 @@ struct SingleBuiltPath : _SingleBuiltPathRaw { using Opaque = DerivedPathOpaque; using Built = SingleBuiltPathBuilt; + bool operator == (const SingleBuiltPath &) const = default; + auto operator <=> (const SingleBuiltPath &) const = default; + inline const Raw & raw() const { return static_cast(*this); } @@ -59,11 +63,13 @@ struct BuiltPathBuilt { ref drvPath; std::map outputs; + bool operator == (const BuiltPathBuilt &) const noexcept; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //std::strong_ordering operator <=> (const BuiltPathBuilt &) const noexcept; + std::string to_string(const StoreDirConfig & store) const; static BuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view); nlohmann::json toJSON(const StoreDirConfig & store) const; - - DECLARE_CMP(BuiltPathBuilt); }; using _BuiltPathRaw = std::variant< @@ -82,6 +88,10 @@ struct BuiltPath : _BuiltPathRaw { using Opaque = DerivedPathOpaque; using Built = BuiltPathBuilt; + bool operator == (const BuiltPath &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=> (const BuiltPath &) const = default; + inline const Raw & raw() const { return static_cast(*this); } diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 369fa6004..67fef1909 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -1,3 +1,5 @@ +#include + #include "command.hh" #include "markdown.hh" #include "store-api.hh" @@ -6,8 +8,7 @@ #include "nixexpr.hh" #include "profiles.hh" #include "repl.hh" - -#include +#include "strings.hh" extern char * * environ __attribute__((weak)); @@ -127,12 +128,12 @@ ref EvalCommand::getEvalState() if (!evalState) { evalState = #if HAVE_BOEHMGC - std::allocate_shared(traceable_allocator(), - searchPath, getEvalStore(), getStore()) + std::allocate_shared( + traceable_allocator(), #else std::make_shared( - searchPath, getEvalStore(), getStore()) #endif + lookupPath, getEvalStore(), fetchSettings, evalSettings, getStore()) ; evalState->repair = repair; @@ -148,7 +149,7 @@ MixOperateOnOptions::MixOperateOnOptions() { addFlag({ .longName = "derivation", - .description = "Operate on the [store derivation](../../glossary.md#gloss-store-derivation) rather than its outputs.", + .description = "Operate on the [store derivation](@docroot@/glossary.md#gloss-store-derivation) rather than its outputs.", .category = installablesCategory, .handler = {&operateOn, OperateOn::Derivation}, }); diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 444ff81c9..fcef92487 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -1,18 +1,57 @@ +#include "fetch-settings.hh" #include "eval-settings.hh" #include "common-eval-args.hh" #include "shared.hh" +#include "config-global.hh" #include "filetransfer.hh" #include "eval.hh" #include "fetchers.hh" #include "registry.hh" #include "flake/flakeref.hh" +#include "flake/settings.hh" #include "store-api.hh" #include "command.hh" #include "tarball.hh" #include "fetch-to-store.hh" +#include "compatibility-settings.hh" +#include "eval-settings.hh" namespace nix { +fetchers::Settings fetchSettings; + +static GlobalConfig::Register rFetchSettings(&fetchSettings); + +EvalSettings evalSettings { + settings.readOnlyMode, + { + { + "flake", + [](ref store, std::string_view rest) { + experimentalFeatureSettings.require(Xp::Flakes); + // FIXME `parseFlakeRef` should take a `std::string_view`. + auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false); + debug("fetching flake search path element '%s''", rest); + auto storePath = flakeRef.resolve(store).fetchTree(store).first; + return store->toRealPath(storePath); + }, + }, + }, +}; + +static GlobalConfig::Register rEvalSettings(&evalSettings); + + +flake::Settings flakeSettings; + +static GlobalConfig::Register rFlakeSettings(&flakeSettings); + + +CompatibilitySettings compatibilitySettings {}; + +static GlobalConfig::Register rCompatibilitySettings(&compatibilitySettings); + + MixEvalArgs::MixEvalArgs() { addFlag({ @@ -20,7 +59,7 @@ MixEvalArgs::MixEvalArgs() .description = "Pass the value *expr* as the argument *name* to Nix functions.", .category = category, .labels = {"name", "expr"}, - .handler = {[&](std::string name, std::string expr) { autoArgs[name] = 'E' + expr; }} + .handler = {[&](std::string name, std::string expr) { autoArgs.insert_or_assign(name, AutoArg{AutoArgExpr{expr}}); }} }); addFlag({ @@ -28,87 +67,40 @@ MixEvalArgs::MixEvalArgs() .description = "Pass the string *string* as the argument *name* to Nix functions.", .category = category, .labels = {"name", "string"}, - .handler = {[&](std::string name, std::string s) { autoArgs[name] = 'S' + s; }}, + .handler = {[&](std::string name, std::string s) { autoArgs.insert_or_assign(name, AutoArg{AutoArgString{s}}); }}, + }); + + addFlag({ + .longName = "arg-from-file", + .description = "Pass the contents of file *path* as the argument *name* to Nix functions.", + .category = category, + .labels = {"name", "path"}, + .handler = {[&](std::string name, std::string path) { autoArgs.insert_or_assign(name, AutoArg{AutoArgFile{path}}); }}, + .completer = completePath + }); + + addFlag({ + .longName = "arg-from-stdin", + .description = "Pass the contents of stdin as the argument *name* to Nix functions.", + .category = category, + .labels = {"name"}, + .handler = {[&](std::string name) { autoArgs.insert_or_assign(name, AutoArg{AutoArgStdin{}}); }}, }); addFlag({ .longName = "include", .shortName = 'I', .description = R"( - Add *path* to the Nix search path. The Nix search path is - initialized from the colon-separated [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) environment - variable, and is used to look up the location of Nix expressions using [paths](@docroot@/language/values.md#type-path) enclosed in angle - brackets (i.e., ``). + Add *path* to search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md) - For instance, passing + This option may be given multiple times. - ``` - -I /home/eelco/Dev - -I /etc/nixos - ``` - - will cause Nix to look for paths relative to `/home/eelco/Dev` and - `/etc/nixos`, in that order. This is equivalent to setting the - `NIX_PATH` environment variable to - - ``` - /home/eelco/Dev:/etc/nixos - ``` - - It is also possible to match paths against a prefix. For example, - passing - - ``` - -I nixpkgs=/home/eelco/Dev/nixpkgs-branch - -I /etc/nixos - ``` - - will cause Nix to search for `` in - `/home/eelco/Dev/nixpkgs-branch/path` and `/etc/nixos/nixpkgs/path`. - - If a path in the Nix search path starts with `http://` or `https://`, - it is interpreted as the URL of a tarball that will be downloaded and - unpacked to a temporary location. The tarball must consist of a single - top-level directory. For example, passing - - ``` - -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/master.tar.gz - ``` - - tells Nix to download and use the current contents of the `master` - branch in the `nixpkgs` repository. - - The URLs of the tarballs from the official `nixos.org` channels - (see [the manual page for `nix-channel`](../nix-channel.md)) can be - abbreviated as `channel:`. For instance, the - following two flags are equivalent: - - ``` - -I nixpkgs=channel:nixos-21.05 - -I nixpkgs=https://nixos.org/channels/nixos-21.05/nixexprs.tar.xz - ``` - - You can also fetch source trees using [flake URLs](./nix3-flake.md#url-like-syntax) and add them to the - search path. For instance, - - ``` - -I nixpkgs=flake:nixpkgs - ``` - - specifies that the prefix `nixpkgs` shall refer to the source tree - downloaded from the `nixpkgs` entry in the flake registry. Similarly, - - ``` - -I nixpkgs=flake:github:NixOS/nixpkgs/nixos-22.05 - ``` - - makes `` refer to a particular branch of the - `NixOS/nixpkgs` repository on GitHub. + Paths added through `-I` take precedence over the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path) and the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH). )", .category = category, .labels = {"path"}, .handler = {[&](std::string s) { - searchPath.elements.emplace_back(SearchPath::Elem::parse(s)); + lookupPath.elements.emplace_back(LookupPath::Elem::parse(s)); }} }); @@ -127,8 +119,8 @@ MixEvalArgs::MixEvalArgs() .category = category, .labels = {"original-ref", "resolved-ref"}, .handler = {[&](std::string _from, std::string _to) { - auto from = parseFlakeRef(_from, absPath(".")); - auto to = parseFlakeRef(_to, absPath(".")); + auto from = parseFlakeRef(fetchSettings, _from, absPath(".")); + auto to = parseFlakeRef(fetchSettings, _to, absPath(".")); fetchers::Attrs extraAttrs; if (to.subdir != "") extraAttrs["dir"] = to.subdir; fetchers::overrideRegistry(from.input, to.input, extraAttrs); @@ -154,13 +146,23 @@ MixEvalArgs::MixEvalArgs() Bindings * MixEvalArgs::getAutoArgs(EvalState & state) { auto res = state.buildBindings(autoArgs.size()); - for (auto & i : autoArgs) { + for (auto & [name, arg] : autoArgs) { auto v = state.allocValue(); - if (i.second[0] == 'E') - state.mkThunk_(*v, state.parseExprFromString(i.second.substr(1), state.rootPath("."))); - else - v->mkString(((std::string_view) i.second).substr(1)); - res.insert(state.symbols.create(i.first), v); + std::visit(overloaded { + [&](const AutoArgExpr & arg) { + state.mkThunk_(*v, state.parseExprFromString(arg.expr, compatibilitySettings.nixShellShebangArgumentsRelativeToScript ? state.rootPath(absPath(getCommandBaseDir())) : state.rootPath("."))); + }, + [&](const AutoArgString & arg) { + v->mkString(arg.s); + }, + [&](const AutoArgFile & arg) { + v->mkString(readFile(arg.path.string())); + }, + [&](const AutoArgStdin & arg) { + v->mkString(readFile(STDIN_FILENO)); + } + }, arg); + res.insert(state.symbols.create(name), v); } return res.finish(); } @@ -176,7 +178,7 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas else if (hasPrefix(s, "flake:")) { experimentalFeatureSettings.require(Xp::Flakes); - auto flakeRef = parseFlakeRef(std::string(s.substr(6)), {}, true, false); + auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false); auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first; return state.rootPath(CanonPath(state.store->toRealPath(storePath))); } diff --git a/src/libcmd/common-eval-args.hh b/src/libcmd/common-eval-args.hh index 2eb63e15d..c62365b32 100644 --- a/src/libcmd/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh @@ -6,13 +6,42 @@ #include "common-args.hh" #include "search-path.hh" +#include + namespace nix { class Store; + +namespace fetchers { struct Settings; } + class EvalState; +struct EvalSettings; +struct CompatibilitySettings; class Bindings; struct SourcePath; +namespace flake { struct Settings; } + +/** + * @todo Get rid of global setttings variables + */ +extern fetchers::Settings fetchSettings; + +/** + * @todo Get rid of global setttings variables + */ +extern EvalSettings evalSettings; + +/** + * @todo Get rid of global setttings variables + */ +extern flake::Settings flakeSettings; + +/** + * Settings that control behaviors that have changed since Nix 2.3. + */ +extern CompatibilitySettings compatibilitySettings; + struct MixEvalArgs : virtual Args, virtual MixRepair { static constexpr auto category = "Common evaluation options"; @@ -21,14 +50,24 @@ struct MixEvalArgs : virtual Args, virtual MixRepair Bindings * getAutoArgs(EvalState & state); - SearchPath searchPath; + LookupPath lookupPath; std::optional evalStoreUrl; private: - std::map autoArgs; + struct AutoArgExpr { std::string expr; }; + struct AutoArgString { std::string s; }; + struct AutoArgFile { std::filesystem::path path; }; + struct AutoArgStdin { }; + + using AutoArg = std::variant; + + std::map autoArgs; }; +/** + * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) + */ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * baseDir = nullptr); } diff --git a/src/libcmd/compatibility-settings.hh b/src/libcmd/compatibility-settings.hh new file mode 100644 index 000000000..a129a957a --- /dev/null +++ b/src/libcmd/compatibility-settings.hh @@ -0,0 +1,36 @@ +#pragma once +#include "config.hh" + +namespace nix { +struct CompatibilitySettings : public Config +{ + + CompatibilitySettings() = default; + + // Added in Nix 2.24, July 2024. + Setting nixShellAlwaysLooksForShellNix{this, true, "nix-shell-always-looks-for-shell-nix", R"( + Before Nix 2.24, [`nix-shell`](@docroot@/command-ref/nix-shell.md) would only look at `shell.nix` if it was in the working directory - when no file was specified. + + Since Nix 2.24, `nix-shell` always looks for a `shell.nix`, whether that's in the working directory, or in a directory that was passed as an argument. + + You may set this to `false` to temporarily revert to the behavior of Nix 2.23 and older. + + Using this setting is not recommended. + It will be deprecated and removed. + )"}; + + // Added in Nix 2.24, July 2024. + Setting nixShellShebangArgumentsRelativeToScript{ + this, true, "nix-shell-shebang-arguments-relative-to-script", R"( + Before Nix 2.24, relative file path expressions in arguments in a `nix-shell` shebang were resolved relative to the working directory. + + Since Nix 2.24, `nix-shell` resolves these paths in a manner that is relative to the [base directory](@docroot@/glossary.md#gloss-base-directory), defined as the script's directory. + + You may set this to `false` to temporarily revert to the behavior of Nix 2.23 and older. + + Using this setting is not recommended. + It will be deprecated and removed. + )"}; +}; + +}; diff --git a/src/libcmd/installable-attr-path.cc b/src/libcmd/installable-attr-path.cc index 3ec1c1614..8917e7a01 100644 --- a/src/libcmd/installable-attr-path.cc +++ b/src/libcmd/installable-attr-path.cc @@ -75,6 +75,8 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths() std::set outputsToInstall; for (auto & output : packageInfo.queryOutputs(false, true)) outputsToInstall.insert(output.first); + if (outputsToInstall.empty()) + outputsToInstall.insert("out"); return OutputsSpec::Names { std::move(outputsToInstall) }; }, [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec { diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index ddec7537b..8796ad5ba 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -43,20 +43,6 @@ std::vector InstallableFlake::getActualAttrPaths() return res; } -Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake) -{ - auto vFlake = state.allocValue(); - - callFlake(state, lockedFlake, *vFlake); - - auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); - assert(aOutputs); - - state.forceValue(*aOutputs->value, aOutputs->value->determinePos(noPos)); - - return aOutputs->value; -} - static std::string showAttrPaths(const std::vector & paths) { std::string s; @@ -106,9 +92,14 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() fmt("while evaluating the flake output attribute '%s'", attrPath))) { return { *derivedPathWithInfo }; + } else { + throw Error( + "expected flake output attribute '%s' to be a derivation or path but found %s: %s", + attrPath, + showType(v), + ValuePrinter(*this->state, v, errorPrintOptions) + ); } - else - throw Error("flake output attribute '%s' is not a derivation or path", attrPath); } auto drvPath = attr->forceDerivation(); @@ -205,7 +196,8 @@ std::shared_ptr InstallableFlake::getLockedFlake() const flake::LockFlags lockFlagsApplyConfig = lockFlags; // FIXME why this side effect? lockFlagsApplyConfig.applyNixConfig = true; - _lockedFlake = std::make_shared(lockFlake(*state, flakeRef, lockFlagsApplyConfig)); + _lockedFlake = std::make_shared(lockFlake( + flakeSettings, *state, flakeRef, lockFlagsApplyConfig)); } return _lockedFlake; } diff --git a/src/libcmd/installable-flake.hh b/src/libcmd/installable-flake.hh index 314918c14..8e0a232ef 100644 --- a/src/libcmd/installable-flake.hh +++ b/src/libcmd/installable-flake.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "common-eval-args.hh" #include "installable-value.hh" namespace nix { @@ -52,8 +53,6 @@ struct InstallableFlake : InstallableValue std::vector getActualAttrPaths(); - Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake); - DerivedPathsWithInfo toDerivedPaths() override; std::pair toValue(EvalState & state) override; @@ -80,7 +79,7 @@ struct InstallableFlake : InstallableValue */ static inline FlakeRef defaultNixpkgsFlakeRef() { - return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}}); + return FlakeRef::fromAttrs(fetchSettings, {{"type","indirect"}, {"id", "nixpkgs"}}); } ref openEvalCache( diff --git a/src/libcmd/installable-value.hh b/src/libcmd/installable-value.hh index f300d392b..798cb5e1a 100644 --- a/src/libcmd/installable-value.hh +++ b/src/libcmd/installable-value.hh @@ -66,7 +66,7 @@ struct ExtraPathInfoValue : ExtraPathInfo }; /** - * An Installable which corresponds a Nix langauge value, in addition to + * An Installable which corresponds a Nix language value, in addition to * a collection of \ref DerivedPath "derived paths". */ struct InstallableValue : Installable diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 16d25d3cf..0fe956ec0 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -27,6 +27,8 @@ #include +#include "strings-inline.hh" + namespace nix { void completeFlakeInputPath( @@ -129,7 +131,7 @@ MixFlakeOptions::MixFlakeOptions() lockFlags.writeLockFile = false; lockFlags.inputOverrides.insert_or_assign( flake::parseInputPath(inputPath), - parseFlakeRef(flakeRef, absPath(getCommandBaseDir()), true)); + parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir()), true)); }}, .completer = {[&](AddCompletions & completions, size_t n, std::string_view prefix) { if (n == 0) { @@ -146,7 +148,7 @@ MixFlakeOptions::MixFlakeOptions() .category = category, .labels = {"flake-lock-path"}, .handler = {[&](std::string lockFilePath) { - lockFlags.referenceLockFilePath = lockFilePath; + lockFlags.referenceLockFilePath = {getFSSourceAccessor(), CanonPath(absPath(lockFilePath))}; }}, .completer = completePath }); @@ -170,14 +172,15 @@ MixFlakeOptions::MixFlakeOptions() .handler = {[&](std::string flakeRef) { auto evalState = getEvalState(); auto flake = flake::lockFlake( + flakeSettings, *evalState, - parseFlakeRef(flakeRef, absPath(getCommandBaseDir())), + parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir())), { .writeLockFile = false }); for (auto & [inputName, input] : flake.lockFile.root->inputs) { auto input2 = flake.lockFile.findInput({inputName}); // resolve 'follows' nodes if (auto input3 = std::dynamic_pointer_cast(input2)) { overrideRegistry( - fetchers::Input::fromAttrs({{"type","indirect"}, {"id", inputName}}), + fetchers::Input::fromAttrs(fetchSettings, {{"type","indirect"}, {"id", inputName}}), input3->lockedRef.input, {}); } @@ -288,11 +291,11 @@ void SourceExprCommand::completeInstallable(AddCompletions & completions, std::s state->autoCallFunction(*autoArgs, v1, v2); if (v2.type() == nAttrs) { - for (auto & i : *v2.attrs) { - std::string name = state->symbols[i.name]; + for (auto & i : *v2.attrs()) { + std::string_view name = state->symbols[i.name]; if (name.find(searchWord) == 0) { if (prefix_ == "") - completions.add(name); + completions.add(std::string(name)); else completions.add(prefix_ + "." + name); } @@ -338,10 +341,11 @@ void completeFlakeRefWithFragment( auto flakeRefS = std::string(prefix.substr(0, hash)); // TODO: ideally this would use the command base directory instead of assuming ".". - auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath(".")); + auto flakeRef = parseFlakeRef(fetchSettings, expandTilde(flakeRefS), absPath(".")); auto evalCache = openEvalCache(*evalState, - std::make_shared(lockFlake(*evalState, flakeRef, lockFlags))); + std::make_shared(lockFlake( + flakeSettings, *evalState, flakeRef, lockFlags))); auto root = evalCache->getRoot(); @@ -372,6 +376,7 @@ void completeFlakeRefWithFragment( auto attrPath2 = (*attr)->getAttrPath(attr2); /* Strip the attrpath prefix. */ attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size()); + // FIXME: handle names with dots completions.add(flakeRefS + "#" + prefixRoot + concatStringsSep(".", evalState->symbols.resolve(attrPath2))); } } @@ -403,7 +408,7 @@ void completeFlakeRef(AddCompletions & completions, ref store, std::strin Args::completeDir(completions, 0, prefix); /* Look for registry entries that match the prefix. */ - for (auto & registry : fetchers::getRegistries(store)) { + for (auto & registry : fetchers::getRegistries(fetchSettings, store)) { for (auto & entry : registry->entries) { auto from = entry.from.to_string(); if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) { @@ -442,13 +447,10 @@ ref openEvalCache( EvalState & state, std::shared_ptr lockedFlake) { - auto fingerprint = lockedFlake->getFingerprint(); - return make_ref( - evalSettings.useEvalCache && evalSettings.pureEval - ? std::optional { std::cref(fingerprint) } - : std::nullopt, - state, - [&state, lockedFlake]() + auto fingerprint = evalSettings.useEvalCache && evalSettings.pureEval + ? lockedFlake->getFingerprint(state.store) + : std::nullopt; + auto rootLoader = [&state, lockedFlake]() { /* For testing whether the evaluation cache is complete. */ @@ -460,11 +462,21 @@ ref openEvalCache( state.forceAttrs(*vFlake, noPos, "while parsing cached flake data"); - auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); + auto aOutputs = vFlake->attrs()->get(state.symbols.create("outputs")); assert(aOutputs); return aOutputs->value; - }); + }; + + if (fingerprint) { + auto search = state.evalCaches.find(fingerprint.value()); + if (search == state.evalCaches.end()) { + search = state.evalCaches.emplace(fingerprint.value(), make_ref(fingerprint, state, rootLoader)).first; + } + return search->second; + } else { + return make_ref(std::nullopt, state, rootLoader); + } } Installables SourceExprCommand::parseInstallables( @@ -527,7 +539,8 @@ Installables SourceExprCommand::parseInstallables( } try { - auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, absPath(getCommandBaseDir())); + auto [flakeRef, fragment] = parseFlakeRefWithFragment( + fetchSettings, std::string { prefix }, absPath(getCommandBaseDir())); result.push_back(make_ref( this, getEvalState(), @@ -594,6 +607,37 @@ std::vector Installable::build( return res; } +static void throwBuildErrors( + std::vector & buildResults, + const Store & store) +{ + std::vector failed; + for (auto & buildResult : buildResults) { + if (!buildResult.success()) { + failed.push_back(buildResult); + } + } + + auto failedResult = failed.begin(); + if (failedResult != failed.end()) { + if (failed.size() == 1) { + failedResult->rethrow(); + } else { + StringSet failedPaths; + for (; failedResult != failed.end(); failedResult++) { + if (!failedResult->errorMsg.empty()) { + logError(ErrorInfo{ + .level = lvlError, + .msg = failedResult->errorMsg, + }); + } + failedPaths.insert(failedResult->path.to_string(store)); + } + throw Error("build of %s failed", concatStringsSep(", ", quoteStrings(failedPaths))); + } + } +} + std::vector, BuiltPathWithResult>> Installable::build2( ref evalStore, ref store, @@ -655,10 +699,9 @@ std::vector, BuiltPathWithResult>> Installable::build if (settings.printMissing) printMissing(store, pathsToBuild, lvlInfo); - for (auto & buildResult : store->buildPathsWithResults(pathsToBuild, bMode, evalStore)) { - if (!buildResult.success()) - buildResult.rethrow(); - + auto buildResults = store->buildPathsWithResults(pathsToBuild, bMode, evalStore); + throwBuildErrors(buildResults, *store); + for (auto & buildResult : buildResults) { for (auto & aux : backmap[buildResult.path]) { std::visit(overloaded { [&](const DerivedPath::Built & bfd) { @@ -814,6 +857,7 @@ std::vector RawInstallablesCommand::getFlakeRefsForCompletion() std::vector res; for (auto i : rawInstallables) res.push_back(parseFlakeRefWithFragment( + fetchSettings, expandTilde(i), absPath(getCommandBaseDir())).first); return res; @@ -836,6 +880,7 @@ std::vector InstallableCommand::getFlakeRefsForCompletion() { return { parseFlakeRefWithFragment( + fetchSettings, expandTilde(_installable), absPath(getCommandBaseDir())).first }; diff --git a/src/libcmd/local.mk b/src/libcmd/local.mk index abb7459a7..a270333f4 100644 --- a/src/libcmd/local.mk +++ b/src/libcmd/local.mk @@ -6,10 +6,10 @@ libcmd_DIR := $(d) libcmd_SOURCES := $(wildcard $(d)/*.cc) -libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers +libcmd_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) $(INCLUDE_libflake) $(INCLUDE_libmain) libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) $(THREAD_LDFLAGS) -libcmd_LIBS = libstore libutil libexpr libmain libfetchers +libcmd_LIBS = libutil libstore libfetchers libflake libexpr libmain $(eval $(call install-file-in, $(buildprefix)$(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644)) diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc index a4e3c5a77..6a0d05d9f 100644 --- a/src/libcmd/markdown.cc +++ b/src/libcmd/markdown.cc @@ -1,21 +1,23 @@ #include "markdown.hh" -#include "util.hh" +#include "environment-variables.hh" +#include "error.hh" #include "finally.hh" #include "terminal.hh" -#include #if HAVE_LOWDOWN -#include +# include +# include #endif namespace nix { -std::string renderMarkdownToTerminal(std::string_view markdown) -{ #if HAVE_LOWDOWN +static std::string doRenderMarkdownToTerminal(std::string_view markdown) +{ int windowWidth = getWindowSize().second; - struct lowdown_opts opts { + struct lowdown_opts opts + { .type = LOWDOWN_TERM, .maxdepth = 20, .cols = (size_t) std::max(windowWidth - 5, 60), @@ -50,10 +52,22 @@ std::string renderMarkdownToTerminal(std::string_view markdown) if (!rndr_res) throw Error("allocation error while rendering Markdown"); - return filterANSIEscapes(std::string(buf->data, buf->size), !shouldANSI()); -#else - return std::string(markdown); -#endif + return filterANSIEscapes(std::string(buf->data, buf->size), !isTTY()); } +std::string renderMarkdownToTerminal(std::string_view markdown) +{ + if (auto e = getEnv("_NIX_TEST_RAW_MARKDOWN"); e && *e == "1") + return std::string(markdown); + else + return doRenderMarkdownToTerminal(markdown); } + +#else +std::string renderMarkdownToTerminal(std::string_view markdown) +{ + return std::string(markdown); +} +#endif + +} // namespace nix diff --git a/src/libcmd/markdown.hh b/src/libcmd/markdown.hh index a04d32a4f..66db1736c 100644 --- a/src/libcmd/markdown.hh +++ b/src/libcmd/markdown.hh @@ -1,10 +1,17 @@ #pragma once ///@file -#include "types.hh" +#include namespace nix { +/** + * Render the given Markdown text to the terminal. + * + * If Nix is compiled without Markdown support, this function will return the input text as-is. + * + * The renderer takes into account the terminal width, and wraps text accordingly. + */ std::string renderMarkdownToTerminal(std::string_view markdown); } diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build new file mode 100644 index 000000000..c484cf998 --- /dev/null +++ b/src/libcmd/meson.build @@ -0,0 +1,130 @@ +project('nix-cmd', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ +] +deps_public_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), + dependency('nix-fetchers'), + dependency('nix-expr'), + dependency('nix-flake'), + dependency('nix-main'), +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') +deps_public += nlohmann_json + +lowdown = dependency('lowdown', version : '>= 0.9.0', required : get_option('markdown')) +deps_private += lowdown +configdata.set('HAVE_LOWDOWN', lowdown.found().to_int()) + +readline_flavor = get_option('readline-flavor') +if readline_flavor == 'editline' + editline = dependency('libeditline', 'editline', version : '>=1.14') + deps_private += editline +elif readline_flavor == 'readline' + readline = dependency('readline') + deps_private += readline + configdata.set( + 'USE_READLINE', + 1, + description: 'Use readline instead of editline', + ) +else + error('illegal editline flavor', readline_flavor) +endif + +config_h = configure_file( + configuration : configdata, + output : 'config-cmd.hh', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + # '-include', 'config-fetchers.h', + '-include', 'config-expr.hh', + '-include', 'config-main.hh', + '-include', 'config-cmd.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'built-path.cc', + 'command-installable-value.cc', + 'command.cc', + 'common-eval-args.cc', + 'editor-for.cc', + 'installable-attr-path.cc', + 'installable-derived-path.cc', + 'installable-flake.cc', + 'installable-value.cc', + 'installables.cc', + 'legacy.cc', + 'markdown.cc', + 'misc-store-flags.cc', + 'network-proxy.cc', + 'repl-interacter.cc', + 'repl.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'built-path.hh', + 'command-installable-value.hh', + 'command.hh', + 'common-eval-args.hh', + 'compatibility-settings.hh', + 'editor-for.hh', + 'installable-attr-path.hh', + 'installable-derived-path.hh', + 'installable-flake.hh', + 'installable-value.hh', + 'installables.hh', + 'legacy.hh', + 'markdown.hh', + 'misc-store-flags.hh', + 'network-proxy.hh', + 'repl-interacter.hh', + 'repl.hh', +) + +this_library = library( + 'nixcmd', + sources, + dependencies : deps_public + deps_private + deps_other, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libcmd/meson.options b/src/libcmd/meson.options new file mode 100644 index 000000000..79ae4fa55 --- /dev/null +++ b/src/libcmd/meson.options @@ -0,0 +1,15 @@ +# vim: filetype=meson + +option( + 'markdown', + type: 'feature', + description: 'Enable Markdown rendering in the Nix binary (requires lowdown)', +) + +option( + 'readline-flavor', + type : 'combo', + choices : ['editline', 'readline'], + value : 'editline', + description : 'Which library to use for nice line editing with the Nix language REPL', +) diff --git a/src/libcmd/misc-store-flags.cc b/src/libcmd/misc-store-flags.cc index e66d3f63b..06552c032 100644 --- a/src/libcmd/misc-store-flags.cc +++ b/src/libcmd/misc-store-flags.cc @@ -81,9 +81,15 @@ Args::Flag fileIngestionMethod(FileIngestionMethod * method) How to compute the hash of the input. One of: - - `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function. + - `nar` (the default): + Serialises the input as a + [Nix Archive](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) + and passes that to the hash function. - - `flat`: Assumes that the input is a single file and directly passes it to the hash function; + - `flat`: + Assumes that the input is a single file and + [directly passes](@docroot@/store/file-system-object/content-address.md#serial-flat) + it to the hash function. )", .labels = {"file-ingestion-method"}, .handler = {[method](std::string s) { @@ -101,15 +107,23 @@ Args::Flag contentAddressMethod(ContentAddressMethod * method) How to compute the content-address of the store object. One of: - - `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function. + - [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive) + (the default): + Serialises the input as a + [Nix Archive](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) + and passes that to the hash function. - - `flat`: Assumes that the input is a single file and directly passes it to the hash function; + - [`flat`](@docroot@/store/store-object/content-address.md#method-flat): + Assumes that the input is a single file and + [directly passes](@docroot@/store/file-system-object/content-address.md#serial-flat) + it to the hash function. - - `text`: Like `flat`, but used for - [derivations](@docroot@/glossary.md#store-derivation) serialized in store object and + - [`text`](@docroot@/store/store-object/content-address.md#method-text): + Like `flat`, but used for + [derivations](@docroot@/glossary.md#store-derivation) serialized in store object and [`builtins.toFile`](@docroot@/language/builtins.html#builtins-toFile). For advanced use-cases only; - for regular usage prefer `nar` and `flat. + for regular usage prefer `nar` and `flat`. )", .labels = {"content-address-method"}, .handler = {[method](std::string s) { diff --git a/src/libcmd/network-proxy.cc b/src/libcmd/network-proxy.cc new file mode 100644 index 000000000..738bf6147 --- /dev/null +++ b/src/libcmd/network-proxy.cc @@ -0,0 +1,50 @@ +#include "network-proxy.hh" + +#include + +#include "environment-variables.hh" + +namespace nix { + +static const StringSet lowercaseVariables{"http_proxy", "https_proxy", "ftp_proxy", "all_proxy", "no_proxy"}; + +static StringSet getAllVariables() +{ + StringSet variables = lowercaseVariables; + for (const auto & variable : lowercaseVariables) { + std::string upperVariable; + std::transform( + variable.begin(), variable.end(), upperVariable.begin(), [](unsigned char c) { return std::toupper(c); }); + variables.insert(std::move(upperVariable)); + } + return variables; +} + +const StringSet networkProxyVariables = getAllVariables(); + +static StringSet getExcludingNoProxyVariables() +{ + static const StringSet excludeVariables{"no_proxy", "NO_PROXY"}; + StringSet variables; + std::set_difference( + networkProxyVariables.begin(), + networkProxyVariables.end(), + excludeVariables.begin(), + excludeVariables.end(), + std::inserter(variables, variables.begin())); + return variables; +} + +static const StringSet excludingNoProxyVariables = getExcludingNoProxyVariables(); + +bool haveNetworkProxyConnection() +{ + for (const auto & variable : excludingNoProxyVariables) { + if (getEnv(variable).has_value()) { + return true; + } + } + return false; +} + +} diff --git a/src/libcmd/network-proxy.hh b/src/libcmd/network-proxy.hh new file mode 100644 index 000000000..0b6856acb --- /dev/null +++ b/src/libcmd/network-proxy.hh @@ -0,0 +1,22 @@ +#pragma once +///@file + +#include "types.hh" + +namespace nix { + +/** + * Environment variables relating to network proxying. These are used by + * a few misc commands. + * + * See the Environment section of https://curl.se/docs/manpage.html for details. + */ +extern const StringSet networkProxyVariables; + +/** + * Heuristically check if there is a proxy connection by checking for defined + * proxy variables. + */ +bool haveNetworkProxyConnection(); + +} diff --git a/src/libcmd/package.nix b/src/libcmd/package.nix new file mode 100644 index 000000000..cde494901 --- /dev/null +++ b/src/libcmd/package.nix @@ -0,0 +1,104 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-util +, nix-store +, nix-fetchers +, nix-expr +, nix-flake +, nix-main +, editline +, readline +, lowdown +, nlohmann_json + +# Configuration Options + +, version + +# Whether to enable Markdown rendering in the Nix binary. +, enableMarkdown ? !stdenv.hostPlatform.isWindows + +# Which interactive line editor library to use for Nix's repl. +# +# Currently supported choices are: +# +# - editline (default) +# - readline +, readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline" +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-cmd"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + ({ inherit editline readline; }.${readlineFlavor}) + ] ++ lib.optional enableMarkdown lowdown; + + propagatedBuildInputs = [ + nix-util + nix-store + nix-fetchers + nix-expr + nix-flake + nix-main + nlohmann_json + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + mesonFlags = [ + (lib.mesonEnable "markdown" enableMarkdown) + (lib.mesonOption "readline-flavor" readlineFlavor) + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libcmd/repl-interacter.cc b/src/libcmd/repl-interacter.cc new file mode 100644 index 000000000..187af46ea --- /dev/null +++ b/src/libcmd/repl-interacter.cc @@ -0,0 +1,206 @@ +#include + +#ifdef USE_READLINE +#include +#include +#else +// editline < 1.15.2 don't wrap their API for C++ usage +// (added in https://github.com/troglobit/editline/commit/91398ceb3427b730995357e9d120539fb9bb7461). +// This results in linker errors due to to name-mangling of editline C symbols. +// For compatibility with these versions, we wrap the API here +// (wrapping multiple times on newer versions is no problem). +extern "C" { +#include +} +#endif + +#include "signals.hh" +#include "finally.hh" +#include "repl-interacter.hh" +#include "file-system.hh" +#include "repl.hh" +#include "environment-variables.hh" + +namespace nix { + +namespace { +// Used to communicate to NixRepl::getLine whether a signal occurred in ::readline. +volatile sig_atomic_t g_signal_received = 0; + +void sigintHandler(int signo) +{ + g_signal_received = signo; +} +}; + +static detail::ReplCompleterMixin * curRepl; // ugly + +#ifndef USE_READLINE +static char * completionCallback(char * s, int * match) +{ + auto possible = curRepl->completePrefix(s); + if (possible.size() == 1) { + *match = 1; + auto * res = strdup(possible.begin()->c_str() + strlen(s)); + if (!res) + throw Error("allocation failure"); + return res; + } else if (possible.size() > 1) { + auto checkAllHaveSameAt = [&](size_t pos) { + auto & first = *possible.begin(); + for (auto & p : possible) { + if (p.size() <= pos || p[pos] != first[pos]) + return false; + } + return true; + }; + size_t start = strlen(s); + size_t len = 0; + while (checkAllHaveSameAt(start + len)) + ++len; + if (len > 0) { + *match = 1; + auto * res = strdup(std::string(*possible.begin(), start, len).c_str()); + if (!res) + throw Error("allocation failure"); + return res; + } + } + + *match = 0; + return nullptr; +} + +static int listPossibleCallback(char * s, char *** avp) +{ + auto possible = curRepl->completePrefix(s); + + if (possible.size() > (std::numeric_limits::max() / sizeof(char *))) + throw Error("too many completions"); + + int ac = 0; + char ** vp = nullptr; + + auto check = [&](auto * p) { + if (!p) { + if (vp) { + while (--ac >= 0) + free(vp[ac]); + free(vp); + } + throw Error("allocation failure"); + } + return p; + }; + + vp = check((char **) malloc(possible.size() * sizeof(char *))); + + for (auto & p : possible) + vp[ac++] = check(strdup(p.c_str())); + + *avp = vp; + + return ac; +} +#endif + +ReadlineLikeInteracter::Guard ReadlineLikeInteracter::init(detail::ReplCompleterMixin * repl) +{ + // Allow nix-repl specific settings in .inputrc + rl_readline_name = "nix-repl"; + try { + createDirs(dirOf(historyFile)); + } catch (SystemError & e) { + logWarning(e.info()); + } +#ifndef USE_READLINE + el_hist_size = 1000; +#endif + read_history(historyFile.c_str()); + auto oldRepl = curRepl; + curRepl = repl; + Guard restoreRepl([oldRepl] { curRepl = oldRepl; }); +#ifndef USE_READLINE + rl_set_complete_func(completionCallback); + rl_set_list_possib_func(listPossibleCallback); +#endif + return restoreRepl; +} + +static constexpr const char * promptForType(ReplPromptType promptType) +{ + switch (promptType) { + case ReplPromptType::ReplPrompt: + return "nix-repl> "; + case ReplPromptType::ContinuationPrompt: + return " "; + } + assert(false); +} + +bool ReadlineLikeInteracter::getLine(std::string & input, ReplPromptType promptType) +{ +#ifndef _WIN32 // TODO use more signals.hh for this + struct sigaction act, old; + sigset_t savedSignalMask, set; + + auto setupSignals = [&]() { + act.sa_handler = sigintHandler; + sigfillset(&act.sa_mask); + act.sa_flags = 0; + if (sigaction(SIGINT, &act, &old)) + throw SysError("installing handler for SIGINT"); + + sigemptyset(&set); + sigaddset(&set, SIGINT); + if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask)) + throw SysError("unblocking SIGINT"); + }; + auto restoreSignals = [&]() { + if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) + throw SysError("restoring signals"); + + if (sigaction(SIGINT, &old, 0)) + throw SysError("restoring handler for SIGINT"); + }; + + setupSignals(); +#endif + char * s = readline(promptForType(promptType)); + Finally doFree([&]() { free(s); }); +#ifndef _WIN32 // TODO use more signals.hh for this + restoreSignals(); +#endif + + if (g_signal_received) { + g_signal_received = 0; + input.clear(); + return true; + } + + // editline doesn't echo the input to the output when non-interactive, unlike readline + // this results in a different behavior when running tests. The echoing is + // quite useful for reading the test output, so we add it here. + if (auto e = getEnv("_NIX_TEST_REPL_ECHO"); s && e && *e == "1") + { +#ifndef USE_READLINE + // This is probably not right for multi-line input, but we don't use that + // in the characterisation tests, so it's fine. + std::cout << promptForType(promptType) << s << std::endl; +#endif + } + + if (!s) + return false; + input += s; + input += '\n'; + + return true; +} + +ReadlineLikeInteracter::~ReadlineLikeInteracter() +{ + write_history(historyFile.c_str()); +} + +}; diff --git a/src/libcmd/repl-interacter.hh b/src/libcmd/repl-interacter.hh new file mode 100644 index 000000000..cc70efd07 --- /dev/null +++ b/src/libcmd/repl-interacter.hh @@ -0,0 +1,48 @@ +#pragma once +/// @file + +#include "finally.hh" +#include "types.hh" +#include +#include + +namespace nix { + +namespace detail { +/** Provides the completion hooks for the repl, without exposing its complete + * internals. */ +struct ReplCompleterMixin { + virtual StringSet completePrefix(const std::string & prefix) = 0; +}; +}; + +enum class ReplPromptType { + ReplPrompt, + ContinuationPrompt, +}; + +class ReplInteracter +{ +public: + using Guard = Finally>; + + virtual Guard init(detail::ReplCompleterMixin * repl) = 0; + /** Returns a boolean of whether the interacter got EOF */ + virtual bool getLine(std::string & input, ReplPromptType promptType) = 0; + virtual ~ReplInteracter(){}; +}; + +class ReadlineLikeInteracter : public virtual ReplInteracter +{ + std::string historyFile; +public: + ReadlineLikeInteracter(std::string historyFile) + : historyFile(historyFile) + { + } + virtual Guard init(detail::ReplCompleterMixin * repl) override; + virtual bool getLine(std::string & input, ReplPromptType promptType) override; + virtual ~ReadlineLikeInteracter() override; +}; + +}; diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 8b83608fa..efc04b029 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -1,34 +1,17 @@ #include #include #include -#include - -#include - -#ifdef USE_READLINE -#include -#include -#else -// editline < 1.15.2 don't wrap their API for C++ usage -// (added in https://github.com/troglobit/editline/commit/91398ceb3427b730995357e9d120539fb9bb7461). -// This results in linker errors due to to name-mangling of editline C symbols. -// For compatibility with these versions, we wrap the API here -// (wrapping multiple times on newer versions is no problem). -extern "C" { -#include -} -#endif +#include "repl-interacter.hh" #include "repl.hh" #include "ansicolor.hh" -#include "signals.hh" #include "shared.hh" +#include "config-global.hh" #include "eval.hh" -#include "eval-cache.hh" -#include "eval-inline.hh" #include "eval-settings.hh" #include "attr-path.hh" +#include "signals.hh" #include "store-api.hh" #include "log-store.hh" #include "common-eval-args.hh" @@ -38,18 +21,21 @@ extern "C" { #include "flake/flake.hh" #include "flake/lockfile.hh" #include "users.hh" -#include "terminal.hh" #include "editor-for.hh" #include "finally.hh" #include "markdown.hh" #include "local-fs-store.hh" #include "print.hh" +#include "ref.hh" +#include "value.hh" #if HAVE_BOEHMGC #define GC_INCLUDE_NEW #include #endif +#include "strings.hh" + namespace nix { /** @@ -75,6 +61,7 @@ enum class ProcessLineResult { struct NixRepl : AbstractNixRepl + , detail::ReplCompleterMixin #if HAVE_BOEHMGC , gc #endif @@ -90,17 +77,16 @@ struct NixRepl int displ; StringSet varNames; - const Path historyFile; + std::unique_ptr interacter; - NixRepl(const SearchPath & searchPath, nix::ref store,ref state, + NixRepl(const LookupPath & lookupPath, nix::ref store,ref state, std::function getValues); - virtual ~NixRepl(); + virtual ~NixRepl() = default; ReplExitStatus mainLoop() override; void initEnv() override; - StringSet completePrefix(const std::string & prefix); - bool getLine(std::string & input, const std::string & prompt); + virtual StringSet completePrefix(const std::string & prefix) override; StorePath getDerivationPath(Value & v); ProcessLineResult processLine(std::string line); @@ -123,7 +109,8 @@ struct NixRepl .force = true, .derivationPaths = true, .maxDepth = maxDepth, - .prettyIndent = 2 + .prettyIndent = 2, + .errors = ErrorPrintBehavior::ThrowTopLevel, }); } }; @@ -137,111 +124,33 @@ std::string removeWhitespace(std::string s) } -NixRepl::NixRepl(const SearchPath & searchPath, nix::ref store, ref state, +NixRepl::NixRepl(const LookupPath & lookupPath, nix::ref store, ref state, std::function getValues) : AbstractNixRepl(state) , debugTraceIndex(0) , getValues(getValues) , staticEnv(new StaticEnv(nullptr, state->staticBaseEnv.get())) - , historyFile(getDataDir() + "/nix/repl-history") + , interacter(make_unique(getDataDir() + "/nix/repl-history")) { } - -NixRepl::~NixRepl() -{ - write_history(historyFile.c_str()); -} - void runNix(Path program, const Strings & args, const std::optional & input = {}) { auto subprocessEnv = getEnv(); subprocessEnv["NIX_CONFIG"] = globalConfig.toKeyValue(); - + //isInteractive avoid grabling interactive commands runProgram2(RunOptions { .program = settings.nixBinDir+ "/" + program, .args = args, .environment = subprocessEnv, .input = input, + .isInteractive = true, }); return; } -static NixRepl * curRepl; // ugly - -static char * completionCallback(char * s, int *match) { - auto possible = curRepl->completePrefix(s); - if (possible.size() == 1) { - *match = 1; - auto *res = strdup(possible.begin()->c_str() + strlen(s)); - if (!res) throw Error("allocation failure"); - return res; - } else if (possible.size() > 1) { - auto checkAllHaveSameAt = [&](size_t pos) { - auto &first = *possible.begin(); - for (auto &p : possible) { - if (p.size() <= pos || p[pos] != first[pos]) - return false; - } - return true; - }; - size_t start = strlen(s); - size_t len = 0; - while (checkAllHaveSameAt(start + len)) ++len; - if (len > 0) { - *match = 1; - auto *res = strdup(std::string(*possible.begin(), start, len).c_str()); - if (!res) throw Error("allocation failure"); - return res; - } - } - - *match = 0; - return nullptr; -} - -static int listPossibleCallback(char *s, char ***avp) { - auto possible = curRepl->completePrefix(s); - - if (possible.size() > (INT_MAX / sizeof(char*))) - throw Error("too many completions"); - - int ac = 0; - char **vp = nullptr; - - auto check = [&](auto *p) { - if (!p) { - if (vp) { - while (--ac >= 0) - free(vp[ac]); - free(vp); - } - throw Error("allocation failure"); - } - return p; - }; - - vp = check((char **)malloc(possible.size() * sizeof(char*))); - - for (auto & p : possible) - vp[ac++] = check(strdup(p.c_str())); - - *avp = vp; - - return ac; -} - -namespace { - // Used to communicate to NixRepl::getLine whether a signal occurred in ::readline. - volatile sig_atomic_t g_signal_received = 0; - - void sigintHandler(int signo) { - g_signal_received = signo; - } -} - static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positions, const DebugTrace & dt) { if (dt.isError) @@ -281,24 +190,7 @@ ReplExitStatus NixRepl::mainLoop() loadFiles(); - // Allow nix-repl specific settings in .inputrc - rl_readline_name = "nix-repl"; - try { - createDirs(dirOf(historyFile)); - } catch (SystemError & e) { - logWarning(e.info()); - } -#ifndef USE_READLINE - el_hist_size = 1000; -#endif - read_history(historyFile.c_str()); - auto oldRepl = curRepl; - curRepl = this; - Finally restoreRepl([&] { curRepl = oldRepl; }); -#ifndef USE_READLINE - rl_set_complete_func(completionCallback); - rl_set_list_possib_func(listPossibleCallback); -#endif + auto _guard = interacter->init(static_cast(this)); std::string input; @@ -307,7 +199,7 @@ ReplExitStatus NixRepl::mainLoop() logger->pause(); // When continuing input from previous lines, don't print a prompt, just align to the same // number of chars as the prompt. - if (!getLine(input, input.empty() ? "nix-repl> " : " ")) { + if (!interacter->getLine(input, input.empty() ? ReplPromptType::ReplPrompt : ReplPromptType::ContinuationPrompt)) { // Ctrl-D should exit the debugger. state->debugStop = false; logger->cout(""); @@ -325,7 +217,7 @@ ReplExitStatus NixRepl::mainLoop() case ProcessLineResult::PromptAgain: break; default: - abort(); + unreachable(); } } catch (ParseError & e) { if (e.msg().find("unexpected end of file") != std::string::npos) { @@ -336,13 +228,7 @@ ReplExitStatus NixRepl::mainLoop() printMsg(lvlError, e.msg()); } } catch (EvalError & e) { - // in debugger mode, an EvalError should trigger another repl session. - // when that session returns the exception will land here. No need to show it again; - // show the error for this repl session instead. - if (state->debugRepl && !state->debugTraces.empty()) - showDebugTrace(std::cout, state->positions, state->debugTraces.front()); - else - printMsg(lvlError, e.msg()); + printMsg(lvlError, e.msg()); } catch (Error & e) { printMsg(lvlError, e.msg()); } catch (Interrupted & e) { @@ -356,51 +242,6 @@ ReplExitStatus NixRepl::mainLoop() } } - -bool NixRepl::getLine(std::string & input, const std::string & prompt) -{ - struct sigaction act, old; - sigset_t savedSignalMask, set; - - auto setupSignals = [&]() { - act.sa_handler = sigintHandler; - sigfillset(&act.sa_mask); - act.sa_flags = 0; - if (sigaction(SIGINT, &act, &old)) - throw SysError("installing handler for SIGINT"); - - sigemptyset(&set); - sigaddset(&set, SIGINT); - if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask)) - throw SysError("unblocking SIGINT"); - }; - auto restoreSignals = [&]() { - if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) - throw SysError("restoring signals"); - - if (sigaction(SIGINT, &old, 0)) - throw SysError("restoring handler for SIGINT"); - }; - - setupSignals(); - char * s = readline(prompt.c_str()); - Finally doFree([&]() { free(s); }); - restoreSignals(); - - if (g_signal_received) { - g_signal_received = 0; - input.clear(); - return true; - } - - if (!s) - return false; - input += s; - input += '\n'; - return true; -} - - StringSet NixRepl::completePrefix(const std::string & prefix) { StringSet completions; @@ -421,11 +262,14 @@ StringSet NixRepl::completePrefix(const std::string & prefix) try { auto dir = std::string(cur, 0, slash); auto prefix2 = std::string(cur, slash + 1); - for (auto & entry : readDirectory(dir == "" ? "/" : dir)) { - if (entry.name[0] != '.' && hasPrefix(entry.name, prefix2)) - completions.insert(prev + dir + "/" + entry.name); + for (auto & entry : std::filesystem::directory_iterator{dir == "" ? "/" : dir}) { + checkInterrupt(); + auto name = entry.path().filename().string(); + if (name[0] != '.' && hasPrefix(name, prefix2)) + completions.insert(prev + entry.path().string()); } } catch (Error &) { + } catch (std::filesystem::filesystem_error &) { } } else if ((dot = cur.rfind('.')) == std::string::npos) { /* This is a variable name; look it up in the current scope. */ @@ -452,7 +296,7 @@ StringSet NixRepl::completePrefix(const std::string & prefix) e->eval(*state, *env, v); state->forceAttrs(v, noPos, "while evaluating an attrset for the purpose of completion (this error should not be displayed; file an issue?)"); - for (auto & i : *v.attrs) { + for (auto & i : *v.attrs()) { std::string_view name = state->symbols[i.name]; if (name.substr(0, cur2.size()) != cur2) continue; completions.insert(concatStrings(prev, expr, ".", name)); @@ -464,6 +308,8 @@ StringSet NixRepl::completePrefix(const std::string & prefix) // Quietly ignore evaluation errors. } catch (BadURL & e) { // Quietly ignore BadURL flake-related errors. + } catch (FileNotFound & e) { + // Quietly ignore non-existent file beeing `import`-ed. } } @@ -519,7 +365,7 @@ ProcessLineResult NixRepl::processLine(std::string line) if (line.empty()) return ProcessLineResult::PromptAgain; - _isInterrupted = false; + setInterrupted(false); std::string command, arg; @@ -548,6 +394,7 @@ ProcessLineResult NixRepl::processLine(std::string line) << " :l, :load Load Nix expression and add it to scope\n" << " :lf, :load-flake Load Nix flake and add it to scope\n" << " :p, :print Evaluate and print expression recursively\n" + << " Strings are printed directly, without escaping.\n" << " :q, :quit Exit nix-repl\n" << " :r, :reload Reload all files\n" << " :sh Build dependencies of derivation, then start\n" @@ -651,7 +498,7 @@ ProcessLineResult NixRepl::processLine(std::string line) auto path = state->coerceToPath(noPos, v, context, "while evaluating the filename to edit"); return {path, 0}; } else if (v.isLambda()) { - auto pos = state->positions[v.lambda.fun->pos]; + auto pos = state->positions[v.payload.lambda.fun->pos]; if (auto path = std::get_if(&pos.origin)) return {*path, pos.line}; else @@ -669,7 +516,7 @@ ProcessLineResult NixRepl::processLine(std::string line) // runProgram redirects stdout to a StringSink, // using runProgram2 to allow editors to display their UI - runProgram2(RunOptions { .program = editor, .searchPath = true, .args = args }); + runProgram2(RunOptions { .program = editor, .lookupPath = true, .args = args , .isInteractive = true }); // Reload right after exiting the editor state->resetFileCache(); @@ -755,7 +602,11 @@ ProcessLineResult NixRepl::processLine(std::string line) else if (command == ":p" || command == ":print") { Value v; evalString(arg, v); - printValue(std::cout, v); + if (v.type() == nString) { + std::cout << v.string_view(); + } else { + printValue(std::cout, v); + } std::cout << std::endl; } @@ -766,6 +617,35 @@ ProcessLineResult NixRepl::processLine(std::string line) else if (command == ":doc") { Value v; + + auto expr = parseString(arg); + std::string fallbackName; + PosIdx fallbackPos; + DocComment fallbackDoc; + if (auto select = dynamic_cast(expr)) { + Value vAttrs; + auto name = select->evalExceptFinalSelect(*state, *env, vAttrs); + fallbackName = state->symbols[name]; + + state->forceAttrs(vAttrs, noPos, "while evaluating an attribute set to look for documentation"); + auto attrs = vAttrs.attrs(); + assert(attrs); + auto attr = attrs->get(name); + if (!attr) { + // When missing, trigger the normal exception + // e.g. :doc builtins.foo + // behaves like + // nix-repl> builtins.foo + // error: attribute 'foo' missing + evalString(arg, v); + assert(false); + } + if (attr->pos) { + fallbackPos = attr->pos; + fallbackDoc = state->getDocCommentForPos(fallbackPos); + } + } + evalString(arg, v); if (auto doc = state->getDoc(v)) { std::string markdown; @@ -783,6 +663,19 @@ ProcessLineResult NixRepl::processLine(std::string line) markdown += stripIndentation(doc->doc); logger->cout(trim(renderMarkdownToTerminal(markdown))); + } else if (fallbackPos) { + std::stringstream ss; + ss << "Attribute `" << fallbackName << "`\n\n"; + ss << " … defined at " << state->positions[fallbackPos] << "\n\n"; + if (fallbackDoc) { + ss << fallbackDoc.getInnerText(state->positions); + } else { + ss << "No documentation found.\n\n"; + } + + auto markdown = ss.str(); + logger->cout(trim(renderMarkdownToTerminal(markdown))); + } else throw Error("value does not have documentation"); } @@ -840,14 +733,14 @@ void NixRepl::loadFlake(const std::string & flakeRefS) if (flakeRefS.empty()) throw Error("cannot use ':load-flake' without a path specified. (Use '.' for the current working directory.)"); - auto flakeRef = parseFlakeRef(flakeRefS, absPath("."), true); + auto flakeRef = parseFlakeRef(fetchSettings, flakeRefS, absPath("."), true); if (evalSettings.pureEval && !flakeRef.input.isLocked()) throw Error("cannot use ':load-flake' on locked flake reference '%s' (use --impure to override)", flakeRefS); Value v; flake::callFlake(*state, - flake::lockFlake(*state, flakeRef, + flake::lockFlake(flakeSettings, *state, flakeRef, flake::LockFlags { .updateLockFile = false, .useRegistries = !evalSettings.pureEval, @@ -899,17 +792,17 @@ void NixRepl::loadFiles() void NixRepl::addAttrsToScope(Value & attrs) { state->forceAttrs(attrs, [&]() { return attrs.determinePos(noPos); }, "while evaluating an attribute set to be merged in the global scope"); - if (displ + attrs.attrs->size() >= envSize) + if (displ + attrs.attrs()->size() >= envSize) throw Error("environment full; cannot add more variables"); - for (auto & i : *attrs.attrs) { + for (auto & i : *attrs.attrs()) { staticEnv->vars.emplace_back(i.name, displ); env->values[displ++] = i.value; varNames.emplace(state->symbols[i.name]); } staticEnv->sort(); staticEnv->deduplicate(); - notice("Added %1% variables.", attrs.attrs->size()); + notice("Added %1% variables.", attrs.attrs()->size()); } @@ -941,11 +834,11 @@ void NixRepl::evalString(std::string s, Value & v) std::unique_ptr AbstractNixRepl::create( - const SearchPath & searchPath, nix::ref store, ref state, + const LookupPath & lookupPath, nix::ref store, ref state, std::function getValues) { return std::make_unique( - searchPath, + lookupPath, openStore(), state, getValues @@ -961,9 +854,9 @@ ReplExitStatus AbstractNixRepl::runSimple( NixRepl::AnnotatedValues values; return values; }; - SearchPath searchPath = {}; + LookupPath lookupPath = {}; auto repl = std::make_unique( - searchPath, + lookupPath, openStore(), evalState, getValues diff --git a/src/libcmd/repl.hh b/src/libcmd/repl.hh index 21aa8bfc7..3fd4b2c39 100644 --- a/src/libcmd/repl.hh +++ b/src/libcmd/repl.hh @@ -3,11 +3,6 @@ #include "eval.hh" -#if HAVE_BOEHMGC -#define GC_INCLUDE_NEW -#include -#endif - namespace nix { struct AbstractNixRepl @@ -25,7 +20,7 @@ struct AbstractNixRepl typedef std::vector> AnnotatedValues; static std::unique_ptr create( - const SearchPath & searchPath, nix::ref store, ref state, + const LookupPath & lookupPath, nix::ref store, ref state, std::function getValues); static ReplExitStatus runSimple( diff --git a/src/libexpr-c/.version b/src/libexpr-c/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libexpr-c/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libexpr-c/build-utils-meson b/src/libexpr-c/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libexpr-c/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libexpr-c/local.mk b/src/libexpr-c/local.mk new file mode 100644 index 000000000..227a4095b --- /dev/null +++ b/src/libexpr-c/local.mk @@ -0,0 +1,25 @@ +libraries += libexprc + +libexprc_NAME = libnixexprc + +libexprc_DIR := $(d) + +libexprc_SOURCES := \ + $(wildcard $(d)/*.cc) \ + +# Not just for this library itself, but also for downstream libraries using this library + +INCLUDE_libexprc := -I $(d) +libexprc_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libutilc) \ + $(INCLUDE_libfetchers) \ + $(INCLUDE_libstore) $(INCLUDE_libstorec) \ + $(INCLUDE_libexpr) $(INCLUDE_libexprc) + +libexprc_LIBS = libutil libutilc libstore libstorec libfetchers libexpr + +libexprc_LDFLAGS += $(THREAD_LDFLAGS) + +$(eval $(call install-file-in, $(d)/nix-expr-c.pc, $(libdir)/pkgconfig, 0644)) + +libexprc_FORCE_INSTALL := 1 + diff --git a/src/libexpr-c/meson.build b/src/libexpr-c/meson.build new file mode 100644 index 000000000..6db5b83b8 --- /dev/null +++ b/src/libexpr-c/meson.build @@ -0,0 +1,93 @@ +project('nix-expr-c', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), + dependency('nix-expr'), +] +deps_public_maybe_subproject = [ + dependency('nix-util-c'), + dependency('nix-store-c'), +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +# TODO rename, because it will conflict with downstream projects +configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) + +config_h = configure_file( + configuration : configdata, + output : 'config-expr.h', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + + # From C++ libraries, only for internals + '-include', 'config-util.hh', + '-include', 'config-store.hh', + '-include', 'config-expr.hh', + + # From C libraries, for our public, installed headers too + '-include', 'config-util.h', + '-include', 'config-store.h', + '-include', 'config-expr.h', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'nix_api_expr.cc', + 'nix_api_external.cc', + 'nix_api_value.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'nix_api_expr.h', + 'nix_api_external.h', + 'nix_api_value.h', +) + +# TODO move this header to libexpr, maybe don't use it in tests? +headers += files('nix_api_expr_internal.h') + +subdir('build-utils-meson/export-all-symbols') + +this_library = library( + 'nixexprc', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + link_args: linker_export_flags, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libexpr-c/nix-expr-c.pc.in b/src/libexpr-c/nix-expr-c.pc.in new file mode 100644 index 000000000..06897064d --- /dev/null +++ b/src/libexpr-c/nix-expr-c.pc.in @@ -0,0 +1,10 @@ +prefix=@prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: Nix +Description: Nix Language Evaluator - C API +Version: @PACKAGE_VERSION@ +Requires: nix-store-c +Libs: -L${libdir} -lnixexprc +Cflags: -I${includedir}/nix diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc new file mode 100644 index 000000000..8f21d7022 --- /dev/null +++ b/src/libexpr-c/nix_api_expr.cc @@ -0,0 +1,213 @@ +#include +#include +#include + +#include "eval.hh" +#include "eval-gc.hh" +#include "globals.hh" +#include "eval-settings.hh" + +#include "nix_api_expr.h" +#include "nix_api_expr_internal.h" +#include "nix_api_store.h" +#include "nix_api_store_internal.h" +#include "nix_api_util.h" +#include "nix_api_util_internal.h" + +#if HAVE_BOEHMGC +# include +# define GC_INCLUDE_NEW 1 +# include "gc_cpp.h" +#endif + +nix_err nix_libexpr_init(nix_c_context * context) +{ + if (context) + context->last_err_code = NIX_OK; + { + auto ret = nix_libutil_init(context); + if (ret != NIX_OK) + return ret; + } + { + auto ret = nix_libstore_init(context); + if (ret != NIX_OK) + return ret; + } + try { + nix::initGC(); + } + NIXC_CATCH_ERRS +} + +nix_err nix_expr_eval_from_string( + nix_c_context * context, EvalState * state, const char * expr, const char * path, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + nix::Expr * parsedExpr = state->state.parseExprFromString(expr, state->state.rootPath(nix::CanonPath(path))); + state->state.eval(parsedExpr, value->value); + state->state.forceValue(value->value, nix::noPos); + } + NIXC_CATCH_ERRS +} + +nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, nix_value * arg, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + state->state.callFunction(fn->value, arg->value, value->value, nix::noPos); + state->state.forceValue(value->value, nix::noPos); + } + NIXC_CATCH_ERRS +} + +nix_err nix_value_call_multi(nix_c_context * context, EvalState * state, nix_value * fn, size_t nargs, nix_value ** args, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + state->state.callFunction(fn->value, nargs, (nix::Value * *)args, value->value, nix::noPos); + state->state.forceValue(value->value, nix::noPos); + } + NIXC_CATCH_ERRS +} + +nix_err nix_value_force(nix_c_context * context, EvalState * state, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + state->state.forceValue(value->value, nix::noPos); + } + NIXC_CATCH_ERRS +} + +nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + state->state.forceValueDeep(value->value); + } + NIXC_CATCH_ERRS +} + +EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c, Store * store) +{ + if (context) + context->last_err_code = NIX_OK; + try { + nix::Strings lookupPath; + if (lookupPath_c != nullptr) + for (size_t i = 0; lookupPath_c[i] != nullptr; i++) + lookupPath.push_back(lookupPath_c[i]); + + void * p = ::operator new( + sizeof(EvalState), + static_cast(alignof(EvalState))); + auto * p2 = static_cast(p); + new (p) EvalState { + .fetchSettings = nix::fetchers::Settings{}, + .settings = nix::EvalSettings{ + nix::settings.readOnlyMode, + }, + .state = nix::EvalState( + nix::LookupPath::parse(lookupPath), + store->ptr, + p2->fetchSettings, + p2->settings), + }; + loadConfFile(p2->settings); + return p2; + } + NIXC_CATCH_ERRS_NULL +} + +void nix_state_free(EvalState * state) +{ + delete state; +} + +#if HAVE_BOEHMGC +std::unordered_map< + const void *, + unsigned int, + std::hash, + std::equal_to, + traceable_allocator
; }' - > { a = ; } - > ``` - > - > For human-readable output, `nix eval` (experimental) is more informative: - > - > ```console - > $ nix-instantiate --eval --expr 'a: a' - > - > $ nix eval --expr 'a: a' - > «lambda @ «string»:1:1» - > ``` - > - > For machine-readable output, the `--xml` option produces unambiguous - > output: - > - > ```console - > $ nix-instantiate --eval --xml --expr '{ foo = ; }' - > - > - > - > - > - > - > - > - > ``` + Just parse the input files, and print their abstract syntax trees on + standard output as a Nix expression. - - `--find-file`\ - Look up the given files in Nix’s search path (as specified by the - `NIX_PATH` environment variable). If found, print the corresponding - absolute paths on standard output. For instance, if `NIX_PATH` is - `nixpkgs=/home/alice/nixpkgs`, then `nix-instantiate --find-file - nixpkgs/default.nix` will print `/home/alice/nixpkgs/default.nix`. +- `--eval` - - `--strict`\ - When used with `--eval`, recursively evaluate list elements and - attributes. Normally, such sub-expressions are left unevaluated - (since the Nix language is lazy). + Just parse and evaluate the input files, and print the resulting + values on standard output. No instantiation of store derivations + takes place. - > **Warning** - > - > This option can cause non-termination, because lazy data - > structures can be infinitely large. + > **Warning** + > + > This option produces output which can be parsed as a Nix expression which + > will produce a different result than the input expression when evaluated. + > For example, these two Nix expressions print the same result despite + > having different meaning: + > + > ```console + > $ nix-instantiate --eval --expr '{ a = {}; }' + > { a = ; } + > $ nix-instantiate --eval --expr '{ a = ; }' + > { a = ; } + > ``` + > + > For human-readable output, `nix eval` (experimental) is more informative: + > + > ```console + > $ nix-instantiate --eval --expr 'a: a' + > + > $ nix eval --expr 'a: a' + > «lambda @ «string»:1:1» + > ``` + > + > For machine-readable output, the `--xml` option produces unambiguous + > output: + > + > ```console + > $ nix-instantiate --eval --xml --expr '{ foo = ; }' + > + > + > + > + > + > + > + > + > ``` - - `--json`\ - When used with `--eval`, print the resulting value as an JSON - representation of the abstract syntax tree rather than as a Nix expression. +- `--find-file` - - `--xml`\ - When used with `--eval`, print the resulting value as an XML - representation of the abstract syntax tree rather than as a Nix expression. - The schema is the same as that used by the [`toXML` - built-in](../language/builtins.md). + Look up the given files in Nix’s search path (as specified by the + `NIX_PATH` environment variable). If found, print the corresponding + absolute paths on standard output. For instance, if `NIX_PATH` is + `nixpkgs=/home/alice/nixpkgs`, then `nix-instantiate --find-file + nixpkgs/default.nix` will print `/home/alice/nixpkgs/default.nix`. - - `--read-write-mode`\ - When used with `--eval`, perform evaluation in read/write mode so - nix language features that require it will still work (at the cost - of needing to do instantiation of every evaluated derivation). If - this option is not enabled, there may be uninstantiated store paths - in the final output. +- `--strict` + + When used with `--eval`, recursively evaluate list elements and + attributes. Normally, such sub-expressions are left unevaluated + (since the Nix language is lazy). + + > **Warning** + > + > This option can cause non-termination, because lazy data + > structures can be infinitely large. + +- `--json` + + When used with `--eval`, print the resulting value as an JSON + representation of the abstract syntax tree rather than as a Nix expression. + +- `--xml` + + When used with `--eval`, print the resulting value as an XML + representation of the abstract syntax tree rather than as a Nix expression. + The schema is the same as that used by the [`toXML` + built-in](../language/builtins.md). + +- `--read-write-mode` + + When used with `--eval`, perform evaluation in read/write mode so + nix language features that require it will still work (at the cost + of needing to do instantiation of every evaluated derivation). If + this option is not enabled, there may be uninstantiated store paths + in the final output. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-prefetch-url.md b/doc/manual/src/command-ref/nix-prefetch-url.md index 45ef01e02..ffab94b8a 100644 --- a/doc/manual/src/command-ref/nix-prefetch-url.md +++ b/doc/manual/src/command-ref/nix-prefetch-url.md @@ -39,27 +39,32 @@ the path of the downloaded file in the Nix store is also printed. # Options - - `--type` *hashAlgo*\ - Use the specified cryptographic hash algorithm, - which can be one of `md5`, `sha1`, `sha256`, and `sha512`. - The default is `sha256`. +- `--type` *hashAlgo* - - `--print-path`\ - Print the store path of the downloaded file on standard output. + Use the specified cryptographic hash algorithm, + which can be one of `md5`, `sha1`, `sha256`, and `sha512`. + The default is `sha256`. - - `--unpack`\ - Unpack the archive (which must be a tarball or zip file) and add the - result to the Nix store. The resulting hash can be used with - functions such as Nixpkgs’s `fetchzip` or `fetchFromGitHub`. +- `--print-path` - - `--executable`\ - Set the executable bit on the downloaded file. + Print the store path of the downloaded file on standard output. - - `--name` *name*\ - Override the name of the file in the Nix store. By default, this is - `hash-basename`, where *basename* is the last component of *url*. - Overriding the name is necessary when *basename* contains characters - that are not allowed in Nix store paths. +- `--unpack` + + Unpack the archive (which must be a tarball or zip file) and add the + result to the Nix store. The resulting hash can be used with + functions such as Nixpkgs’s `fetchzip` or `fetchFromGitHub`. + +- `--executable` + + Set the executable bit on the downloaded file. + +- `--name` *name* + + Override the name of the file in the Nix store. By default, this is + `hash-basename`, where *basename* is the last component of *url*. + Overriding the name is necessary when *basename* contains characters + that are not allowed in Nix store paths. # Examples diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index 1eaf3c36a..69a711bd5 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -60,55 +60,63 @@ All options not listed here are passed to `nix-store --realise`, except for `--arg` and `--attr` / `-A` which are passed to `nix-instantiate`. - - `--command` *cmd*\ - In the environment of the derivation, run the shell command *cmd*. - This command is executed in an interactive shell. (Use `--run` to - use a non-interactive shell instead.) However, a call to `exit` is - implicitly added to the command, so the shell will exit after - running the command. To prevent this, add `return` at the end; - e.g. `--command "echo Hello; return"` will print `Hello` and then - drop you into the interactive shell. This can be useful for doing - any additional initialisation. +- `--command` *cmd* - - `--run` *cmd*\ - Like `--command`, but executes the command in a non-interactive - shell. This means (among other things) that if you hit Ctrl-C while - the command is running, the shell exits. + In the environment of the derivation, run the shell command *cmd*. + This command is executed in an interactive shell. (Use `--run` to + use a non-interactive shell instead.) However, a call to `exit` is + implicitly added to the command, so the shell will exit after + running the command. To prevent this, add `return` at the end; + e.g. `--command "echo Hello; return"` will print `Hello` and then + drop you into the interactive shell. This can be useful for doing + any additional initialisation. - - `--exclude` *regexp*\ - Do not build any dependencies whose store path matches the regular - expression *regexp*. This option may be specified multiple times. +- `--run` *cmd* - - `--pure`\ - If this flag is specified, the environment is almost entirely - cleared before the interactive shell is started, so you get an - environment that more closely corresponds to the “real†Nix build. A - few variables, in particular `HOME`, `USER` and `DISPLAY`, are - retained. + Like `--command`, but executes the command in a non-interactive + shell. This means (among other things) that if you hit Ctrl-C while + the command is running, the shell exits. - - `--packages` / `-p` *packages*…\ - Set up an environment in which the specified packages are present. - The command line arguments are interpreted as attribute names inside - the Nix Packages collection. Thus, `nix-shell --packages libjpeg openjdk` - will start a shell in which the packages denoted by the attribute - names `libjpeg` and `openjdk` are present. +- `--exclude` *regexp* - - `-i` *interpreter*\ - The chained script interpreter to be invoked by `nix-shell`. Only - applicable in `#!`-scripts (described below). + Do not build any dependencies whose store path matches the regular + expression *regexp*. This option may be specified multiple times. - - `--keep` *name*\ - When a `--pure` shell is started, keep the listed environment - variables. +- `--pure` + + If this flag is specified, the environment is almost entirely + cleared before the interactive shell is started, so you get an + environment that more closely corresponds to the “real†Nix build. A + few variables, in particular `HOME`, `USER` and `DISPLAY`, are + retained. + +- `--packages` / `-p` *packages*… + + Set up an environment in which the specified packages are present. + The command line arguments are interpreted as attribute names inside + the Nix Packages collection. Thus, `nix-shell --packages libjpeg openjdk` + will start a shell in which the packages denoted by the attribute + names `libjpeg` and `openjdk` are present. + +- `-i` *interpreter* + + The chained script interpreter to be invoked by `nix-shell`. Only + applicable in `#!`-scripts (described below). + +- `--keep` *name* + + When a `--pure` shell is started, keep the listed environment + variables. {{#include ./opt-common.md}} # Environment variables - - `NIX_BUILD_SHELL`\ - Shell used to start the interactive environment. Defaults to the - `bash` found in ``, falling back to the `bash` found in - `PATH` if not found. +- `NIX_BUILD_SHELL` + + Shell used to start the interactive environment. Defaults to the + `bash` found in ``, falling back to the `bash` found in + `PATH` if not found. {{#include ./env-common.md}} @@ -202,14 +210,14 @@ For example, here is a Python script that depends on Python and the ```python #! /usr/bin/env nix-shell -#! nix-shell -i python --packages python pythonPackages.prettytable +#! nix-shell -i python3 --packages python3 python3Packages.prettytable import prettytable # Print a simple table. t = prettytable.PrettyTable(["N", "N^2"]) for n in range(1, 10): t.add_row([n, n * n]) -print t +print(t) ``` Similarly, the following is a Perl script that specifies that it @@ -289,3 +297,8 @@ with import {}; runCommand "dummy" { buildInputs = [ python pythonPackages.prettytable ]; } "" ``` + +The script's file name is passed as the first argument to the interpreter specified by the `-i` flag. + +Aside from the very first line, which is a directive to the operating system, the additional `#! nix-shell` lines do not need to be at the beginning of the file. +This allows wrapping them in block comments for languages where `#` does not start a comment, such as ECMAScript, Erlang, PHP, or Ruby. diff --git a/doc/manual/src/command-ref/nix-store/add-fixed.md b/doc/manual/src/command-ref/nix-store/add-fixed.md index d25db091c..bebf15026 100644 --- a/doc/manual/src/command-ref/nix-store/add-fixed.md +++ b/doc/manual/src/command-ref/nix-store/add-fixed.md @@ -16,9 +16,10 @@ public url or broke since the download expression was written. This operation has the following options: - - `--recursive`\ - Use recursive instead of flat hashing mode, used when adding - directories to the store. +- `--recursive` + + Use recursive instead of flat hashing mode, used when adding + directories to the store. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/dump.md b/doc/manual/src/command-ref/nix-store/dump.md index c2f3c42ef..3de0e27b0 100644 --- a/doc/manual/src/command-ref/nix-store/dump.md +++ b/doc/manual/src/command-ref/nix-store/dump.md @@ -1,6 +1,6 @@ # Name -`nix-store --dump` - write a single path to a Nix Archive +`nix-store --dump` - write a single path to a [Nix Archive] ## Synopsis @@ -8,7 +8,7 @@ ## Description -The operation `--dump` produces a NAR (Nix ARchive) file containing the +The operation `--dump` produces a [Nix archive](@docroot@/glossary.md#gloss-nar) (NAR) file containing the contents of the file system tree rooted at *path*. The archive is written to standard output. @@ -30,8 +30,9 @@ NAR archives support filenames of unlimited length and 64-bit file sizes. They can contain regular files, directories, and symbolic links, but not other types of files (such as device nodes). -A Nix archive can be unpacked using `nix-store ---restore`. +A Nix archive can be unpacked using [`nix-store --restore`](@docroot@/command-ref/nix-store/restore.md). + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/export.md b/doc/manual/src/command-ref/nix-store/export.md index 1bc46f53b..ba772eb43 100644 --- a/doc/manual/src/command-ref/nix-store/export.md +++ b/doc/manual/src/command-ref/nix-store/export.md @@ -1,6 +1,6 @@ # Name -`nix-store --export` - export store paths to a Nix Archive +`nix-store --export` - export store paths to a [Nix Archive] ## Synopsis @@ -8,16 +8,22 @@ ## Description -The operation `--export` writes a serialisation of the specified store -paths to standard output in a format that can be imported into another -Nix store with `nix-store --import`. This is like `nix-store ---dump`, except that the NAR archive produced by that command doesn’t -contain the necessary meta-information to allow it to be imported into -another Nix store (namely, the set of references of the path). +The operation `--export` writes a serialisation of the given [store objects](@docroot@/glossary.md#gloss-store-object) to standard output in a format that can be imported into another [Nix store](@docroot@/store/index.md) with [`nix-store --import`](./import.md). -This command does not produce a *closure* of the specified paths, so if -a store path references other store paths that are missing in the target -Nix store, the import will fail. +> **Warning** +> +> This command *does not* produce a [closure](@docroot@/glossary.md#gloss-closure) of the specified store paths. +> Trying to import a store object that refers to store paths not available in the target Nix store will fail. +> +> Use [`nix-store --query`](@docroot@/command-ref/nix-store/query.md) to obtain the closure of a store path. + +This command is different from [`nix-store --dump`](./dump.md), which produces a [Nix archive](@docroot@/glossary.md#gloss-nar) that *does not* contain the set of [references](@docroot@/glossary.md#gloss-reference) of a given store path. + +> **Note** +> +> For efficient transfer of closures to remote machines over SSH, use [`nix-copy-closure`](@docroot@/command-ref/nix-copy-closure.md). + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive {{#include ./opt-common.md}} @@ -27,15 +33,21 @@ Nix store, the import will fail. # Examples -To copy a whole closure, do something -like: - -```console -$ nix-store --export $(nix-store --query --requisites paths) > out -``` - -To import the whole closure again, run: - -```console -$ nix-store --import < out -``` +> **Example** +> +> Deploy GNU Hello to an airgapped machine via USB stick. +> +> Write the closure to the block device on a machine with internet connection: +> +> ```shell-session +> [alice@itchy]$ storePath=$(nix-build '' -I nixpkgs=channel:nixpkgs-unstable -A hello --no-out-link) +> [alice@itchy]$ nix-store --export $(nix-store --query --requisites $storePath) | sudo dd of=/dev/usb +> ``` +> +> Read the closure from the block device on the machine without internet connection: +> +> ```shell-session +> [bob@scratchy]$ hello=$(sudo dd if=/dev/usb | nix-store --import | tail -1) +> [bob@scratchy]$ $hello/bin/hello +> Hello, world! +> ``` diff --git a/doc/manual/src/command-ref/nix-store/gc.md b/doc/manual/src/command-ref/nix-store/gc.md index 7be0d559a..f432e00eb 100644 --- a/doc/manual/src/command-ref/nix-store/gc.md +++ b/doc/manual/src/command-ref/nix-store/gc.md @@ -14,30 +14,34 @@ reachable via file system references from a set of “rootsâ€, are deleted. The following suboperations may be specified: - - `--print-roots`\ - This operation prints on standard output the set of roots used by - the garbage collector. +- `--print-roots` - - `--print-live`\ - This operation prints on standard output the set of “live†store - paths, which are all the store paths reachable from the roots. Live - paths should never be deleted, since that would break consistency — - it would become possible that applications are installed that - reference things that are no longer present in the store. + This operation prints on standard output the set of roots used by + the garbage collector. - - `--print-dead`\ - This operation prints out on standard output the set of “dead†store - paths, which is just the opposite of the set of live paths: any path - in the store that is not live (with respect to the roots) is dead. +- `--print-live` + + This operation prints on standard output the set of “live†store + paths, which are all the store paths reachable from the roots. Live + paths should never be deleted, since that would break consistency — + it would become possible that applications are installed that + reference things that are no longer present in the store. + +- `--print-dead` + + This operation prints out on standard output the set of “dead†store + paths, which is just the opposite of the set of live paths: any path + in the store that is not live (with respect to the roots) is dead. By default, all unreachable paths are deleted. The following options control what gets deleted and in what order: - - `--max-freed` *bytes*\ - Keep deleting paths until at least *bytes* bytes have been deleted, - then stop. The argument *bytes* can be followed by the - multiplicative suffix `K`, `M`, `G` or `T`, denoting KiB, MiB, GiB - or TiB units. +- `--max-freed` *bytes* + + Keep deleting paths until at least *bytes* bytes have been deleted, + then stop. The argument *bytes* can be followed by the + multiplicative suffix `K`, `M`, `G` or `T`, denoting KiB, MiB, GiB + or TiB units. The behaviour of the collector is also influenced by the `keep-outputs` and `keep-derivations` settings in the Nix diff --git a/doc/manual/src/command-ref/nix-store/import.md b/doc/manual/src/command-ref/nix-store/import.md index 2711316a7..3f6b3d076 100644 --- a/doc/manual/src/command-ref/nix-store/import.md +++ b/doc/manual/src/command-ref/nix-store/import.md @@ -1,6 +1,8 @@ # Name -`nix-store --import` - import Nix Archive into the store +`nix-store --import` - import [Nix Archive] into the store + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive # Synopsis @@ -8,14 +10,34 @@ # Description -The operation `--import` reads a serialisation of a set of store paths -produced by `nix-store --export` from standard input and adds those -store paths to the Nix store. Paths that already exist in the Nix store -are ignored. If a path refers to another path that doesn’t exist in the -Nix store, the import fails. +The operation `--import` reads a serialisation of a set of [store objects](@docroot@/glossary.md#gloss-store-object) produced by [`nix-store --export`](./export.md) from standard input, and adds those store objects to the specified [Nix store](@docroot@/store/index.md). +Paths that already exist in the target Nix store are ignored. +If a path [refers](@docroot@/glossary.md#gloss-reference) to another path that doesn’t exist in the target Nix store, the import fails. + +> **Note** +> +> For efficient transfer of closures to remote machines over SSH, use [`nix-copy-closure`](@docroot@/command-ref/nix-copy-closure.md). {{#include ./opt-common.md}} {{#include ../opt-common.md}} {{#include ../env-common.md}} + +# Examples + +> **Example** +> +> Given a closure of GNU Hello as a file: +> +> ```shell-session +> $ storePath="$(nix-build '' -I nixpkgs=channel:nixpkgs-unstable -A hello --no-out-link)" +> $ nix-store --export $(nix-store --query --requisites $storePath) > hello.closure +> ``` +> +> Import the closure into a [remote SSH store](@docroot@/store/types/ssh-store.md) using the [`--store`](@docroot@/command-ref/conf-file.md#conf-store) option: +> +> ```console +> $ nix-store --import --store ssh://alice@itchy.example.org < hello.closure +> ``` + diff --git a/doc/manual/src/command-ref/nix-store/optimise.md b/doc/manual/src/command-ref/nix-store/optimise.md index dc392aeb8..b257466b2 100644 --- a/doc/manual/src/command-ref/nix-store/optimise.md +++ b/doc/manual/src/command-ref/nix-store/optimise.md @@ -12,7 +12,7 @@ The operation `--optimise` reduces Nix store disk space usage by finding identical files in the store and hard-linking them to each other. It typically reduces the size of the store by something like 25-35%. Only regular files and symlinks are hard-linked in this manner. Files are -considered identical when they have the same NAR archive serialisation: +considered identical when they have the same [Nix Archive (NAR)][Nix Archive] serialisation: that is, regular files must have the same contents and permission (executable or non-executable), and symlinks must have the same contents. @@ -38,3 +38,4 @@ hashing files in `/nix/store/qhqx7l2f1kmwihc9bnxs7rc159hsxnf3-gcc-4.1.1' there are 114486 files with equal contents out of 215894 files in total ``` +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive diff --git a/doc/manual/src/command-ref/nix-store/query.md b/doc/manual/src/command-ref/nix-store/query.md index a158c76aa..b4efa734e 100644 --- a/doc/manual/src/command-ref/nix-store/query.md +++ b/doc/manual/src/command-ref/nix-store/query.md @@ -24,122 +24,138 @@ symlink. # Common query options - - `--use-output`; `-u`\ - For each argument to the query that is a [store derivation], apply the - query to the output path of the derivation instead. +- `--use-output` / `-u` - - `--force-realise`; `-f`\ - Realise each argument to the query first (see [`nix-store --realise`](./realise.md)). + For each argument to the query that is a [store derivation], apply the + query to the output path of the derivation instead. + +- `--force-realise` / `-f` + + Realise each argument to the query first (see [`nix-store --realise`](./realise.md)). [store derivation]: @docroot@/glossary.md#gloss-store-derivation # Queries - - `--outputs`\ - Prints out the [output paths] of the store - derivations *paths*. These are the paths that will be produced when - the derivation is built. +- `--outputs` - [output paths]: ../../glossary.md#gloss-output-path + Prints out the [output paths] of the store + derivations *paths*. These are the paths that will be produced when + the derivation is built. - - `--requisites`; `-R`\ - Prints out the [closure] of the store path *paths*. + [output paths]: @docroot@/glossary.md#gloss-output-path - [closure]: ../../glossary.md#gloss-closure +- `--requisites` / `-R` - This query has one option: + Prints out the [closure] of the store path *paths*. - - `--include-outputs` - Also include the existing output paths of [store derivation]s, - and their closures. + [closure]: @docroot@/glossary.md#gloss-closure - This query can be used to implement various kinds of deployment. A - *source deployment* is obtained by distributing the closure of a - store derivation. A *binary deployment* is obtained by distributing - the closure of an output path. A *cache deployment* (combined - source/binary deployment, including binaries of build-time-only - dependencies) is obtained by distributing the closure of a store - derivation and specifying the option `--include-outputs`. + This query has one option: - - `--references`\ - Prints the set of [references] of the store paths - *paths*, that is, their immediate dependencies. (For *all* - dependencies, use `--requisites`.) + - `--include-outputs` + Also include the existing output paths of [store derivation]s, + and their closures. - [references]: ../../glossary.md#gloss-reference + This query can be used to implement various kinds of deployment. A + *source deployment* is obtained by distributing the closure of a + store derivation. A *binary deployment* is obtained by distributing + the closure of an output path. A *cache deployment* (combined + source/binary deployment, including binaries of build-time-only + dependencies) is obtained by distributing the closure of a store + derivation and specifying the option `--include-outputs`. - - `--referrers`\ - Prints the set of *referrers* of the store paths *paths*, that is, - the store paths currently existing in the Nix store that refer to - one of *paths*. Note that contrary to the references, the set of - referrers is not constant; it can change as store paths are added or - removed. +- `--references` - - `--referrers-closure`\ - Prints the closure of the set of store paths *paths* under the - referrers relation; that is, all store paths that directly or - indirectly refer to one of *paths*. These are all the path currently - in the Nix store that are dependent on *paths*. + Prints the set of [references] of the store paths + *paths*, that is, their immediate dependencies. (For *all* + dependencies, use `--requisites`.) - - `--deriver`; `-d`\ - Prints the [deriver] that was used to build the store paths *paths*. If - the path has no deriver (e.g., if it is a source file), or if the - deriver is not known (e.g., in the case of a binary-only - deployment), the string `unknown-deriver` is printed. - The returned deriver is not guaranteed to exist in the local store, for - example when *paths* were substituted from a binary cache. - Use `--valid-derivers` instead to obtain valid paths only. + [references]: @docroot@/glossary.md#gloss-reference - [deriver]: ../../glossary.md#gloss-deriver +- `--referrers` - - `--valid-derivers`\ - Prints a set of derivation files (`.drv`) which are supposed produce - said paths when realized. Might print nothing, for example for source paths - or paths subsituted from a binary cache. + Prints the set of *referrers* of the store paths *paths*, that is, + the store paths currently existing in the Nix store that refer to + one of *paths*. Note that contrary to the references, the set of + referrers is not constant; it can change as store paths are added or + removed. - - `--graph`\ - Prints the references graph of the store paths *paths* in the format - of the `dot` tool of AT\&T's [Graphviz - package](http://www.graphviz.org/). This can be used to visualise - dependency graphs. To obtain a build-time dependency graph, apply - this to a store derivation. To obtain a runtime dependency graph, - apply it to an output path. +- `--referrers-closure` - - `--tree`\ - Prints the references graph of the store paths *paths* as a nested - ASCII tree. References are ordered by descending closure size; this - tends to flatten the tree, making it more readable. The query only - recurses into a store path when it is first encountered; this - prevents a blowup of the tree representation of the graph. + Prints the closure of the set of store paths *paths* under the + referrers relation; that is, all store paths that directly or + indirectly refer to one of *paths*. These are all the path currently + in the Nix store that are dependent on *paths*. - - `--graphml`\ - Prints the references graph of the store paths *paths* in the - [GraphML](http://graphml.graphdrawing.org/) file format. This can be - used to visualise dependency graphs. To obtain a build-time - dependency graph, apply this to a [store derivation]. To obtain a - runtime dependency graph, apply it to an output path. +- `--deriver` / `-d` - - `--binding` *name*; `-b` *name*\ - Prints the value of the attribute *name* (i.e., environment - variable) of the [store derivation]s *paths*. It is an error for a - derivation to not have the specified attribute. + Prints the [deriver] that was used to build the store paths *paths*. If + the path has no deriver (e.g., if it is a source file), or if the + deriver is not known (e.g., in the case of a binary-only + deployment), the string `unknown-deriver` is printed. + The returned deriver is not guaranteed to exist in the local store, for + example when *paths* were substituted from a binary cache. + Use `--valid-derivers` instead to obtain valid paths only. - - `--hash`\ - Prints the SHA-256 hash of the contents of the store paths *paths* - (that is, the hash of the output of `nix-store --dump` on the given - paths). Since the hash is stored in the Nix database, this is a fast - operation. + [deriver]: @docroot@/glossary.md#gloss-deriver - - `--size`\ - Prints the size in bytes of the contents of the store paths *paths* - — to be precise, the size of the output of `nix-store --dump` on - the given paths. Note that the actual disk space required by the - store paths may be higher, especially on filesystems with large - cluster sizes. +- `--valid-derivers` - - `--roots`\ - Prints the garbage collector roots that point, directly or - indirectly, at the store paths *paths*. + Prints a set of derivation files (`.drv`) which are supposed produce + said paths when realized. Might print nothing, for example for source paths + or paths subsituted from a binary cache. + +- `--graph` + + Prints the references graph of the store paths *paths* in the format + of the `dot` tool of AT\&T's [Graphviz + package](http://www.graphviz.org/). This can be used to visualise + dependency graphs. To obtain a build-time dependency graph, apply + this to a store derivation. To obtain a runtime dependency graph, + apply it to an output path. + +- `--tree` + + Prints the references graph of the store paths *paths* as a nested + ASCII tree. References are ordered by descending closure size; this + tends to flatten the tree, making it more readable. The query only + recurses into a store path when it is first encountered; this + prevents a blowup of the tree representation of the graph. + +- `--graphml` + + Prints the references graph of the store paths *paths* in the + [GraphML](http://graphml.graphdrawing.org/) file format. This can be + used to visualise dependency graphs. To obtain a build-time + dependency graph, apply this to a [store derivation]. To obtain a + runtime dependency graph, apply it to an output path. + +- `--binding` *name* / `-b` *name* + + Prints the value of the attribute *name* (i.e., environment + variable) of the [store derivation]s *paths*. It is an error for a + derivation to not have the specified attribute. + +- `--hash` + + Prints the SHA-256 hash of the contents of the store paths *paths* + (that is, the hash of the output of `nix-store --dump` on the given + paths). Since the hash is stored in the Nix database, this is a fast + operation. + +- `--size` + + Prints the size in bytes of the contents of the store paths *paths* + — to be precise, the size of the output of `nix-store --dump` on + the given paths. Note that the actual disk space required by the + store paths may be higher, especially on filesystems with large + cluster sizes. + +- `--roots` + + Prints the garbage collector roots that point, directly or + indirectly, at the store paths *paths*. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index 5428d57fa..a899758df 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -25,14 +25,14 @@ Each of *paths* is processed as follows: If no substitutes are available and no store derivation is given, realisation fails. -[store paths]: @docroot@/glossary.md#gloss-store-path +[store paths]: @docroot@/store/store-path.md [valid]: @docroot@/glossary.md#gloss-validity [store derivation]: @docroot@/glossary.md#gloss-store-derivation [output paths]: @docroot@/glossary.md#gloss-output-path -[store objects]: @docroot@/glossary.md#gloss-store-object +[store objects]: @docroot@/store/store-object.md [closure]: @docroot@/glossary.md#gloss-closure [substituters]: @docroot@/command-ref/conf-file.md#conf-substituters -[content-addressed derivations]: @docroot@/contributing/experimental-features.md#xp-feature-ca-derivations +[content-addressed derivations]: @docroot@/development/experimental-features.md#xp-feature-ca-derivations [Nix database]: @docroot@/glossary.md#gloss-nix-database The resulting paths are printed on standard output. @@ -42,23 +42,26 @@ For non-derivation arguments, the argument itself is printed. # Options - - `--dry-run`\ - Print on standard error a description of what packages would be - built or downloaded, without actually performing the operation. +- `--dry-run` - - `--ignore-unknown`\ - If a non-derivation path does not have a substitute, then silently - ignore it. + Print on standard error a description of what packages would be + built or downloaded, without actually performing the operation. - - `--check`\ - This option allows you to check whether a derivation is - deterministic. It rebuilds the specified derivation and checks - whether the result is bitwise-identical with the existing outputs, - printing an error if that’s not the case. The outputs of the - specified derivation must already exist. When used with `-K`, if an - output path is not identical to the corresponding output from the - previous build, the new output path is left in - `/nix/store/name.check.` +- `--ignore-unknown` + + If a non-derivation path does not have a substitute, then silently + ignore it. + +- `--check` + + This option allows you to check whether a derivation is + deterministic. It rebuilds the specified derivation and checks + whether the result is bitwise-identical with the existing outputs, + printing an error if that’s not the case. The outputs of the + specified derivation must already exist. When used with `-K`, if an + output path is not identical to the corresponding output from the + previous build, the new output path is left in + `/nix/store/name.check.` {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/restore.md b/doc/manual/src/command-ref/nix-store/restore.md index fcba43df4..2d0aa3127 100644 --- a/doc/manual/src/command-ref/nix-store/restore.md +++ b/doc/manual/src/command-ref/nix-store/restore.md @@ -8,9 +8,11 @@ ## Description -The operation `--restore` unpacks a NAR archive to *path*, which must +The operation `--restore` unpacks a [Nix Archive (NAR)][Nix Archive] to *path*, which must not already exist. The archive is read from standard input. +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive + {{#include ./opt-common.md}} {{#include ../opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/serve.md b/doc/manual/src/command-ref/nix-store/serve.md index 0f90f65ae..9a4cf5216 100644 --- a/doc/manual/src/command-ref/nix-store/serve.md +++ b/doc/manual/src/command-ref/nix-store/serve.md @@ -14,10 +14,11 @@ access to a restricted ssh user. The following flags are available: - - `--write`\ - Allow the connected client to request the realization of - derivations. In effect, this can be used to make the host act as a - remote builder. +- `--write` + + Allow the connected client to request the realization of + derivations. In effect, this can be used to make the host act as a + remote builder. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/verify.md b/doc/manual/src/command-ref/nix-store/verify.md index 2695b3361..40c9180db 100644 --- a/doc/manual/src/command-ref/nix-store/verify.md +++ b/doc/manual/src/command-ref/nix-store/verify.md @@ -16,18 +16,20 @@ being modified by non-Nix tools, or of bugs in Nix itself. This operation has the following options: - - `--check-contents`\ - Checks that the contents of every valid store path has not been - altered by computing a SHA-256 hash of the contents and comparing it - with the hash stored in the Nix database at build time. Paths that - have been modified are printed out. For large stores, - `--check-contents` is obviously quite slow. +- `--check-contents` - - `--repair`\ - If any valid path is missing from the store, or (if - `--check-contents` is given) the contents of a valid path has been - modified, then try to repair the path by redownloading it. See - `nix-store --repair-path` for details. + Checks that the contents of every valid store path has not been + altered by computing a SHA-256 hash of the contents and comparing it + with the hash stored in the Nix database at build time. Paths that + have been modified are printed out. For large stores, + `--check-contents` is obviously quite slow. + +- `--repair` + + If any valid path is missing from the store, or (if + `--check-contents` is given) the contents of a valid path has been + modified, then try to repair the path by redownloading it. See + `nix-store --repair-path` for details. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index 114b292f9..69a700207 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -37,7 +37,7 @@ Most Nix commands accept the following command-line options: Print even more informational messages. - `4` “Debug†- + Print debug information. - `5` “Vomit†@@ -143,7 +143,7 @@ Most Nix commands accept the following command-line options: This option is accepted by `nix-env`, `nix-instantiate`, `nix-shell` and `nix-build`. When evaluating Nix expressions, the expression evaluator will automatically try to call functions that it encounters. - It can automatically call functions for which every argument has a [default value](@docroot@/language/constructs.md#functions) (e.g., `{ argName ? defaultValue }: ...`). + It can automatically call functions for which every argument has a [default value](@docroot@/language/syntax.md#functions) (e.g., `{ argName ? defaultValue }: ...`). With `--arg`, you can also call functions that have arguments without a default value (or override a default value). That is, if the evaluator encounters a function with an argument named *name*, it will call it with value *value*. @@ -187,11 +187,12 @@ Most Nix commands accept the following command-line options: For `nix-shell`, this option is commonly used to give you a shell in which you can build the packages returned by the expression. If you want to get a shell which contain the *built* packages ready for use, give your expression to the `nix-shell --packages ` convenience flag instead. -- [`-I`](#opt-I) *path* +- [`-I` / `--include`](#opt-I) *path* - Add an entry to the [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path). + Add an entry to the list of search paths used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). This option may be given multiple times. - Paths added through `-I` take precedence over [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH). + + Paths added through `-I` take precedence over the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path) and the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH). - [`--option`](#opt-option) *name* *value* diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/development/building.md similarity index 67% rename from doc/manual/src/contributing/hacking.md rename to doc/manual/src/development/building.md index 916ec3077..5a5fb3368 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/development/building.md @@ -1,24 +1,67 @@ -# Hacking +# Building Nix -This section provides some notes on how to hack on Nix. To get the -latest version of Nix from GitHub: +This section provides some notes on how to start hacking on Nix. +To get the latest version of Nix from GitHub: ```console $ git clone https://github.com/NixOS/nix.git $ cd nix ``` -The following instructions assume you already have some version of Nix installed locally, so that you can use it to set up the development environment. If you don't have it installed, follow the [installation instructions]. +> **Note** +> +> The following instructions assume you already have some version of Nix installed locally, so that you can use it to set up the development environment. +> If you don't have it installed, follow the [installation instructions](../installation/index.md). -[installation instructions]: ../installation/index.md + +To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: + +```console +$ nix-shell +``` + +To get a shell with one of the other [supported compilation environments](#compilation-environments): + +```console +$ nix-shell --attr devShells.x86_64-linux.native-clangStdenvPackages +``` + +> **Note** +> +> You can use `native-ccacheStdenvPackages` to drastically improve rebuild time. +> By default, [ccache](https://ccache.dev) keeps artifacts in `~/.cache/ccache/`. + +To build Nix itself in this shell: + +```console +[nix-shell]$ autoreconfPhase +[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out +[nix-shell]$ make -j $NIX_BUILD_CORES +``` + +To install it in `$(pwd)/outputs` and test it: + +```console +[nix-shell]$ make install +[nix-shell]$ make installcheck -j $NIX_BUILD_CORES +[nix-shell]$ ./outputs/out/bin/nix --version +nix (Nix) 2.12 +``` + +To build a release version of Nix for the current operating system and CPU architecture: + +```console +$ nix-build +``` + +You can also build Nix for one of the [supported platforms](#platforms). ## Building Nix with flakes This section assumes you are using Nix with the [`flakes`] and [`nix-command`] experimental features enabled. -See the [Building Nix](#building-nix) section for equivalent instructions using stable Nix interfaces. -[`flakes`]: @docroot@/contributing/experimental-features.md#xp-feature-flakes -[`nix-command`]: @docroot@/contributing/experimental-features.md#xp-nix-command +[`flakes`]: @docroot@/development/experimental-features.md#xp-feature-flakes +[`nix-command`]: @docroot@/development/experimental-features.md#xp-nix-command To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: @@ -67,50 +110,6 @@ $ nix build You can also build Nix for one of the [supported platforms](#platforms). -## Building Nix - -To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: - -```console -$ nix-shell -``` - -To get a shell with one of the other [supported compilation environments](#compilation-environments): - -```console -$ nix-shell --attr devShells.x86_64-linux.native-clangStdenvPackages -``` - -> **Note** -> -> You can use `native-ccacheStdenvPackages` to drastically improve rebuild time. -> By default, [ccache](https://ccache.dev) keeps artifacts in `~/.cache/ccache/`. - -To build Nix itself in this shell: - -```console -[nix-shell]$ autoreconfPhase -[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out -[nix-shell]$ make -j $NIX_BUILD_CORES -``` - -To install it in `$(pwd)/outputs` and test it: - -```console -[nix-shell]$ make install -[nix-shell]$ make installcheck -j $NIX_BUILD_CORES -[nix-shell]$ ./outputs/out/bin/nix --version -nix (Nix) 2.12 -``` - -To build a release version of Nix for the current operating system and CPU architecture: - -```console -$ nix-build -``` - -You can also build Nix for one of the [supported platforms](#platforms). - ## Makefile variables You may need `profiledir=$out/etc/profile.d` and `sysconfdir=$out/etc` to run `make install`. @@ -122,7 +121,6 @@ Run `make` with [`-e` / `--environment-overrides`](https://www.gnu.org/software/ The docs can take a while to build, so you may want to disable this for local development. - `ENABLE_FUNCTIONAL_TESTS=yes` to enable building the functional tests. -- `ENABLE_UNIT_TESTS=yes` to enable building the unit tests. - `OPTIMIZE=1` to enable optimizations. - `libraries=libutil programs=` to only build a specific library. @@ -144,6 +142,7 @@ Nix can be built for various platforms, as specified in [`flake.nix`]: - `aarch64-darwin` - `armv6l-linux` - `armv7l-linux` +- `riscv64-linux` In order to build Nix for a different platform than the one you're currently on, you need a way for your current Nix installation to build code for that @@ -166,7 +165,10 @@ or for Nix with the [`flakes`] and [`nix-command`] experimental features enabled $ nix build .#packages.aarch64-linux.default ``` -Cross-compiled builds are available for ARMv6 (`armv6l-linux`) and ARMv7 (`armv7l-linux`). +Cross-compiled builds are available for: +- `armv6l-linux` +- `armv7l-linux` +- `riscv64-linux` Add more [system types](#system-type) to `crossSystems` in `flake.nix` to bootstrap Nix on unsupported platforms. ### Building for multiple platforms at once @@ -196,7 +198,7 @@ In order to facilitate this, Nix has some support for being built out of tree ## System type -Nix uses a string with he following format to identify the *system type* or *platform* it runs on: +Nix uses a string with the following format to identify the *system type* or *platform* it runs on: ``` -[-] @@ -258,10 +260,10 @@ See [supported compilation environments](#compilation-environments) and instruct To use the LSP with your editor, you first need to [set up `clangd`](https://clangd.llvm.org/installation#project-setup) by running: ```console -make clean && bear -- make -j$NIX_BUILD_CORES default check install +make compile_commands.json ``` -Configure your editor to use the `clangd` from the shell, either by running it inside the development shell, or by using [nix-direnv](https://github.com/nix-community/nix-direnv) and [the appropriate editor plugin](https://github.com/direnv/direnv/wiki#editor-integration). +Configure your editor to use the `clangd` from the `.#native-clangStdenvPackages` shell. You can do that either by running it inside the development shell, or by using [nix-direnv](https://github.com/nix-community/nix-direnv) and [the appropriate editor plugin](https://github.com/direnv/direnv/wiki#editor-integration). > **Note** > @@ -269,80 +271,25 @@ Configure your editor to use the `clangd` from the shell, either by running it i > Some other editors (e.g. Emacs, Vim) need a plugin to support LSP servers in general (e.g. [lsp-mode](https://github.com/emacs-lsp/lsp-mode) for Emacs and [vim-lsp](https://github.com/prabirshrestha/vim-lsp) for vim). > Editor-specific setup is typically opinionated, so we will not cover it here in more detail. -## Add a release note +## Formatting and pre-commit hooks -`doc/manual/rl-next` contains release notes entries for all unreleased changes. +You may run the formatters as a one-off using: -User-visible changes should come with a release note. - -### Add an entry - -Here's what a complete entry looks like. The file name is not incorporated in the document. - -``` ---- -synopsis: Basically a title -issues: 1234 -prs: 1238 ---- - -Here's one or more paragraphs that describe the change. - -- It's markdown -- Add references to the manual using @docroot@ +```console +make format ``` -Significant changes should add the following header, which moves them to the top. +If you'd like to run the formatters before every commit, install the hooks: ``` -significance: significant +pre-commit-hooks-install ``` - -See also the [format documentation](https://github.com/haskell/cabal/blob/master/CONTRIBUTING.md#changelog). +This installs [pre-commit](https://pre-commit.com) using [cachix/git-hooks.nix](https://github.com/cachix/git-hooks.nix). -### Build process +When making a commit, pay attention to the console output. +If it fails, run `git add --patch` to approve the suggestions _and commit again_. -Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`. - -## Branches - -- [`master`](https://github.com/NixOS/nix/commits/master) - - The main development branch. All changes are approved and merged here. - When developing a change, create a branch based on the latest `master`. - - Maintainers try to [keep it in a release-worthy state](#reverting). - -- [`maintenance-*.*`](https://github.com/NixOS/nix/branches/all?query=maintenance) - - These branches are the subject of backports only, and are - also [kept](#reverting) in a release-worthy state. - - See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) - -- [`latest-release`](https://github.com/NixOS/nix/tree/latest-release) - - The latest patch release of the latest minor version. - - See [`maintainers/release-process.md`](https://github.com/NixOS/nix/blob/master/maintainers/release-process.md) - -- [`backport-*-to-*`](https://github.com/NixOS/nix/branches/all?query=backport) - - Generally branches created by the backport action. - - See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) - -- [_other_](https://github.com/NixOS/nix/branches/all) - - Branches that do not conform to the above patterns should be feature branches. - -## Reverting - -If a change turns out to be merged by mistake, or contain a regression, it may be reverted. -A revert is not a rejection of the contribution, but merely part of an effective development process. -It makes sure that development keeps running smoothly, with minimal uncertainty, and less overhead. -If maintainers have to worry too much about avoiding reverts, they would not be able to merge as much. -By embracing reverts as a good part of the development process, everyone wins. - -However, taking a step back may be frustrating, so maintainers will be extra supportive on the next try. +To refresh pre-commit hook's config file, do the following: +1. Exit the development shell and start it again by running `nix develop`. +2. If you also use the pre-commit hook, also run `pre-commit-hooks-install` again. diff --git a/doc/manual/src/contributing/cli-guideline.md b/doc/manual/src/development/cli-guideline.md similarity index 88% rename from doc/manual/src/contributing/cli-guideline.md rename to doc/manual/src/development/cli-guideline.md index e90d6de8d..23df844ec 100644 --- a/doc/manual/src/contributing/cli-guideline.md +++ b/doc/manual/src/development/cli-guideline.md @@ -389,88 +389,6 @@ colors, no emojis and using ASCII instead of Unicode symbols). The same should happen when TTY is not detected on STDERR. We should not display progress / status section, but only print warnings and errors. -## Returning future proof JSON - -The schema of JSON output should allow for backwards compatible extension. This section explains how to achieve this. - -Two definitions are helpful here, because while JSON only defines one "key-value" -object type, we use it to cover two use cases: - - - **dictionary**: a map from names to value that all have the same type. In - C++ this would be a `std::map` with string keys. - - **record**: a fixed set of attributes each with their own type. In C++, this - would be represented by a `struct`. - -It is best not to mix these use cases, as that may lead to incompatibilities when the schema changes. For example, adding a record field to a dictionary breaks consumers that assume all JSON object fields to have the same meaning and type. - -This leads to the following guidelines: - - - The top-level (root) value must be a record. - - Otherwise, one can not change the structure of a command's output. - - - The value of a dictionary item must be a record. - - Otherwise, the item type can not be extended. - - - List items should be records. - - Otherwise, one can not change the structure of the list items. - - If the order of the items does not matter, and each item has a unique key that is a string, consider representing the list as a dictionary instead. If the order of the items needs to be preserved, return a list of records. - - - Streaming JSON should return records. - - An example of a streaming JSON format is [JSON lines](https://jsonlines.org/), where each line represents a JSON value. These JSON values can be considered top-level values or list items, and they must be records. - -### Examples - - -This is bad, because all keys must be assumed to be store types: - -```json -{ - "local": { ... }, - "remote": { ... }, - "http": { ... } -} -``` - -This is good, because the it is extensible at the root, and is somewhat self-documenting: - -```json -{ - "storeTypes": { "local": { ... }, ... }, - "pluginSupport": true -} -``` - -While the dictionary of store types seems like a very complete response at first, a use case may arise that warrants returning additional information. -For example, the presence of plugin support may be crucial information for a client to proceed when their desired store type is missing. - - - -The following representation is bad because it is not extensible: - -```json -{ "outputs": [ "out" "bin" ] } -``` - -However, simply converting everything to records is not enough, because the order of outputs must be preserved: - -```json -{ "outputs": { "bin": {}, "out": {} } } -``` - -The first item is the default output. Deriving this information from the outputs ordering is not great, but this is how Nix currently happens to work. -While it is possible for a JSON parser to preserve the order of fields, we can not rely on this capability to be present in all JSON libraries. - -This representation is extensible and preserves the ordering: - -```json -{ "outputs": [ { "outputName": "out" }, { "outputName": "bin" } ] } -``` - ## Dialog with the user CLIs don't always make it clear when an action has taken place. For every diff --git a/doc/manual/src/development/contributing.md b/doc/manual/src/development/contributing.md new file mode 100644 index 000000000..7de7489dc --- /dev/null +++ b/doc/manual/src/development/contributing.md @@ -0,0 +1,79 @@ +# Contributing + +## Add a release note + +`doc/manual/rl-next` contains release notes entries for all unreleased changes. + +User-visible changes should come with a release note. + +### Add an entry + +Here's what a complete entry looks like. The file name is not incorporated in the document. + +``` +--- +synopsis: Basically a title +issues: 1234 +prs: 1238 +--- + +Here's one or more paragraphs that describe the change. + +- It's markdown +- Add references to the manual using @docroot@ +``` + +Significant changes should add the following header, which moves them to the top. + +``` +significance: significant +``` + + +See also the [format documentation](https://github.com/haskell/cabal/blob/master/CONTRIBUTING.md#changelog). + +### Build process + +Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`. + +## Branches + +- [`master`](https://github.com/NixOS/nix/commits/master) + + The main development branch. All changes are approved and merged here. + When developing a change, create a branch based on the latest `master`. + + Maintainers try to [keep it in a release-worthy state](#reverting). + +- [`maintenance-*.*`](https://github.com/NixOS/nix/branches/all?query=maintenance) + + These branches are the subject of backports only, and are + also [kept](#reverting) in a release-worthy state. + + See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) + +- [`latest-release`](https://github.com/NixOS/nix/tree/latest-release) + + The latest patch release of the latest minor version. + + See [`maintainers/release-process.md`](https://github.com/NixOS/nix/blob/master/maintainers/release-process.md) + +- [`backport-*-to-*`](https://github.com/NixOS/nix/branches/all?query=backport) + + Generally branches created by the backport action. + + See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) + +- [_other_](https://github.com/NixOS/nix/branches/all) + + Branches that do not conform to the above patterns should be feature branches. + +## Reverting + +If a change turns out to be merged by mistake, or contain a regression, it may be reverted. +A revert is not a rejection of the contribution, but merely part of an effective development process. +It makes sure that development keeps running smoothly, with minimal uncertainty, and less overhead. +If maintainers have to worry too much about avoiding reverts, they would not be able to merge as much. +By embracing reverts as a good part of the development process, everyone wins. + +However, taking a step back may be frustrating, so maintainers will be extra supportive on the next try. diff --git a/doc/manual/src/contributing/cxx.md b/doc/manual/src/development/cxx.md similarity index 100% rename from doc/manual/src/contributing/cxx.md rename to doc/manual/src/development/cxx.md diff --git a/doc/manual/src/contributing/documentation.md b/doc/manual/src/development/documentation.md similarity index 89% rename from doc/manual/src/contributing/documentation.md rename to doc/manual/src/development/documentation.md index 1dddb207c..63f574ab7 100644 --- a/doc/manual/src/contributing/documentation.md +++ b/doc/manual/src/development/documentation.md @@ -24,14 +24,12 @@ nix build .#^doc and open `./result-doc/share/doc/nix/manual/index.html`. -To build the manual incrementally, [enter the development shell](./hacking.md) and run: +To build the manual incrementally, [enter the development shell](./building.md) and run: ```console -make manual-html -j $NIX_BUILD_CORES +make manual-html-open -j $NIX_BUILD_CORES ``` -and open `./outputs/out/share/doc/nix/manual/language/index.html`. - In order to reflect changes to the [Makefile for the manual], clear all generated files before re-building: [Makefile for the manual]: https://github.com/NixOS/nix/blob/master/doc/manual/local.mk @@ -149,7 +147,7 @@ Please observe these guidelines to ease reviews: ``` A [store object] contains a [file system object] and [references] to other store objects. - [store object]: @docroot@/glossary.md#gloss-store-object + [store object]: @docroot@/store/store-object.md [file system object]: @docroot@/architecture/file-system-object.md [references]: @docroot@/glossary.md#gloss-reference ``` @@ -198,13 +196,35 @@ You can also build and view it yourself: [Doxygen API documentation]: https://hydra.nixos.org/job/nix/master/internal-api-docs/latest/download-by-type/doc/internal-api-docs ```console -# nix build .#hydraJobs.internal-api-docs -# xdg-open ./result/share/doc/nix/internal-api/html/index.html +$ nix build .#hydraJobs.internal-api-docs +$ xdg-open ./result/share/doc/nix/internal-api/html/index.html +``` + +or inside `nix-shell` or `nix develop`: + +```console +$ mesonConfigurePhase +$ ninja src/internal-api-docs/html +$ xdg-open src/internal-api-docs/html/index.html +``` + +## C API documentation + +Note that the C API is not yet stable. +[C API documentation] is available online. +You can also build and view it yourself: + +[C API documentation]: https://hydra.nixos.org/job/nix/master/external-api-docs/latest/download-by-type/doc/external-api-docs + +```console +$ nix build .#hydraJobs.external-api-docs +$ xdg-open ./result/share/doc/nix/external-api/html/index.html ``` or inside `nix-shell` or `nix develop`: ``` -# make internal-api-html -# xdg-open ./outputs/doc/share/doc/nix/internal-api/html/index.html +$ mesonConfigurePhase +$ ninja src/external-api-docs/html +$ xdg-open src/external-api-docs/html/index.html ``` diff --git a/doc/manual/src/contributing/experimental-features.md b/doc/manual/src/development/experimental-features.md similarity index 100% rename from doc/manual/src/contributing/experimental-features.md rename to doc/manual/src/development/experimental-features.md diff --git a/doc/manual/src/contributing/index.md b/doc/manual/src/development/index.md similarity index 77% rename from doc/manual/src/contributing/index.md rename to doc/manual/src/development/index.md index 4d55c17a4..6403c3e66 100644 --- a/doc/manual/src/contributing/index.md +++ b/doc/manual/src/development/index.md @@ -5,4 +5,4 @@ Check the [contributing guide](https://github.com/NixOS/nix/blob/master/CONTRIBU This chapter is a collection of guides for making changes to the code and documentation. -If you're not sure where to start, try to [compile Nix from source](./hacking.md) and consider [making improvements to documentation](./documentation.md). +If you're not sure where to start, try to [compile Nix from source](./building.md) and consider [making improvements to documentation](./documentation.md). diff --git a/doc/manual/src/development/json-guideline.md b/doc/manual/src/development/json-guideline.md new file mode 100644 index 000000000..b4bc92af9 --- /dev/null +++ b/doc/manual/src/development/json-guideline.md @@ -0,0 +1,128 @@ +# JSON guideline + +Nix consumes and produces JSON in a variety of contexts. +These guidelines ensure consistent practices for all our JSON interfaces, for ease of use, and so that experience in one part carries over to another. + +## Extensibility + +The schema of JSON input and output should allow for backwards compatible extension. +This section explains how to achieve this. + +Two definitions are helpful here, because while JSON only defines one "key-value" object type, we use it to cover two use cases: + + - **dictionary**: a map from names to value that all have the same type. + In C++ this would be a `std::map` with string keys. + + - **record**: a fixed set of attributes each with their own type. + In C++, this would be represented by a `struct`. + +It is best not to mix these use cases, as that may lead to incompatibilities when the schema changes. +For example, adding a record field to a dictionary breaks consumers that assume all JSON object fields to have the same meaning and type, and dictionary items with a colliding name can not be represented anymore. + +This leads to the following guidelines: + + - The top-level (root) value must be a record. + + Otherwise, one can not change the structure of a command's output. + + - The value of a dictionary item must be a record. + + Otherwise, the item type can not be extended. + + - List items should be records. + + Otherwise, one can not change the structure of the list items. + + If the order of the items does not matter, and each item has a unique key that is a string, consider representing the list as a dictionary instead. + If the order of the items needs to be preserved, return a list of records. + + - Streaming JSON should return records. + + An example of a streaming JSON format is [JSON lines](https://jsonlines.org/), where each line represents a JSON value. + These JSON values can be considered top-level values or list items, and they must be records. + +### Examples + +This is bad, because all keys must be assumed to be store types: + +```json +{ + "local": { ... }, + "remote": { ... }, + "http": { ... } +} +``` + +This is good, because the it is extensible at the root, and is somewhat self-documenting: + +```json +{ + "storeTypes": { "local": { ... }, ... }, + "pluginSupport": true +} +``` + +While the dictionary of store types seems like a very complete response at first, a use case may arise that warrants returning additional information. +For example, the presence of plugin support may be crucial information for a client to proceed when their desired store type is missing. + + + +The following representation is bad because it is not extensible: + +```json +{ "outputs": [ "out" "bin" ] } +``` + +However, simply converting everything to records is not enough, because the order of outputs must be preserved: + +```json +{ "outputs": { "bin": {}, "out": {} } } +``` + +The first item is the default output. Deriving this information from the outputs ordering is not great, but this is how Nix currently happens to work. +While it is possible for a JSON parser to preserve the order of fields, we can not rely on this capability to be present in all JSON libraries. + +This representation is extensible and preserves the ordering: + +```json +{ "outputs": [ { "outputName": "out" }, { "outputName": "bin" } ] } +``` + +## Self-describing values + +As described in the previous section, it's crucial that schemas can be extended with with new fields without breaking compatibility. +However, that should *not* mean we use the presence/absence of fields to indicate optional information *within* a version of the schema. +Instead, always include the field, and use `null` to indicate the "nothing" case. + +### Examples + +Here are two JSON objects: + +```json +{ + "foo": {} +} +``` +```json +{ + "foo": {}, + "bar": {} +} +``` + +Since they differ in which fields they contain, they should *not* both be valid values of the same schema. +At most, they can match two different schemas where the second (with `foo` and `bar`) is considered a newer version of the first (with just `foo`). +Within each version, all fields are mandatory (always `foo`, and always `foo` and `bar`). +Only *between* each version, `bar` gets added as a new mandatory field. + +Here are another two JSON objects: + +```json +{ "foo": null } +``` +```json +{ "foo": { "bar": 1 } } +``` + +Since they both contain a `foo` field, they could be valid values of the same schema. +The schema would have `foo` has an optional field, which is either `null` or an object where `bar` is an integer. diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/development/testing.md similarity index 81% rename from doc/manual/src/contributing/testing.md rename to doc/manual/src/development/testing.md index 31c39c16c..3949164d5 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/development/testing.md @@ -59,15 +59,15 @@ The unit tests are defined using the [googletest] and [rapidcheck] frameworks. > … > ``` -The tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `tests/unit/${library_name_without-nix}`. -Given a interface (header) and implementation pair in the original library, say, `src/libexpr/value/context.{hh,cc}`, we write tests for it in `tests/unit/libexpr/tests/value/context.cc`, and (possibly) declare/define additional interfaces for testing purposes in `tests/unit/libexpr-support/tests/value/context.{hh,cc}`. +The tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `src/${library_name_without-nix}-test`. +Given an interface (header) and implementation pair in the original library, say, `src/libexpr/value/context.{hh,cc}`, we write tests for it in `src/nix-expr-tests/value/context.cc`, and (possibly) declare/define additional interfaces for testing purposes in `src/nix-expr-test-support/tests/value/context.{hh,cc}`. Data for unit tests is stored in a `data` subdir of the directory for each unit test executable. -For example, `libnixstore` code is in `src/libstore`, and its test data is in `tests/unit/libstore/data`. -The path to the `tests/unit/data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`. +For example, `libnixstore` code is in `src/libstore`, and its test data is in `src/nix-store-tests/data`. +The path to the `src/${library_name_without-nix}-test/data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`. Note that each executable only gets the data for its tests. -The unit test libraries are in `tests/unit/${library_name_without-nix}-lib`. +The unit test libraries are in `src/${library_name_without-nix}-test-support`. All headers are in a `tests` subdirectory so they are included with `#include "tests/"`. The use of all these separate directories for the unit tests might seem inconvenient, as for example the tests are not "right next to" the part of the code they are testing. @@ -76,8 +76,25 @@ there is no risk of any build-system wildcards for the library accidentally pick ### Running tests -You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. -Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option, or the `GTEST_FILTER` environment variable, e.g. `GTEST_FILTER='ErrorTraceTest.*' make check`. +You can run the whole testsuite with `meson test` from the Meson build directory, or the tests for a specific component with `meson test nix-store-tests`. +A environment variables that Google Test accepts are also worth knowing: + +1. [`GTEST_FILTER`](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) + + This is used for finer-grained filtering of which tests to run. + + +2. [`GTEST_BRIEF`](https://google.github.io/googletest/advanced.html#suppressing-test-passes) + + This is used to avoid logging passing tests. + +Putting the two together, one might run + +```bash +GTEST_BRIEF=1 GTEST_FILTER='ErrorTraceTest.*' meson test nix-expr-tests -v +``` + +for short but comprensive output. ### Characterisation testing { #characaterisation-testing-unit } @@ -86,7 +103,7 @@ See [functional characterisation testing](#characterisation-testing-functional) Like with the functional characterisation, `_NIX_TEST_ACCEPT=1` is also used. For example: ```shell-session -$ _NIX_TEST_ACCEPT=1 make libstore-tests_RUN +$ _NIX_TEST_ACCEPT=1 meson test nix-store-tests -v ... [ SKIPPED ] WorkerProtoTest.string_read [ SKIPPED ] WorkerProtoTest.string_write @@ -114,6 +131,8 @@ On other platforms they wouldn't be run at all. The functional tests reside under the `tests/functional` directory and are listed in `tests/functional/local.mk`. Each test is a bash script. +Functional tests are run during `installCheck` in the `nix` package build, as well as separately from the build, in VM tests. + ### Running the whole test suite The whole test suite can be run with: @@ -162,14 +181,14 @@ ran test tests/functional/${testName}.sh... [PASS] or without `make`: ```shell-session -$ ./mk/run-test.sh tests/functional/${testName}.sh tests/functional/init.sh +$ ./mk/run-test.sh tests/functional/${testName}.sh ran test tests/functional/${testName}.sh... [PASS] ``` To see the complete output, one can also run: ```shell-session -$ ./mk/debug-test.sh tests/functional/${testName}.sh tests/functional/init.sh +$ ./mk/debug-test.sh tests/functional/${testName}.sh +(${testName}.sh:1) foo output from foo +(${testName}.sh:2) bar @@ -204,7 +223,7 @@ edit it like so: Then, running the test with `./mk/debug-test.sh` will drop you into GDB once the script reaches that point: ```shell-session -$ ./mk/debug-test.sh tests/functional/${testName}.sh tests/functional/init.sh +$ ./mk/debug-test.sh tests/functional/${testName}.sh ... + gdb blash blub GNU gdb (GDB) 12.1 @@ -252,13 +271,30 @@ Regressions are caught, and improvements always show up in code review. To ensure that characterisation testing doesn't make it harder to intentionally change these interfaces, there always must be an easy way to regenerate the expected output, as we do with `_NIX_TEST_ACCEPT=1`. +### Running functional tests on NixOS + +We run the functional tests not just in the build, but also in VM tests. +This helps us ensure that Nix works correctly on NixOS, and environments that have similar characteristics that are hard to reproduce in a build environment. + +The recommended way to run these tests during development is: + +```shell +nix build .#hydraJobs.tests.functional_user.quickBuild +``` + +The `quickBuild` attribute configures the test to use a `nix` package that's built without integration tests, so that you can iterate on the tests without performing recompilations due to the changed sources for `installCheck`. + +Generally, this build is sufficient, but in nightly or CI we also test the attributes `functional_root` and `functional_trusted`, in which the test suite is run with different levels of authorization. + ## Integration tests The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute. These tests include everything that needs to interact with external services or run Nix in a non-trivial distributed setup. Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on ). -You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}` +You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}`. + +If you are testing a build of `nix` that you haven't compiled yet, you may iterate faster by appending the `quickBuild` attribute: `nix build .#hydraJobs.tests.{testName}.quickBuild`. ## Installer tests diff --git a/doc/manual/src/favicon.png b/doc/manual/src/favicon.png new file mode 100644 index 000000000..1ed2b5fe0 Binary files /dev/null and b/doc/manual/src/favicon.png differ diff --git a/doc/manual/src/favicon.svg b/doc/manual/src/favicon.svg new file mode 100644 index 000000000..1d2a6e835 --- /dev/null +++ b/doc/manual/src/favicon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index a71b2e2b3..877c4668b 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -1,5 +1,24 @@ # Glossary +- [content address]{#gloss-content-address} + + A + [*content address*](https://en.wikipedia.org/wiki/Content-addressable_storage) + is a secure way to reference immutable data. + The reference is calculated directly from the content of the data being referenced, which means the reference is + [*tamper proof*](https://en.wikipedia.org/wiki/Tamperproofing) + --- variations of the data should always calculate to distinct content addresses. + + For how Nix uses content addresses, see: + + - [Content-Addressing File System Objects](@docroot@/store/file-system-object/content-address.md) + - [Content-Addressing Store Objects](@docroot@/store/store-object/content-address.md) + - [content-addressed derivation](#gloss-content-addressed-derivation) + + Software Heritage's writing on [*Intrinsic and Extrinsic identifiers*](https://www.softwareheritage.org/2020/07/09/intrinsic-vs-extrinsic-identifiers) is also a good introduction to the value of content-addressing over other referencing schemes. + + Besides content addressing, the Nix store also uses [input addressing](#gloss-input-addressed-store-object). + - [derivation]{#gloss-derivation} A description of a build task. The result of a derivation is a @@ -52,10 +71,9 @@ [`__contentAddressed`](./language/advanced-attributes.md#adv-attr-__contentAddressed) attribute set to `true`. -- [fixed-output derivation]{#gloss-fixed-output-derivation} +- [fixed-output derivation]{#gloss-fixed-output-derivation} (FOD) - A derivation which includes the - [`outputHash`](./language/advanced-attributes.md#adv-attr-outputHash) attribute. + A [derivation] where a cryptographic hash of the [output] is determined in advance using the [`outputHash`](./language/advanced-attributes.md#adv-attr-outputHash) attribute, and where the [`builder`](@docroot@/language/derivations.md#attr-builder) executable has access to the network. - [store]{#gloss-store} @@ -86,7 +104,7 @@ [store path]: #gloss-store-path -- [file system object]{#gloss-store-object} +- [file system object]{#gloss-file-system-object} The Nix data model for representing simplified file system data. @@ -118,9 +136,12 @@ - [content-addressed store object]{#gloss-content-addressed-store-object} - A [store object] whose [store path] is determined by its contents. + A [store object] which is [content-addressed](#gloss-content-address), + i.e. whose [store path] is determined by its contents. This includes derivations, the outputs of [content-addressed derivations](#gloss-content-addressed-derivation), and the outputs of [fixed-output derivations](#gloss-fixed-output-derivation). + See [Content-Addressing Store Objects](@docroot@/store/store-object/content-address.md) for details. + - [substitute]{#gloss-substitute} A substitute is a command invocation stored in the [Nix database] that @@ -147,7 +168,7 @@ - [impure derivation]{#gloss-impure-derivation} - [An experimental feature](#@docroot@/contributing/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, + [An experimental feature](#@docroot@/development/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, so that they are always rebuilt, and their outputs not reused by subsequent calls to realise them. - [Nix database]{#gloss-nix-database} @@ -215,6 +236,20 @@ [output path]: #gloss-output-path +- [output closure]{#gloss-output-closure}\ + The [closure] of an [output path]. It only contains what is [reachable] from the output. + +- [deriving path]{#gloss-deriving-path} + + Deriving paths are a way to refer to [store objects][store object] that ar not yet [realised][realise]. + This is necessary because, in general and particularly for [content-addressed derivations][content-addressed derivation], the [output path] of an [output] is not known in advance. + There are two forms: + + - *constant*: just a [store path] + It can be made [valid][validity] by copying it into the store: from the evaluator, command line interface or another store. + + - *output*: a pair of a [store path] to a [derivation] and an [output] name. + - [deriver]{#gloss-deriver} The [store derivation] that produced an [output path]. @@ -252,13 +287,15 @@ See [installables](./command-ref/new-cli/nix.md#installables) for [`nix` commands](./command-ref/new-cli/nix.md) (experimental) for details. -- [NAR]{#gloss-nar} +- [Nix Archive (NAR)]{#gloss-nar} A *N*ix *AR*chive. This is a serialisation of a path in the Nix store. It can contain regular files, directories and symbolic links. NARs are generated and unpacked using `nix-store --dump` and `nix-store --restore`. + See [Nix Archive](store/file-system-object/content-address.html#serial-nix-archive) for details. + - [`∅`]{#gloss-emtpy-set} The empty set symbol. In the context of profile history, this denotes a package is not present in a particular version of the profile. @@ -275,7 +312,7 @@ - [package attribute set]{#package-attribute-set} - An [attribute set](@docroot@/language/values.md#attribute-set) containing the attribute `type = "derivation";` (derivation for historical reasons), as well as other attributes, such as + An [attribute set](@docroot@/language/types.md#attribute-set) containing the attribute `type = "derivation";` (derivation for historical reasons), as well as other attributes, such as - attributes that refer to the files of a [package], typically in the form of [derivation outputs](#output), - attributes that declare something about how the package is supposed to be installed or used, - other metadata or arbitrary attributes. @@ -288,16 +325,35 @@ See [String interpolation](./language/string-interpolation.md) for details. - [string]: ./language/values.md#type-string - [path]: ./language/values.md#type-path - [attribute name]: ./language/values.md#attribute-set + [string]: ./language/types.md#type-string + [path]: ./language/types.md#type-path + [attribute name]: ./language/types.md#attribute-set + +- [base directory]{#gloss-base-directory} + + The location from which relative paths are resolved. + + - For expressions in a file, the base directory is the directory containing that file. + This is analogous to the directory of a [base URL](https://datatracker.ietf.org/doc/html/rfc1808#section-3.3). + + + + - For expressions written in command line arguments with [`--expr`](@docroot@/command-ref/opt-common.html#opt-expr), the base directory is the current working directory. + + [base directory]: #gloss-base-directory - [experimental feature]{#gloss-experimental-feature} Not yet stabilized functionality guarded by named experimental feature flags. These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting. - See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/contributing/experimental-features.md). + See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/development/experimental-features.md). [Nix language]: ./language/index.md diff --git a/doc/manual/src/installation/env-variables.md b/doc/manual/src/installation/env-variables.md index db98f52ff..035090421 100644 --- a/doc/manual/src/installation/env-variables.md +++ b/doc/manual/src/installation/env-variables.md @@ -53,7 +53,8 @@ ssl-cert-file = /etc/ssl/my-certificate-bundle.crt The Nix installer has special handling for these proxy-related environment variables: `http_proxy`, `https_proxy`, `ftp_proxy`, -`no_proxy`, `HTTP_PROXY`, `HTTPS_PROXY`, `FTP_PROXY`, `NO_PROXY`. +`all_proxy`, `no_proxy`, `HTTP_PROXY`, `HTTPS_PROXY`, `FTP_PROXY`, +`ALL_PROXY`, `NO_PROXY`. If any of these variables are set when running the Nix installer, then the installer will create an override file at diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index 0dc989159..6a168ff3d 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -50,7 +50,7 @@ Supported systems: To explicitly instruct the installer to perform a multi-user installation on your system: ```console -$ curl -L https://nixos.org/nix/install | sh -s -- --daemon +$ bash <(curl -L https://nixos.org/nix/install) --daemon ``` You can run this under your usual user account or `root`. @@ -61,7 +61,7 @@ The script will invoke `sudo` as needed. To explicitly select a single-user installation on your system: ```console -$ curl -L https://nixos.org/nix/install | sh -s -- --no-daemon +$ bash <(curl -L https://nixos.org/nix/install) --no-daemon ``` In a single-user installation, `/nix` is owned by the invoking user. @@ -77,7 +77,7 @@ $ su root # Installing from a binary tarball You can also download a binary tarball that contains Nix and all its dependencies: -- Choose a [version](https://releases.nixos.org/?prefix=nix/) and [system type](../contributing/hacking.md#platforms) +- Choose a [version](https://releases.nixos.org/?prefix=nix/) and [system type](../development/building.md#platforms) - Download and unpack the tarball - Run the installer diff --git a/doc/manual/src/installation/uninstall.md b/doc/manual/src/installation/uninstall.md index 9ead5e53c..590327fea 100644 --- a/doc/manual/src/installation/uninstall.md +++ b/doc/manual/src/installation/uninstall.md @@ -1,16 +1,8 @@ # Uninstalling Nix -## Single User - -If you have a [single-user installation](./installing-binary.md#single-user-installation) of Nix, uninstall it by running: - -```console -$ rm -rf /nix -``` - ## Multi User -Removing a [multi-user installation](./installing-binary.md#multi-user-installation) of Nix is more involved, and depends on the operating system. +Removing a [multi-user installation](./installing-binary.md#multi-user-installation) depends on the operating system. ### Linux @@ -51,7 +43,15 @@ which you may remove. ### macOS -1. Edit `/etc/zshrc`, `/etc/bashrc`, and `/etc/bash.bashrc` to remove the lines sourcing `nix-daemon.sh`, which should look like this: +1. If system-wide shell initialisation files haven't been altered since installing Nix, use the backups made by the installer: + + ```console + sudo mv /etc/zshrc.backup-before-nix /etc/zshrc + sudo mv /etc/bashrc.backup-before-nix /etc/bashrc + sudo mv /etc/bash.bashrc.backup-before-nix /etc/bash.bashrc + ``` + + Otherwise, edit `/etc/zshrc`, `/etc/bashrc`, and `/etc/bash.bashrc` to remove the lines sourcing `nix-daemon.sh`, which should look like this: ```bash # Nix @@ -61,18 +61,6 @@ which you may remove. # End Nix ``` - If these files haven't been altered since installing Nix you can simply put - the backups back in place: - - ```console - sudo mv /etc/zshrc.backup-before-nix /etc/zshrc - sudo mv /etc/bashrc.backup-before-nix /etc/bashrc - sudo mv /etc/bash.bashrc.backup-before-nix /etc/bash.bashrc - ``` - - This will stop shells from sourcing the file and bringing everything you - installed using Nix in scope. - 2. Stop and remove the Nix daemon services: ```console @@ -82,8 +70,7 @@ which you may remove. sudo rm /Library/LaunchDaemons/org.nixos.darwin-store.plist ``` - This stops the Nix daemon and prevents it from being started next time you - boot the system. + This stops the Nix daemon and prevents it from being started next time you boot the system. 3. Remove the `nixbld` group and the `_nixbuildN` users: @@ -94,25 +81,42 @@ which you may remove. This will remove all the build users that no longer serve a purpose. -4. Edit fstab using `sudo vifs` to remove the line mounting the Nix Store - volume on `/nix`, which looks like - `UUID= /nix apfs rw,noauto,nobrowse,suid,owners` or - `LABEL=Nix\040Store /nix apfs rw,nobrowse`. This will prevent automatic - mounting of the Nix Store volume. +4. Edit fstab using `sudo vifs` to remove the line mounting the Nix Store volume on `/nix`, which looks like -5. Edit `/etc/synthetic.conf` to remove the `nix` line. If this is the only - line in the file you can remove it entirely, `sudo rm /etc/synthetic.conf`. - This will prevent the creation of the empty `/nix` directory to provide a - mountpoint for the Nix Store volume. + ``` + UUID= /nix apfs rw,noauto,nobrowse,suid,owners + ``` + or -6. Remove the files Nix added to your system: + ``` + LABEL=Nix\040Store /nix apfs rw,nobrowse + ``` + + by setting the cursor on the respective line using the error keys, and pressing `dd`, and then `:wq` to save the file. + + This will prevent automatic mounting of the Nix Store volume. + +5. Edit `/etc/synthetic.conf` to remove the `nix` line. + If this is the only line in the file you can remove it entirely: + + ```bash + if [ -f /etc/synthetic.conf ]; then + if [ "$(cat /etc/synthetic.conf)" = "nix" ]; then + sudo rm /etc/synthetic.conf + else + sudo vi /etc/synthetic.conf + fi + fi + ``` + + This will prevent the creation of the empty `/nix` directory. + +6. Remove the files Nix added to your system, except for the store: ```console sudo rm -rf /etc/nix /var/root/.nix-profile /var/root/.nix-defexpr /var/root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels ``` - This gets rid of any data Nix may have created except for the store which is - removed next. 7. Remove the Nix Store volume: @@ -120,29 +124,32 @@ which you may remove. sudo diskutil apfs deleteVolume /nix ``` - This will remove the Nix Store volume and everything that was added to the - store. + This will remove the Nix Store volume and everything that was added to the store. - If the output indicates that the command couldn't remove the volume, you should - make sure you don't have an _unmounted_ Nix Store volume. Look for a - "Nix Store" volume in the output of the following command: + If the output indicates that the command couldn't remove the volume, you should make sure you don't have an _unmounted_ Nix Store volume. + Look for a "Nix Store" volume in the output of the following command: ```console diskutil list ``` - If you _do_ see a "Nix Store" volume, delete it by re-running the diskutil - deleteVolume command, but replace `/nix` with the store volume's `diskXsY` - identifier. + If you _do_ find a "Nix Store" volume, delete it by running `diskutil deleteVolume` with the store volume's `diskXsY` identifier. > **Note** > -> After you complete the steps here, you will still have an empty `/nix` -> directory. This is an expected sign of a successful uninstall. The empty -> `/nix` directory will disappear the next time you reboot. +> After you complete the steps here, you will still have an empty `/nix` directory. +> This is an expected sign of a successful uninstall. +> The empty `/nix` directory will disappear the next time you reboot. > -> You do not have to reboot to finish uninstalling Nix. The uninstall is -> complete. macOS (Catalina+) directly controls root directories and its -> read-only root will prevent you from manually deleting the empty `/nix` -> mountpoint. +> You do not have to reboot to finish uninstalling Nix. +> The uninstall is complete. +> macOS (Catalina+) directly controls root directories, and its read-only root will prevent you from manually deleting the empty `/nix` mountpoint. +## Single User + +To remove a [single-user installation](./installing-binary.md#single-user-installation) of Nix, run: + +```console +$ rm -rf /nix ~/.nix-channels ~/.nix-defexpr ~/.nix-profile +``` +You might also want to manually remove references to Nix from your `~/.profile`. diff --git a/doc/manual/src/installation/upgrading.md b/doc/manual/src/installation/upgrading.md index 38edcdbc5..a433f1d30 100644 --- a/doc/manual/src/installation/upgrading.md +++ b/doc/manual/src/installation/upgrading.md @@ -28,7 +28,7 @@ $ sudo su ## macOS multi-user ```console -$ sudo nix-env --install --file '' --attr nix -I nixpkgs=channel:nixpkgs-unstable +$ sudo nix-env --install --file '' --attr nix cacert -I nixpkgs=channel:nixpkgs-unstable $ sudo launchctl remove org.nixos.nix-daemon $ sudo launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist ``` diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 7306fc182..51b83fc8a 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -113,19 +113,18 @@ Derivations can declare some infrequently used optional attributes. > `nix-build`. If the [`configurable-impure-env` experimental - feature](@docroot@/contributing/experimental-features.md#xp-feature-configurable-impure-env) + feature](@docroot@/development/experimental-features.md#xp-feature-configurable-impure-env) is enabled, these environment variables can also be controlled through the [`impure-env`](@docroot@/command-ref/conf-file.md#conf-impure-env) configuration setting. - [`outputHash`]{#adv-attr-outputHash}; [`outputHashAlgo`]{#adv-attr-outputHashAlgo}; [`outputHashMode`]{#adv-attr-outputHashMode}\ - These attributes declare that the derivation is a so-called - *fixed-output derivation*, which means that a cryptographic hash of - the output is already known in advance. When the build of a - fixed-output derivation finishes, Nix computes the cryptographic - hash of the output and compares it to the hash declared with these - attributes. If there is a mismatch, the build fails. + These attributes declare that the derivation is a so-called *fixed-output derivation* (FOD), which means that a cryptographic hash of the output is already known in advance. + + As opposed to regular derivations, the [`builder`] executable of a fixed-output derivation has access to the network. + Nix computes a cryptographic hash of its output and compares that to the hash declared with these attributes. + If there is a mismatch, the derivation fails. The rationale for fixed-output derivations is derivations such as those produced by the `fetchurl` function. This function downloads a @@ -188,38 +187,49 @@ Derivations can declare some infrequently used optional attributes. } ``` - The `outputHashAlgo` attribute specifies the hash algorithm used to - compute the hash. It can currently be `"sha1"`, `"sha256"` or - `"sha512"`. + The `outputHash` attribute must be a string containing the hash in either hexadecimal or "nix32" encoding, or following the format for integrity metadata as defined by [SRI](https://www.w3.org/TR/SRI/). + The "nix32" encoding is an adaptation of base-32 encoding. + The [`convertHash`](@docroot@/language/builtins.md#builtins-convertHash) function shows how to convert between different encodings, and the [`nix-hash` command](../command-ref/nix-hash.md) has information about obtaining the hash for some contents, as well as converting to and from encodings. + + The `outputHashAlgo` attribute specifies the hash algorithm used to compute the hash. + It can currently be `"sha1"`, `"sha256"`, `"sha512"`, or `null`. + `outputHashAlgo` can only be `null` when `outputHash` follows the SRI format. The `outputHashMode` attribute determines how the hash is computed. - It must be one of the following two values: + It must be one of the following values: - - `"flat"`\ - The output must be a non-executable regular file. If it isn’t, - the build fails. The hash is simply computed over the contents - of that file (so it’s equal to what Unix commands like - `sha256sum` or `sha1sum` produce). + - [`"flat"`](@docroot@/store/store-object/content-address.md#method-flat) This is the default. - - `"recursive"`\ - The hash is computed over the NAR archive dump of the output - (i.e., the result of [`nix-store --dump`](@docroot@/command-ref/nix-store/dump.md)). In - this case, the output can be anything, including a directory - tree. + - [`"recursive"` or `"nar"`](@docroot@/store/store-object/content-address.md#method-nix-archive) - The `outputHash` attribute, finally, must be a string containing - the hash in either hexadecimal or base-32 notation. (See the - [`nix-hash` command](../command-ref/nix-hash.md) for information - about converting to and from base-32 notation.) + > **Compatibility** + > + > `"recursive"` is the traditional way of indicating this, + > and is supported since 2005 (virtually the entire history of Nix). + > `"nar"` is more clear, and consistent with other parts of Nix (such as the CLI), + > however support for it is only added in Nix version 2.21. + + - [`"text"`](@docroot@/store/store-object/content-address.md#method-text) + + > **Warning** + > + > The use of this method for derivation outputs is part of the [`dynamic-derivations`][xp-feature-dynamic-derivations] experimental feature. + + - [`"git"`](@docroot@/store/store-object/content-address.md#method-git) + + > **Warning** + > + > This method is part of the [`git-hashing`][xp-feature-git-hashing] experimental feature. - [`__contentAddressed`]{#adv-attr-__contentAddressed} + > **Warning** - > This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md). + > This attribute is part of an [experimental feature](@docroot@/development/experimental-features.md). > > To use this attribute, you must enable the - > [`ca-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-ca-derivations) experimental feature. + > [`ca-derivations`][xp-feature-ca-derivations] experimental feature. > For example, in [nix.conf](../command-ref/conf-file.md) you could add: > > ``` @@ -268,7 +278,9 @@ Derivations can declare some infrequently used optional attributes. > **Note** > - > If set to `false`, the [`builder`](./derivations.md#attr-builder) should be able to run on the system type specified in the [`system` attribute](./derivations.md#attr-system), since the derivation cannot be substituted. + > If set to `false`, the [`builder`] should be able to run on the system type specified in the [`system` attribute](./derivations.md#attr-system), since the derivation cannot be substituted. + + [`builder`]: ./derivations.md#attr-builder - [`__structuredAttrs`]{#adv-attr-structuredAttrs}\ If the special attribute `__structuredAttrs` is set to `true`, the other derivation @@ -290,6 +302,12 @@ Derivations can declare some infrequently used optional attributes. (associative) arrays. For example, the attribute `hardening.format = true` ends up as the Bash associative array element `${hardening[format]}`. + > **Warning** + > + > If set to `true`, other advanced attributes such as [`allowedReferences`](#adv-attr-allowedReferences), [`allowedReferences`](#adv-attr-allowedReferences), [`allowedRequisites`](#adv-attr-allowedRequisites), + [`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites), maxSize, and maxClosureSize. + will have no effect. + - [`outputChecks`]{#adv-attr-outputChecks}\ When using [structured attributes](#adv-attr-structuredAttrs), the `outputChecks` attribute allows defining checks per-output. @@ -299,7 +317,7 @@ Derivations can declare some infrequently used optional attributes. [`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites), the following attributes are available: - - `maxSize` defines the maximum size of the resulting [store object](../glossary.md#gloss-store-object). + - `maxSize` defines the maximum size of the resulting [store object](@docroot@/store/store-object.md). - `maxClosureSize` defines the maximum size of the output's closure. - `ignoreSelfRefs` controls whether self-references should be considered when checking for allowed references/requisites. @@ -351,3 +369,7 @@ Derivations can declare some infrequently used optional attributes. ``` ensures that the derivation can only be built on a machine with the `kvm` feature. + +[xp-feature-ca-derivations]: @docroot@/development/experimental-features.md#xp-feature-ca-derivations +[xp-feature-dynamic-derivations]: @docroot@/development/experimental-features.md#xp-feature-dynamic-derivations +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/doc/manual/src/language/builtin-constants-prefix.md b/doc/manual/src/language/builtin-constants-prefix.md deleted file mode 100644 index 50f43006d..000000000 --- a/doc/manual/src/language/builtin-constants-prefix.md +++ /dev/null @@ -1,5 +0,0 @@ -# Built-in Constants - -These constants are built into the Nix language evaluator: - - diff --git a/doc/manual/src/language/builtin-constants-suffix.md b/doc/manual/src/language/builtin-constants-suffix.md deleted file mode 100644 index a74db2857..000000000 --- a/doc/manual/src/language/builtin-constants-suffix.md +++ /dev/null @@ -1 +0,0 @@ - diff --git a/doc/manual/src/language/builtins-prefix.md b/doc/manual/src/language/builtins-prefix.md index 7b2321466..fb983bb7f 100644 --- a/doc/manual/src/language/builtins-prefix.md +++ b/doc/manual/src/language/builtins-prefix.md @@ -1,9 +1,11 @@ -# Built-in Functions +# Built-ins -This section lists the functions built into the Nix language evaluator. -All built-in functions are available through the global [`builtins`](./builtin-constants.md#builtins-builtins) constant. +This section lists the values and functions built into the Nix language evaluator. +All built-ins are available through the global [`builtins`](#builtins-builtins) constant. -For convenience, some built-ins can be accessed directly: +Some built-ins are also exposed directly in the global scope: + + - [`derivation`](#builtins-derivation) - [`import`](#builtins-import) diff --git a/doc/manual/src/language/constructs.md b/doc/manual/src/language/constructs.md deleted file mode 100644 index a82ec5960..000000000 --- a/doc/manual/src/language/constructs.md +++ /dev/null @@ -1,408 +0,0 @@ -# Language Constructs - -## Recursive sets - -Recursive sets are like normal [attribute sets](./values.md#attribute-set), but the attributes can refer to each other. - -> *rec-attrset* = `rec {` [ *name* `=` *expr* `;` `]`... `}` - -Example: - -```nix -rec { - x = y; - y = 123; -}.x -``` - -This evaluates to `123`. - -Note that without `rec` the binding `x = y;` would -refer to the variable `y` in the surrounding scope, if one exists, and -would be invalid if no such variable exists. That is, in a normal -(non-recursive) set, attributes are not added to the lexical scope; in a -recursive set, they are. - -Recursive sets of course introduce the danger of infinite recursion. For -example, the expression - -```nix -rec { - x = y; - y = x; -}.x -``` - -will crash with an `infinite recursion encountered` error message. - -## Let-expressions - -A let-expression allows you to define local variables for an expression. - -> *let-in* = `let` [ *identifier* = *expr* ]... `in` *expr* - -Example: - -```nix -let - x = "foo"; - y = "bar"; -in x + y -``` - -This evaluates to `"foobar"`. - -## Inheriting attributes - -When defining an [attribute set](./values.md#attribute-set) or in a [let-expression](#let-expressions) it is often convenient to copy variables from the surrounding lexical scope (e.g., when you want to propagate attributes). -This can be shortened using the `inherit` keyword. - -Example: - -```nix -let x = 123; in -{ - inherit x; - y = 456; -} -``` - -is equivalent to - -```nix -let x = 123; in -{ - x = x; - y = 456; -} -``` - -and both evaluate to `{ x = 123; y = 456; }`. - -> **Note** -> -> This works because `x` is added to the lexical scope by the `let` construct. - -It is also possible to inherit attributes from another attribute set. - -Example: - -In this fragment from `all-packages.nix`, - -```nix -graphviz = (import ../tools/graphics/graphviz) { - inherit fetchurl stdenv libpng libjpeg expat x11 yacc; - inherit (xorg) libXaw; -}; - -xorg = { - libX11 = ...; - libXaw = ...; - ... -} - -libpng = ...; -libjpg = ...; -... -``` - -the set used in the function call to the function defined in -`../tools/graphics/graphviz` inherits a number of variables from the -surrounding scope (`fetchurl` ... `yacc`), but also inherits `libXaw` -(the X Athena Widgets) from the `xorg` set. - -Summarizing the fragment - -```nix -... -inherit x y z; -inherit (src-set) a b c; -... -``` - -is equivalent to - -```nix -... -x = x; y = y; z = z; -a = src-set.a; b = src-set.b; c = src-set.c; -... -``` - -when used while defining local variables in a let-expression or while -defining a set. - -In a `let` expression, `inherit` can be used to selectively bring specific attributes of a set into scope. For example - - -```nix -let - x = { a = 1; b = 2; }; - inherit (builtins) attrNames; -in -{ - names = attrNames x; -} -``` - -is equivalent to - -```nix -let - x = { a = 1; b = 2; }; -in -{ - names = builtins.attrNames x; -} -``` - -both evaluate to `{ names = [ "a" "b" ]; }`. - -## Functions - -Functions have the following form: - -```nix -pattern: body -``` - -The pattern specifies what the argument of the function must look like, -and binds variables in the body to (parts of) the argument. There are -three kinds of patterns: - - - If a pattern is a single identifier, then the function matches any - argument. Example: - - ```nix - let negate = x: !x; - concat = x: y: x + y; - in if negate true then concat "foo" "bar" else "" - ``` - - Note that `concat` is a function that takes one argument and returns - a function that takes another argument. This allows partial - parameterisation (i.e., only filling some of the arguments of a - function); e.g., - - ```nix - map (concat "foo") [ "bar" "bla" "abc" ] - ``` - - evaluates to `[ "foobar" "foobla" "fooabc" ]`. - - - A *set pattern* of the form `{ name1, name2, …, nameN }` matches a - set containing the listed attributes, and binds the values of those - attributes to variables in the function body. For example, the - function - - ```nix - { x, y, z }: z + y + x - ``` - - can only be called with a set containing exactly the attributes `x`, - `y` and `z`. No other attributes are allowed. If you want to allow - additional arguments, you can use an ellipsis (`...`): - - ```nix - { x, y, z, ... }: z + y + x - ``` - - This works on any set that contains at least the three named - attributes. - - It is possible to provide *default values* for attributes, in - which case they are allowed to be missing. A default value is - specified by writing `name ? e`, where *e* is an arbitrary - expression. For example, - - ```nix - { x, y ? "foo", z ? "bar" }: z + y + x - ``` - - specifies a function that only requires an attribute named `x`, but - optionally accepts `y` and `z`. - - - An `@`-pattern provides a means of referring to the whole value - being matched: - - ```nix - args@{ x, y, z, ... }: z + y + x + args.a - ``` - - but can also be written as: - - ```nix - { x, y, z, ... } @ args: z + y + x + args.a - ``` - - Here `args` is bound to the argument *as passed*, which is further - matched against the pattern `{ x, y, z, ... }`. - The `@`-pattern makes mainly sense with an ellipsis(`...`) as - you can access attribute names as `a`, using `args.a`, which was - given as an additional attribute to the function. - - > **Warning** - > - > `args@` binds the name `args` to the attribute set that is passed to the function. - > In particular, `args` does *not* include any default values specified with `?` in the function's set pattern. - > - > For instance - > - > ```nix - > let - > f = args@{ a ? 23, ... }: [ a args ]; - > in - > f {} - > ``` - > - > is equivalent to - > - > ```nix - > let - > f = args @ { ... }: [ (args.a or 23) args ]; - > in - > f {} - > ``` - > - > and both expressions will evaluate to: - > - > ```nix - > [ 23 {} ] - > ``` - -Note that functions do not have names. If you want to give them a name, -you can bind them to an attribute, e.g., - -```nix -let concat = { x, y }: x + y; -in concat { x = "foo"; y = "bar"; } -``` - -## Conditionals - -Conditionals look like this: - -```nix -if e1 then e2 else e3 -``` - -where *e1* is an expression that should evaluate to a Boolean value -(`true` or `false`). - -## Assertions - -Assertions are generally used to check that certain requirements on or -between features and dependencies hold. They look like this: - -```nix -assert e1; e2 -``` - -where *e1* is an expression that should evaluate to a Boolean value. If -it evaluates to `true`, *e2* is returned; otherwise expression -evaluation is aborted and a backtrace is printed. - -Here is a Nix expression for the Subversion package that shows how -assertions can be used:. - -```nix -{ localServer ? false -, httpServer ? false -, sslSupport ? false -, pythonBindings ? false -, javaSwigBindings ? false -, javahlBindings ? false -, stdenv, fetchurl -, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null -}: - -assert localServer -> db4 != null; â‘ -assert httpServer -> httpd != null && httpd.expat == expat; â‘¡ -assert sslSupport -> openssl != null && (httpServer -> httpd.openssl == openssl); â‘¢ -assert pythonBindings -> swig != null && swig.pythonSupport; -assert javaSwigBindings -> swig != null && swig.javaSupport; -assert javahlBindings -> j2sdk != null; - -stdenv.mkDerivation { - name = "subversion-1.1.1"; - ... - openssl = if sslSupport then openssl else null; â‘£ - ... -} -``` - -The points of interest are: - -1. This assertion states that if Subversion is to have support for - local repositories, then Berkeley DB is needed. So if the Subversion - function is called with the `localServer` argument set to `true` but - the `db4` argument set to `null`, then the evaluation fails. - - Note that `->` is the [logical - implication](https://en.wikipedia.org/wiki/Truth_table#Logical_implication) - Boolean operation. - -2. This is a more subtle condition: if Subversion is built with Apache - (`httpServer`) support, then the Expat library (an XML library) used - by Subversion should be same as the one used by Apache. This is - because in this configuration Subversion code ends up being linked - with Apache code, and if the Expat libraries do not match, a build- - or runtime link error or incompatibility might occur. - -3. This assertion says that in order for Subversion to have SSL support - (so that it can access `https` URLs), an OpenSSL library must be - passed. Additionally, it says that *if* Apache support is enabled, - then Apache's OpenSSL should match Subversion's. (Note that if - Apache support is not enabled, we don't care about Apache's - OpenSSL.) - -4. The conditional here is not really related to assertions, but is - worth pointing out: it ensures that if SSL support is disabled, then - the Subversion derivation is not dependent on OpenSSL, even if a - non-`null` value was passed. This prevents an unnecessary rebuild of - Subversion if OpenSSL changes. - -## With-expressions - -A *with-expression*, - -```nix -with e1; e2 -``` - -introduces the set *e1* into the lexical scope of the expression *e2*. -For instance, - -```nix -let as = { x = "foo"; y = "bar"; }; -in with as; x + y -``` - -evaluates to `"foobar"` since the `with` adds the `x` and `y` attributes -of `as` to the lexical scope in the expression `x + y`. The most common -use of `with` is in conjunction with the `import` function. E.g., - -```nix -with (import ./definitions.nix); ... -``` - -makes all attributes defined in the file `definitions.nix` available as -if they were defined locally in a `let`-expression. - -The bindings introduced by `with` do not shadow bindings introduced by -other means, e.g. - -```nix -let a = 3; in with { a = 1; }; let a = 4; in with { a = 2; }; ... -``` - -establishes the same scope as - -```nix -let a = 1; in let a = 2; in let a = 3; in let a = 4; in ... -``` - -## Comments - -Comments can be single-line, started with a `#` character, or -inline/multi-line, enclosed within `/* ... */`. diff --git a/doc/manual/src/language/constructs/lookup-path.md b/doc/manual/src/language/constructs/lookup-path.md index e87d2922b..11b9fe88c 100644 --- a/doc/manual/src/language/constructs/lookup-path.md +++ b/doc/manual/src/language/constructs/lookup-path.md @@ -4,9 +4,9 @@ > > *lookup-path* = `<` *identifier* [ `/` *identifier* ]... `>` -A lookup path is an identifier with an optional path suffix that resolves to a [path value](@docroot@/language/values.md#type-path) if the identifier matches a search path entry. +A lookup path is an identifier with an optional path suffix that resolves to a [path value](@docroot@/language/types.md#type-path) if the identifier matches a search path entry. -The value of a lookup path is determined by [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath). +The value of a lookup path is determined by [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath). See [`builtins.findFile`](@docroot@/language/builtins.md#builtins-findFile) for details on lookup path resolution. diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index 75f824a34..8e3f0f791 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -12,12 +12,12 @@ It outputs an attribute set, and produces a [store derivation] as a side effect ### Required -- [`name`]{#attr-name} ([String](@docroot@/language/values.md#type-string)) +- [`name`]{#attr-name} ([String](@docroot@/language/types.md#type-string)) A symbolic name for the derivation. It is added to the [store path] of the corresponding [store derivation] as well as to its [output paths](@docroot@/glossary.md#gloss-output-path). - [store path]: @docroot@/glossary.md#gloss-store-path + [store path]: @docroot@/store/store-path.md > **Example** > @@ -31,7 +31,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect > The store derivation's path will be `/nix/store/-hello.drv`. > The [output](#attr-outputs) paths will be of the form `/nix/store/-hello[-]` -- [`system`]{#attr-system} ([String](@docroot@/language/values.md#type-string)) +- [`system`]{#attr-system} ([String](@docroot@/language/types.md#type-string)) The system type on which the [`builder`](#attr-builder) executable is meant to be run. @@ -64,9 +64,9 @@ It outputs an attribute set, and produces a [store derivation] as a side effect > } > ``` > - > [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) has the value of the [`system` configuration option], and defaults to the system type of the current Nix installation. + > [`builtins.currentSystem`](@docroot@/language/builtins.md#builtins-currentSystem) has the value of the [`system` configuration option], and defaults to the system type of the current Nix installation. -- [`builder`]{#attr-builder} ([Path](@docroot@/language/values.md#type-path) | [String](@docroot@/language/values.md#type-string)) +- [`builder`]{#attr-builder} ([Path](@docroot@/language/types.md#type-path) | [String](@docroot@/language/types.md#type-string)) Path to an executable that will perform the build. @@ -113,7 +113,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect ### Optional -- [`args`]{#attr-args} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) +- [`args`]{#attr-args} ([List](@docroot@/language/types.md#list) of [String](@docroot@/language/types.md#type-string)) Default: `[ ]` @@ -132,7 +132,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect > }; > ``` -- [`outputs`]{#attr-outputs} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) +- [`outputs`]{#attr-outputs} ([List](@docroot@/language/types.md#list) of [String](@docroot@/language/types.md#type-string)) Default: `[ "out" ]` @@ -141,7 +141,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect By default, a derivation produces a single output called `out`. However, derivations can produce multiple outputs. - This allows the associated [store objects](@docroot@/glossary.md#gloss-store-object) and their [closures](@docroot@/glossary.md#gloss-closure) to be copied or garbage-collected separately. + This allows the associated [store objects](@docroot@/store/store-object.md) and their [closures](@docroot@/glossary.md#gloss-closure) to be copied or garbage-collected separately. > **Example** > diff --git a/doc/manual/src/language/identifiers.md b/doc/manual/src/language/identifiers.md new file mode 100644 index 000000000..861ee3e20 --- /dev/null +++ b/doc/manual/src/language/identifiers.md @@ -0,0 +1,51 @@ +# Identifiers + +An *identifier* is an [ASCII](https://en.wikipedia.org/wiki/ASCII) character sequence that: +- Starts with a letter (`a-z`, `A-Z`) or underscore (`_`) +- Can contain any number of: + - Letters (`a-z`, `A-Z`) + - Digits (`0-9`) + - Underscores (`_`) + - Apostrophes (`'`) + - Hyphens (`-`) +- Is not one of the [keywords](#keywords) + +> **Syntax** +> +> *identifier* ~ `[A-Za-z_][A-Za-z0-9_'-]*` + +# Names + +A *name* can be written as an [identifier](#identifier) or a [string literal](./syntax.md#string-literal). + +> **Syntax** +> +> *name* → *identifier* | *string* + +Names are used in [attribute sets](./syntax.md#attrs-literal), [`let` bindings](./syntax.md#let-expressions), and [`inherit`](./syntax.md#inheriting-attributes). +Two names are the same if they represent the same sequence of characters, regardless of whether they are written as identifiers or strings. + +# Keywords + +These keywords are reserved and cannot be used as [identifiers](#identifiers): + +- [`assert`](./syntax.md#assertions) +- [`else`][if] +- [`if`][if] +- [`in`][let] +- [`inherit`](./syntax.md#inheriting-attributes) +- [`let`][let] +- [`or`](./operators.md#attribute-selection) (see note) +- [`rec`](./syntax.md#recursive-sets) +- [`then`][if] +- [`with`](./syntax.md#with-expressions) + +[if]: ./syntax.md#conditionals +[let]: ./syntax.md#let-expressions + +> **Note** +> +> The Nix language evaluator currently allows `or` to be used as a name in some contexts, for backwards compatibility reasons. +> Users are advised not to rely on this. +> +> There are long-standing issues with how `or` is parsed as a name, which can't be resolved without making a breaking change to the language. diff --git a/doc/manual/src/language/import-from-derivation.md b/doc/manual/src/language/import-from-derivation.md index fb12ba51a..e901f5bcf 100644 --- a/doc/manual/src/language/import-from-derivation.md +++ b/doc/manual/src/language/import-from-derivation.md @@ -2,9 +2,9 @@ The value of a Nix expression can depend on the contents of a [store object]. -[store object]: @docroot@/glossary.md#gloss-store-object +[store object]: @docroot@/store/store-object.md -Passing an expression `expr` that evaluates to a [store path](@docroot@/glossary.md#gloss-store-path) to any built-in function which reads from the filesystem constitutes Import From Derivation (IFD): +Passing an expression `expr` that evaluates to a [store path](@docroot@/store/store-path.md) to any built-in function which reads from the filesystem constitutes Import From Derivation (IFD): - [`import`](./builtins.md#builtins-import)` expr` - [`builtins.readFile`](./builtins.md#builtins-readFile)` expr` diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md index a26e43a05..2bfdbb8a0 100644 --- a/doc/manual/src/language/index.md +++ b/doc/manual/src/language/index.md @@ -1,7 +1,13 @@ # Nix Language The Nix language is designed for conveniently creating and composing *derivations* – precise descriptions of how contents of existing files are used to derive new files. -It is: + +> **Tip** +> +> These pages are written as a reference. +> If you are learning Nix, nix.dev has a good [introduction to the Nix language](https://nix.dev/tutorials/nix-language). + +The language is: - *domain-specific* @@ -47,7 +53,7 @@ This is an incomplete overview of language features, by example. - *Basic values* + *Basic values ([primitives](@docroot@/language/types.md#primitives))* @@ -65,7 +71,7 @@ This is an incomplete overview of language features, by example. - A string + A [string](@docroot@/language/types.md#type-string) @@ -88,6 +94,18 @@ This is an incomplete overview of language features, by example. + + + + `# Explanation` + + + + + A [comment](@docroot@/language/syntax.md#comments). + + + @@ -100,7 +118,7 @@ This is an incomplete overview of language features, by example. - String interpolation (expands to `"hello world"`, `"1 2 3"`, `"/nix/store/-bash-/bin/sh"`) + [String interpolation](@docroot@/language/string-interpolation.md) (expands to `"hello world"`, `"1 2 3"`, `"/nix/store/-bash-/bin/sh"`) @@ -112,7 +130,7 @@ This is an incomplete overview of language features, by example. - Booleans + [Booleans](@docroot@/language/types.md#type-boolean) @@ -124,7 +142,7 @@ This is an incomplete overview of language features, by example. - Null value + [Null](@docroot@/language/types.md#type-null) value @@ -136,7 +154,7 @@ This is an incomplete overview of language features, by example. - An integer + An [integer](@docroot@/language/types.md#type-int) @@ -148,7 +166,7 @@ This is an incomplete overview of language features, by example. - A floating point number + A [floating point number](@docroot@/language/types.md#type-float) @@ -160,7 +178,7 @@ This is an incomplete overview of language features, by example. - An absolute path + An absolute [path](@docroot@/language/types.md#type-path) @@ -172,7 +190,7 @@ This is an incomplete overview of language features, by example. - A path relative to the file containing this Nix expression + A [path](@docroot@/language/types.md#type-path) relative to the file containing this Nix expression @@ -184,7 +202,7 @@ This is an incomplete overview of language features, by example. - A home path. Evaluates to the `"/.config"`. + A home [path](@docroot@/language/types.md#type-path). Evaluates to the `"/.config"`. @@ -196,7 +214,7 @@ This is an incomplete overview of language features, by example. - Search path for Nix files. Value determined by [`$NIX_PATH` environment variable](../command-ref/env-common.md#env-NIX_PATH). + A [lookup path](@docroot@/language/constructs/lookup-path.md) for Nix files. Value determined by [`$NIX_PATH` environment variable](../command-ref/env-common.md#env-NIX_PATH). @@ -220,7 +238,7 @@ This is an incomplete overview of language features, by example. - A set with attributes named `x` and `y` + An [attribute set](@docroot@/language/types.md#attribute-set) with attributes named `x` and `y` @@ -244,7 +262,7 @@ This is an incomplete overview of language features, by example. - A recursive set, equivalent to `{ x = "foo"; y = "foobar"; }` + A [recursive set](@docroot@/language/syntax.md#recursive-sets), equivalent to `{ x = "foo"; y = "foobar"; }`. @@ -260,7 +278,7 @@ This is an incomplete overview of language features, by example. - Lists with three elements. + [Lists](@docroot@/language/types.md#list) with three elements. @@ -344,7 +362,7 @@ This is an incomplete overview of language features, by example. - Attribute selection (evaluates to `1`) + [Attribute selection](@docroot@/language/types.md#attribute-set) (evaluates to `1`) @@ -356,7 +374,7 @@ This is an incomplete overview of language features, by example. - Attribute selection with default (evaluates to `3`) + [Attribute selection](@docroot@/language/types.md#attribute-set) with default (evaluates to `3`) @@ -392,7 +410,7 @@ This is an incomplete overview of language features, by example. - Conditional expression + [Conditional expression](@docroot@/language/syntax.md#conditionals). @@ -404,7 +422,7 @@ This is an incomplete overview of language features, by example. - Assertion check (evaluates to `"yes!"`). + [Assertion](@docroot@/language/syntax.md#assertions) check (evaluates to `"yes!"`). @@ -416,7 +434,7 @@ This is an incomplete overview of language features, by example. - Variable definition + Variable definition. See [`let`-expressions](@docroot@/language/syntax.md#let-expressions). @@ -428,14 +446,44 @@ This is an incomplete overview of language features, by example. - Add all attributes from the given set to the scope (evaluates to `1`) + Add all attributes from the given set to the scope (evaluates to `1`). + + See [`with`-expressions](@docroot@/language/syntax.md#with-expressions) for details and shadowing caveats. - *Functions (lambdas)* + `inherit pkgs src;` + + + + + Adds the variables to the current scope (attribute set or `let` binding). + Desugars to `pkgs = pkgs; src = src;`. + See [Inheriting attributes](@docroot@/language/syntax.md#inheriting-attributes). + + + + + + + `inherit (pkgs) lib stdenv;` + + + + + Adds the attributes, from the attribute set in parentheses, to the current scope (attribute set or `let` binding). + Desugars to `lib = pkgs.lib; stdenv = pkgs.stdenv;`. + See [Inheriting attributes](@docroot@/language/syntax.md#inheriting-attributes). + + + + + + + *[Functions](@docroot@/language/syntax.md#functions) (lambdas)* @@ -452,7 +500,7 @@ This is an incomplete overview of language features, by example. - A function that expects an integer and returns it increased by 1 + A [function](@docroot@/language/syntax.md#functions) that expects an integer and returns it increased by 1. @@ -464,7 +512,7 @@ This is an incomplete overview of language features, by example. - Curried function, equivalent to `x: (y: x + y)`. Can be used like a function that takes two arguments and returns their sum. + Curried [function](@docroot@/language/syntax.md#functions), equivalent to `x: (y: x + y)`. Can be used like a function that takes two arguments and returns their sum. @@ -476,7 +524,7 @@ This is an incomplete overview of language features, by example. - A function call (evaluates to 101) + A [function](@docroot@/language/syntax.md#functions) call (evaluates to 101) @@ -488,7 +536,7 @@ This is an incomplete overview of language features, by example. - A function bound to a variable and subsequently called by name (evaluates to 103) + A [function](@docroot@/language/syntax.md#functions) bound to a variable and subsequently called by name (evaluates to 103) @@ -500,7 +548,7 @@ This is an incomplete overview of language features, by example. - A function that expects a set with required attributes `x` and `y` and concatenates them + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attributes `x` and `y` and concatenates them @@ -512,7 +560,7 @@ This is an incomplete overview of language features, by example. - A function that expects a set with required attribute `x` and optional `y`, using `"bar"` as default value for `y` + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attribute `x` and optional `y`, using `"bar"` as default value for `y` @@ -524,7 +572,7 @@ This is an incomplete overview of language features, by example. - A function that expects a set with required attributes `x` and `y` and ignores any other attributes + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attributes `x` and `y` and ignores any other attributes @@ -538,7 +586,7 @@ This is an incomplete overview of language features, by example. - A function that expects a set with required attributes `x` and `y`, and binds the whole set to `args` + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attributes `x` and `y`, and binds the whole set to `args` @@ -562,7 +610,8 @@ This is an incomplete overview of language features, by example. - Load and return Nix expression in given file + Load and return Nix expression in given file. + See [import](@docroot@/language/builtins.md#builtins-import). @@ -574,7 +623,8 @@ This is an incomplete overview of language features, by example. - Apply a function to every element of a list (evaluates to `[ 2 4 6 ]`) + Apply a function to every element of a list (evaluates to `[ 2 4 6 ]`). + See [`map`](@docroot@/language/builtins.md#builtins-map). diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index 6fd66864b..e1c020781 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -26,12 +26,16 @@ | Logical conjunction (`AND`) | *bool* `&&` *bool* | left | 12 | | Logical disjunction (`OR`) | *bool* \|\| *bool* | left | 13 | | [Logical implication] | *bool* `->` *bool* | right | 14 | +| [Pipe operator] (experimental) | *expr* `\|>` *func* | left | 15 | +| [Pipe operator] (experimental) | *func* `<\|` *expr* | right | 15 | -[string]: ./values.md#type-string -[path]: ./values.md#type-path -[number]: ./values.md#type-number -[list]: ./values.md#list -[attribute set]: ./values.md#attribute-set +[string]: ./types.md#type-string +[path]: ./types.md#type-path +[number]: ./types.md#type-float +[list]: ./types.md#list +[attribute set]: ./types.md#attribute-set + + ## Attribute selection @@ -42,12 +46,6 @@ Select the attribute denoted by attribute path *attrpath* from [attribute set] *attrset*. If the attribute doesn’t exist, return the *expr* after `or` if provided, otherwise abort evaluation. -An attribute path is a dot-separated list of [attribute names](./values.md#attribute-set). - -> **Syntax** -> -> *attrpath* = *name* [ `.` *name* ]... - [Attribute selection]: #attribute-selection ## Has attribute @@ -61,7 +59,7 @@ The result is a [Boolean] value. See also: [`builtins.hasAttr`](@docroot@/language/builtins.md#builtins-hasAttr) -[Boolean]: ./values.md#type-boolean +[Boolean]: ./types.md#type-boolean [Has attribute]: #has-attribute @@ -128,8 +126,8 @@ The result is a string. > The file or directory at *path* must exist and is copied to the [store]. > The path appears in the result as the corresponding [store path]. -[store path]: ../glossary.md#gloss-store-path -[store]: ../glossary.md#gloss-store +[store path]: @docroot@/store/store-path.md +[store]: @docroot@/glossary.md#gloss-store [String and path concatenation]: #string-and-path-concatenation @@ -141,7 +139,7 @@ The result is a string. Update [attribute set] *attrset1* with names and values from *attrset2*. -The returned attribute set will have of all the attributes in *attrset1* and *attrset2*. +The returned attribute set will have all of the attributes in *attrset1* and *attrset2*. If an attribute name is present in both, the attribute value from the latter is taken. [Update]: #update @@ -172,7 +170,7 @@ All comparison operators are implemented in terms of `<`, and the following equi - Numbers are type-compatible, see [arithmetic] operators. - Floating point numbers only differ up to a limited precision. -[function]: ./constructs.md#functions +[function]: ./syntax.md#functions [Equality]: #equality @@ -182,3 +180,34 @@ Equivalent to `!`*b1* `||` *b2*. [Logical implication]: #logical-implication +## Pipe operators + +- *a* `|>` *b* is equivalent to *b* *a* +- *a* `<|` *b* is equivalent to *a* *b* + +> **Example** +> +> ``` +> nix-repl> 1 |> builtins.add 2 |> builtins.mul 3 +> 9 +> +> nix-repl> builtins.add 1 <| builtins.mul 2 <| 3 +> 7 +> ``` + +> **Warning** +> +> This syntax is part of an +> [experimental feature](@docroot@/development/experimental-features.md) +> and may change in future releases. +> +> To use this syntax, make sure the +> [`pipe-operators` experimental feature](@docroot@/development/experimental-features.md#xp-feature-pipe-operators) +> is enabled. +> For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): +> +> ``` +> extra-experimental-features = pipe-operators +> ``` + +[Pipe operator]: #pipe-operators diff --git a/doc/manual/src/language/scope.md b/doc/manual/src/language/scope.md new file mode 100644 index 000000000..9373324e2 --- /dev/null +++ b/doc/manual/src/language/scope.md @@ -0,0 +1,28 @@ +# Scoping rules + +A *scope* in the Nix language is a dictionary keyed by [name](./identifiers.md#names), mapping each name to an expression and a *definition type*. +The definition type is either *explicit* or *implicit*. +Each entry in this dictionary is a *definition*. + +Explicit definitions are created by the following expressions: +- [let-expressions](syntax.md#let-expressions) +- [recursive attribute set literals](syntax.md#recursive-sets) (`rec`) +- [function literals](syntax.md#functions) + +Implicit definitions are only created by [with-expressions](./syntax.md#with-expressions). + +Every expression is *enclosed* by a scope. +The outermost expression is enclosed by the [built-in, global scope](./builtins.md), which contains only explicit definitions. +The expressions listed above *extend* their enclosing scope by adding new definitions, or replacing existing ones with the same name. +An explicit definition can replace a definition of any type; an implicit definition can only replace another implicit definition. + +Each of the above expressions defines which of its subexpressions are enclosed by the extended scope. +In all other cases, the same scope that encloses an expression is the enclosing scope for its subexpressions. + +The Nix language is [statically scoped](https://en.wikipedia.org/wiki/Scope_(computer_science)#Lexical_scope); +the value of a variable is determined only by the variable's enclosing scope, and not by the dynamic context in which the variable is evaluated. + +> **Note** +> +> Expressions entered into the [Nix REPL](@docroot@/command-ref/new-cli/nix3-repl.md) are enclosed by a scope that can be extended by command line arguments or previous REPL commands. +> These ways of extending scope are not, strictly speaking, part of the Nix language. diff --git a/doc/manual/src/language/string-context.md b/doc/manual/src/language/string-context.md new file mode 100644 index 000000000..6a3482cfd --- /dev/null +++ b/doc/manual/src/language/string-context.md @@ -0,0 +1,134 @@ +# String context + +> **Note** +> +> This is an advanced topic. +> The Nix language is designed to be used without the programmer consciously dealing with string contexts or even knowing what they are. + +A string in the Nix language is not just a sequence of characters like strings in other languages. +It is actually a pair of a sequence of characters and a *string context*. +The string context is an (unordered) set of *string context elements*. + +The purpose of string contexts is to collect non-string values attached to strings via +[string concatenation](./operators.md#string-concatenation), +[string interpolation](./string-interpolation.md), +and similar operations. +The idea is that a user can combine together values to create a build instructions for derivations without manually keeping track of where they come from. +Then the Nix language implicitly does that bookkeeping to efficiently obtain the closure of derivation inputs. + +> **Note** +> +> String contexts are *not* explicitly manipulated in idiomatic Nix language code. + +String context elements come in different forms: + +- [deriving path]{#string-context-element-derived-path} + + A string context element of this type is a [deriving path](@docroot@/glossary.md#gloss-deriving-path). + They can be either of type [constant](#string-context-constant) or [output](#string-context-output), which correspond to the types of deriving paths. + + - [Constant string context elements]{#string-context-constant} + + > **Example** + > + > [`builtins.storePath`] creates a string with a single constant string context element: + > + > ```nix + > builtins.getContext (builtins.storePath "/nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10") + > ``` + > evaluates to + > ```nix + > { + > "/nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10" = { + > path = true; + > }; + > } + > ``` + + [deriving path]: @docroot@/glossary.md#gloss-deriving-path + [store path]: @docroot@/glossary.md#gloss-store-path + [`builtins.storePath`]: ./builtins.md#builtins-storePath + + - [Output string context elements]{#string-context-output} + + > **Example** + > + > The behavior of string contexts are best demonstrated with a built-in function that is still experimental: [`builtins.outputOf`]. + > This example will *not* work with stable Nix! + > + > ```nix + > builtins.getContext + > (builtins.outputOf + > (builtins.storePath "/nix/store/fvchh9cvcr7kdla6n860hshchsba305w-hello-2.12.drv") + > "out") + > ``` + > evaluates to + > ```nix + > { + > "/nix/store/fvchh9cvcr7kdla6n860hshchsba305w-hello-2.12.drv" = { + > outputs = [ "out" ]; + > }; + > } + > ``` + + [`builtins.outputOf`]: ./builtins.md#builtins-outputOf + +- [*derivation deep*]{#string-context-element-derivation-deep} + + *derivation deep* is an advanced feature intended to be used with the + [`exportReferencesGraph` derivation attribute](./advanced-attributes.html#adv-attr-exportReferencesGraph). + A *derivation deep* string context element is a derivation path, and refers to both its outputs and the entire build closure of that derivation: + all its outputs, all the other derivations the given derivation depends on, and all the outputs of those. + + > **Example** + > + > The best way to illustrate *derivation deep* string contexts is with [`builtins.addDrvOutputDependencies`]. + > Take a regular constant string context element pointing to a derivation, and transform it into a "Derivation deep" string context element. + > + > ```nix + > builtins.getContext + > (builtins.addDrvOutputDependencies + > (builtins.storePath "/nix/store/fvchh9cvcr7kdla6n860hshchsba305w-hello-2.12.drv")) + > ``` + > evaluates to + > ```nix + > { + > "/nix/store/fvchh9cvcr7kdla6n860hshchsba305w-hello-2.12.drv" = { + > allOutputs = true; + > }; + > } + > ``` + + [`builtins.addDrvOutputDependencies`]: ./builtins.md#builtins-addDrvOutputDependencies + [`builtins.unsafeDiscardOutputDependency`]: ./builtins.md#builtins-unsafeDiscardOutputDependency + +## Inspecting string contexts + +Most basically, [`builtins.hasContext`] will tell whether a string has a non-empty context. + +When more granular information is needed, [`builtins.getContext`] can be used. +It creates an [attribute set] representing the string context, which can be inspected as usual. + +[`builtins.hasContext`]: ./builtins.md#builtins-hasContext +[`builtins.getContext`]: ./builtins.md#builtins-getContext +[attribute set]: ./types.md#attribute-set + +## Clearing string contexts + +[`buitins.unsafeDiscardStringContext`](./builtins.md#builtins-unsafeDiscardStringContext) will make a copy of a string, but with an empty string context. +The returned string can be used in more ways, e.g. by operators that require the string context to be empty. +The requirement to explicitly discard the string context in such use cases helps ensure that string context elements are not lost by mistake. +The "unsafe" marker is only there to remind that Nix normally guarantees that dependencies are tracked, whereas the returned string has lost them. + +## Constructing string contexts + +[`builtins.appendContext`] will create a copy of a string, but with additional string context elements. +The context is specified explicitly by an [attribute set] in the format that [`builtins.hasContext`] produces. +A string with arbitrary contexts can be made like this: + +1. Create a string with the desired string context elements. + (The contents of the string do not matter.) +2. Dump its context with [`builtins.getContext`]. +3. Combine it with a base string and repeated [`builtins.appendContext`] calls. + +[`builtins.appendContext`]: ./builtins.md#builtins-appendContext diff --git a/doc/manual/src/language/string-interpolation.md b/doc/manual/src/language/string-interpolation.md index 7d81c2020..1778bdfa0 100644 --- a/doc/manual/src/language/string-interpolation.md +++ b/doc/manual/src/language/string-interpolation.md @@ -4,9 +4,9 @@ String interpolation is a language feature where a [string], [path], or [attribu Such a construct is called *interpolated string*, and the expression inside is an [interpolated expression](#interpolated-expression). -[string]: ./values.md#type-string -[path]: ./values.md#type-path -[attribute set]: ./values.md#attribute-set +[string]: ./types.md#type-string +[path]: ./types.md#type-path +[attribute set]: ./types.md#attribute-set ## Examples @@ -20,7 +20,7 @@ Rather than writing (where `freetype` is a [derivation]), you can instead write -[derivation]: ../glossary.md#gloss-derivation +[derivation]: @docroot@/glossary.md#gloss-derivation ```nix "--with-freetype2-library=${freetype}/lib" @@ -43,6 +43,47 @@ configureFlags = " Note that Nix expressions and strings can be arbitrarily nested; in this case the outer string contains various interpolated expressions that themselves contain strings (e.g., `"-thread"`), some of which in turn contain interpolated expressions (e.g., `${mesa}`). +To write a literal `${` in an regular string, escape it with a backslash (`\`). + +> **Example** +> +> ```nix +> "echo \${PATH}" +> ``` +> +> "echo ${PATH}" + +To write a literal `${` in an indented string, escape it with two single quotes (`''`). + +> **Example** +> +> ```nix +> '' +> echo ''${PATH} +> '' +> ``` +> +> "echo ${PATH}\n" + +`$${` can be written literally in any string. + +> **Example** +> +> In Make, `$` in file names or recipes is represented as `$$`, see [GNU `make`: Basics of Variable Reference](https://www.gnu.org/software/make/manual/html_node/Reference.html#Basics-of-Variable-References). +> This can be expressed directly in the Nix language strings: +> +> ```nix +> '' +> MAKEVAR = Hello +> all: +> @export BASHVAR=world; echo $(MAKEVAR) $${BASHVAR} +> '' +> ``` +> +> "MAKEVAR = Hello\nall:\n\t@export BASHVAR=world; echo $(MAKEVAR) $\${BASHVAR}\n" + +See the [documentation on strings][string] for details. + ### Path Rather than writing @@ -107,9 +148,9 @@ An expression that is interpolated must evaluate to one of the following: A string interpolates to itself. -A path in an interpolated expression is first copied into the Nix store, and the resulting string is the [store path] of the newly created [store object](../glossary.md#gloss-store-object). +A path in an interpolated expression is first copied into the Nix store, and the resulting string is the [store path] of the newly created [store object](@docroot@/store/store-object.md). -[store path]: ../glossary.md#gloss-store-path +[store path]: @docroot@/store/store-path.md > **Example** > diff --git a/doc/manual/src/language/syntax.md b/doc/manual/src/language/syntax.md new file mode 100644 index 000000000..6108bacd6 --- /dev/null +++ b/doc/manual/src/language/syntax.md @@ -0,0 +1,866 @@ +# Language Constructs + +This section covers syntax and semantics of the Nix language. + +## Basic Literals + +### String {#string-literal} + + *Strings* can be written in three ways. + + The most common way is to enclose the string between double quotes, e.g., `"foo bar"`. + Strings can span multiple lines. + The results of other expressions can be included into a string by enclosing them in `${ }`, a feature known as [string interpolation]. + + [string interpolation]: ./string-interpolation.md + + The following must be escaped to represent them within a string, by prefixing with a backslash (`\`): + + - Double quote (`"`) + + > **Example** + > + > ```nix + > "\"" + > ``` + > + > "\"" + + - Backslash (`\`) + + > **Example** + > + > ```nix + > "\\" + > ``` + > + > "\\" + + - Dollar sign followed by an opening curly bracket (`${`) – "dollar-curly" + + > **Example** + > + > ```nix + > "\${" + > ``` + > + > "\${" + + The newline, carriage return, and tab characters can be written as `\n`, `\r` and `\t`, respectively. + + A "double-dollar-curly" (`$${`) can be written literally. + + > **Example** + > + > ```nix + > "$${" + > ``` + > + > "$\${" + + String values are output on the terminal with Nix-specific escaping. + Strings written to files will contain the characters encoded by the escaping. + + The second way to write string literals is as an *indented string*, which is enclosed between pairs of *double single-quotes* (`''`), like so: + + ```nix + '' + This is the first line. + This is the second line. + This is the third line. + '' + ``` + + This kind of string literal intelligently strips indentation from + the start of each line. To be precise, it strips from each line a + number of spaces equal to the minimal indentation of the string as a + whole (disregarding the indentation of empty lines). For instance, + the first and second line are indented two spaces, while the third + line is indented four spaces. Thus, two spaces are stripped from + each line, so the resulting string is + + ```nix + "This is the first line.\nThis is the second line.\n This is the third line.\n" + ``` + + > **Note** + > + > Whitespace and newline following the opening `''` is ignored if there is no non-whitespace text on the initial line. + + > **Warning** + > + > Prefixed tab characters are not stripped. + > + > > **Example** + > > + > > The following indented string is prefixed with tabs: + > > + > > '' + > > all: + > > @echo hello + > > '' + > > + > > "\tall:\n\t\t@echo hello\n" + + Indented strings support [string interpolation]. + + The following must be escaped to represent them in an indented string: + + - `$` is escaped by prefixing it with two single quotes (`''`) + + > **Example** + > + > ```nix + > '' + > ''$ + > '' + > ``` + > + > "$\n" + + - `''` is escaped by prefixing it with one single quote (`'`) + + > **Example** + > + > ```nix + > '' + > ''' + > '' + > ``` + > + > "''\n" + + These special characters are escaped as follows: + - Linefeed (`\n`): `''\n` + - Carriage return (`\r`): `''\r` + - Tab (`\t`): `''\t` + + `''\` escapes any other character. + + A "double-dollar-curly" (`$${`) can be written literally. + + > **Example** + > + > ```nix + > '' + > $${ + > '' + > ``` + > + > "$\${\n" + + Indented strings are primarily useful in that they allow multi-line + string literals to follow the indentation of the enclosing Nix + expression, and that less escaping is typically necessary for + strings representing languages such as shell scripts and + configuration files because `''` is much less common than `"`. + Example: + + ```nix + stdenv.mkDerivation { + ... + postInstall = + '' + mkdir $out/bin $out/etc + cp foo $out/bin + echo "Hello World" > $out/etc/foo.conf + ${if enableBar then "cp bar $out/bin" else ""} + ''; + ... + } + ``` + + Finally, as a convenience, *URIs* as defined in appendix B of + [RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as + is*, without quotes. For instance, the string + `"http://example.org/foo.tar.bz2"` can also be written as + `http://example.org/foo.tar.bz2`. + +### Number {#number-literal} + + + + Numbers, which can be *integers* (like `123`) or *floating point* + (like `123.43` or `.27e13`). + + See [arithmetic] and [comparison] operators for semantics. + + [arithmetic]: ./operators.md#arithmetic + [comparison]: ./operators.md#comparison + +### Path {#path-literal} + + *Paths* can be expressed by path literals such as `./builder.sh`. + + A path literal must contain at least one slash to be recognised as such. + For instance, `builder.sh` is not a path: + it's parsed as an expression that selects the attribute `sh` from the variable `builder`. + + Path literals are resolved relative to their [base directory](@docroot@/glossary.md#gloss-base-directory). + Path literals may also refer to absolute paths by starting with a slash. + + > **Note** + > + > Absolute paths make expressions less portable. + > In the case where a function translates a path literal into an absolute path string for a configuration file, it is recommended to write a string literal instead. + > This avoids some confusion about whether files at that location will be used during evaluation. + > It also avoids unintentional situations where some function might try to copy everything at the location into the store. + + If the first component of a path is a `~`, it is interpreted such that the rest of the path were relative to the user's home directory. + For example, `~/foo` would be equivalent to `/home/edolstra/foo` for a user whose home directory is `/home/edolstra`. + Path literals that start with `~` are not allowed in [pure](@docroot@/command-ref/conf-file.md#conf-pure-eval) evaluation. + + Path literals can also include [string interpolation], besides being [interpolated into other expressions]. + + [interpolated into other expressions]: ./string-interpolation.md#interpolated-expressions + + At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path. + + `a.${foo}/b.${bar}` is a syntactically valid number division operation. + `./a.${foo}/b.${bar}` is a path. + + [Lookup path](./constructs/lookup-path.md) literals such as `` also resolve to path values. + +## List {#list-literal} + +Lists are formed by enclosing a whitespace-separated list of values +between square brackets. For example, + +```nix +[ 123 ./foo.nix "abc" (f { x = y; }) ] +``` + +defines a list of four elements, the last being the result of a call to +the function `f`. Note that function calls have to be enclosed in +parentheses. If they had been omitted, e.g., + +```nix +[ 123 ./foo.nix "abc" f { x = y; } ] +``` + +the result would be a list of five elements, the fourth one being a +function and the fifth being a set. + +Note that lists are only lazy in values, and they are strict in length. + +Elements in a list can be accessed using [`builtins.elemAt`](./builtins.md#builtins-elemAt). + +## Attribute Set {#attrs-literal} + +An attribute set is a collection of name-value-pairs called *attributes*. + +Attribute sets are written enclosed in curly brackets (`{ }`). +Attribute names and attribute values are separated by an equal sign (`=`). +Each value can be an arbitrary expression, terminated by a semicolon (`;`) + +An attribute name is a string without context, and is denoted by a [name] (an [identifier](./identifiers.md#identifiers) or [string literal](#string-literal)). + +[name]: ./identifiers.md#names + +> **Syntax** +> +> *attrset* → `{` { *name* `=` *expr* `;` } `}` + +Attributes can appear in any order. +An attribute name may only occur once in each attribute set. + +> **Example** +> +> This defines an attribute set with attributes named: +> - `x` with the value `123`, an integer +> - `text` with the value `"Hello"`, a string +> - `y` where the value is the result of applying the function `f` to the attribute set `{ bla = 456; }` +> +> ```nix +> { +> x = 123; +> text = "Hello"; +> y = f { bla = 456; }; +> } +> ``` + +Attributes in nested attribute sets can be written using *attribute paths*. + +> **Syntax** +> +> *attrset* → `{` { *attrpath* `=` *expr* `;` } `}` + +An attribute path is a dot-separated list of [names][name]. + +> **Syntax** +> +> *attrpath* = *name* { `.` *name* } + + + +> **Example** +> +> ```nix +> { a.b.c = 1; a.b.d = 2; } +> ``` +> +> { +> a = { +> b = { +> c = 1; +> d = 2; +> }; +> }; +> } + +Attribute names can also be set implicitly by using the [`inherit` keyword](#inheriting-attributes). + +> **Example** +> +> ```nix +> { inherit (builtins) true; } +> ``` +> +> { true = true; } + +Attributes can be accessed with the [`.` operator](./operators.md#attribute-selection). + +Example: + +```nix +{ a = "Foo"; b = "Bar"; }.a +``` + +This evaluates to `"Foo"`. + +It is possible to provide a default value in an attribute selection using the `or` keyword. + +Example: + +```nix +{ a = "Foo"; b = "Bar"; }.c or "Xyzzy" +``` + +```nix +{ a = "Foo"; b = "Bar"; }.c.d.e.f.g or "Xyzzy" +``` + +will both evaluate to `"Xyzzy"` because there is no `c` attribute in the set. + +You can use arbitrary double-quoted strings as attribute names: + +```nix +{ "$!@#?" = 123; }."$!@#?" +``` + +```nix +let bar = "bar"; in +{ "foo ${bar}" = 123; }."foo ${bar}" +``` + +Both will evaluate to `123`. + +Attribute names support [string interpolation]: + +```nix +let bar = "foo"; in +{ foo = 123; }.${bar} +``` + +```nix +let bar = "foo"; in +{ ${bar} = 123; }.foo +``` + +Both will evaluate to `123`. + +In the special case where an attribute name inside of a set declaration +evaluates to `null` (which is normally an error, as `null` cannot be coerced to +a string), that attribute is simply not added to the set: + +```nix +{ ${if foo then "bar" else null} = true; } +``` + +This will evaluate to `{}` if `foo` evaluates to `false`. + +A set that has a `__functor` attribute whose value is callable (i.e. is +itself a function or a set with a `__functor` attribute whose value is +callable) can be applied as if it were a function, with the set itself +passed in first , e.g., + +```nix +let add = { __functor = self: x: x + self.x; }; + inc = add // { x = 1; }; +in inc 1 +``` + +evaluates to `2`. This can be used to attach metadata to a function +without the caller needing to treat it specially, or to implement a form +of object-oriented programming, for example. + +## Recursive sets + +Recursive sets are like normal [attribute sets](./types.md#attribute-set), but the attributes can refer to each other. + +> *rec-attrset* = `rec {` [ *name* `=` *expr* `;` `]`... `}` + +Example: + +```nix +rec { + x = y; + y = 123; +}.x +``` + +This evaluates to `123`. + +Note that without `rec` the binding `x = y;` would +refer to the variable `y` in the surrounding scope, if one exists, and +would be invalid if no such variable exists. That is, in a normal +(non-recursive) set, attributes are not added to the lexical scope; in a +recursive set, they are. + +Recursive sets of course introduce the danger of infinite recursion. For +example, the expression + +```nix +rec { + x = y; + y = x; +}.x +``` + +will crash with an `infinite recursion encountered` error message. + +## Let-expressions + +A let-expression allows you to define local variables for an expression. + +> *let-in* = `let` [ *identifier* = *expr* ]... `in` *expr* + +Example: + +```nix +let + x = "foo"; + y = "bar"; +in x + y +``` + +This evaluates to `"foobar"`. + +## Inheriting attributes + +When defining an [attribute set](./types.md#attribute-set) or in a [let-expression](#let-expressions) it is often convenient to copy variables from the surrounding lexical scope (e.g., when you want to propagate attributes). +This can be shortened using the `inherit` keyword. + +Example: + +```nix +let x = 123; in +{ + inherit x; + y = 456; +} +``` + +is equivalent to + +```nix +let x = 123; in +{ + x = x; + y = 456; +} +``` + +and both evaluate to `{ x = 123; y = 456; }`. + +> **Note** +> +> This works because `x` is added to the lexical scope by the `let` construct. + +It is also possible to inherit attributes from another attribute set. + +Example: + +In this fragment from `all-packages.nix`, + +```nix +graphviz = (import ../tools/graphics/graphviz) { + inherit fetchurl stdenv libpng libjpeg expat x11 yacc; + inherit (xorg) libXaw; +}; + +xorg = { + libX11 = ...; + libXaw = ...; + ... +} + +libpng = ...; +libjpg = ...; +... +``` + +the set used in the function call to the function defined in +`../tools/graphics/graphviz` inherits a number of variables from the +surrounding scope (`fetchurl` ... `yacc`), but also inherits `libXaw` +(the X Athena Widgets) from the `xorg` set. + +Summarizing the fragment + +```nix +... +inherit x y z; +inherit (src-set) a b c; +... +``` + +is equivalent to + +```nix +... +x = x; y = y; z = z; +a = src-set.a; b = src-set.b; c = src-set.c; +... +``` + +when used while defining local variables in a let-expression or while +defining a set. + +In a `let` expression, `inherit` can be used to selectively bring specific attributes of a set into scope. For example + + +```nix +let + x = { a = 1; b = 2; }; + inherit (builtins) attrNames; +in +{ + names = attrNames x; +} +``` + +is equivalent to + +```nix +let + x = { a = 1; b = 2; }; +in +{ + names = builtins.attrNames x; +} +``` + +both evaluate to `{ names = [ "a" "b" ]; }`. + +## Functions + +Functions have the following form: + +```nix +pattern: body +``` + +The pattern specifies what the argument of the function must look like, +and binds variables in the body to (parts of) the argument. There are +three kinds of patterns: + + - If a pattern is a single identifier, then the function matches any + argument. Example: + + ```nix + let negate = x: !x; + concat = x: y: x + y; + in if negate true then concat "foo" "bar" else "" + ``` + + Note that `concat` is a function that takes one argument and returns + a function that takes another argument. This allows partial + parameterisation (i.e., only filling some of the arguments of a + function); e.g., + + ```nix + map (concat "foo") [ "bar" "bla" "abc" ] + ``` + + evaluates to `[ "foobar" "foobla" "fooabc" ]`. + + - A *set pattern* of the form `{ name1, name2, …, nameN }` matches a + set containing the listed attributes, and binds the values of those + attributes to variables in the function body. For example, the + function + + ```nix + { x, y, z }: z + y + x + ``` + + can only be called with a set containing exactly the attributes `x`, + `y` and `z`. No other attributes are allowed. If you want to allow + additional arguments, you can use an ellipsis (`...`): + + ```nix + { x, y, z, ... }: z + y + x + ``` + + This works on any set that contains at least the three named + attributes. + + It is possible to provide *default values* for attributes, in + which case they are allowed to be missing. A default value is + specified by writing `name ? e`, where *e* is an arbitrary + expression. For example, + + ```nix + { x, y ? "foo", z ? "bar" }: z + y + x + ``` + + specifies a function that only requires an attribute named `x`, but + optionally accepts `y` and `z`. + + - An `@`-pattern provides a means of referring to the whole value + being matched: + + ```nix + args@{ x, y, z, ... }: z + y + x + args.a + ``` + + but can also be written as: + + ```nix + { x, y, z, ... } @ args: z + y + x + args.a + ``` + + Here `args` is bound to the argument *as passed*, which is further + matched against the pattern `{ x, y, z, ... }`. + The `@`-pattern makes mainly sense with an ellipsis(`...`) as + you can access attribute names as `a`, using `args.a`, which was + given as an additional attribute to the function. + + > **Warning** + > + > `args@` binds the name `args` to the attribute set that is passed to the function. + > In particular, `args` does *not* include any default values specified with `?` in the function's set pattern. + > + > For instance + > + > ```nix + > let + > f = args@{ a ? 23, ... }: [ a args ]; + > in + > f {} + > ``` + > + > is equivalent to + > + > ```nix + > let + > f = args @ { ... }: [ (args.a or 23) args ]; + > in + > f {} + > ``` + > + > and both expressions will evaluate to: + > + > ```nix + > [ 23 {} ] + > ``` + +Note that functions do not have names. If you want to give them a name, +you can bind them to an attribute, e.g., + +```nix +let concat = { x, y }: x + y; +in concat { x = "foo"; y = "bar"; } +``` + +## Conditionals + +Conditionals look like this: + +```nix +if e1 then e2 else e3 +``` + +where *e1* is an expression that should evaluate to a Boolean value +(`true` or `false`). + +## Assertions + +Assertions are generally used to check that certain requirements on or +between features and dependencies hold. They look like this: + +```nix +assert e1; e2 +``` + +where *e1* is an expression that should evaluate to a Boolean value. If +it evaluates to `true`, *e2* is returned; otherwise expression +evaluation is aborted and a backtrace is printed. + +Here is a Nix expression for the Subversion package that shows how +assertions can be used:. + +```nix +{ localServer ? false +, httpServer ? false +, sslSupport ? false +, pythonBindings ? false +, javaSwigBindings ? false +, javahlBindings ? false +, stdenv, fetchurl +, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null +}: + +assert localServer -> db4 != null; â‘ +assert httpServer -> httpd != null && httpd.expat == expat; â‘¡ +assert sslSupport -> openssl != null && (httpServer -> httpd.openssl == openssl); â‘¢ +assert pythonBindings -> swig != null && swig.pythonSupport; +assert javaSwigBindings -> swig != null && swig.javaSupport; +assert javahlBindings -> j2sdk != null; + +stdenv.mkDerivation { + name = "subversion-1.1.1"; + ... + openssl = if sslSupport then openssl else null; â‘£ + ... +} +``` + +The points of interest are: + +1. This assertion states that if Subversion is to have support for + local repositories, then Berkeley DB is needed. So if the Subversion + function is called with the `localServer` argument set to `true` but + the `db4` argument set to `null`, then the evaluation fails. + + Note that `->` is the [logical + implication](https://en.wikipedia.org/wiki/Truth_table#Logical_implication) + Boolean operation. + +2. This is a more subtle condition: if Subversion is built with Apache + (`httpServer`) support, then the Expat library (an XML library) used + by Subversion should be same as the one used by Apache. This is + because in this configuration Subversion code ends up being linked + with Apache code, and if the Expat libraries do not match, a build- + or runtime link error or incompatibility might occur. + +3. This assertion says that in order for Subversion to have SSL support + (so that it can access `https` URLs), an OpenSSL library must be + passed. Additionally, it says that *if* Apache support is enabled, + then Apache's OpenSSL should match Subversion's. (Note that if + Apache support is not enabled, we don't care about Apache's + OpenSSL.) + +4. The conditional here is not really related to assertions, but is + worth pointing out: it ensures that if SSL support is disabled, then + the Subversion derivation is not dependent on OpenSSL, even if a + non-`null` value was passed. This prevents an unnecessary rebuild of + Subversion if OpenSSL changes. + +## With-expressions + +A *with-expression*, + +```nix +with e1; e2 +``` + +introduces the set *e1* into the lexical scope of the expression *e2*. +For instance, + +```nix +let as = { x = "foo"; y = "bar"; }; +in with as; x + y +``` + +evaluates to `"foobar"` since the `with` adds the `x` and `y` attributes +of `as` to the lexical scope in the expression `x + y`. The most common +use of `with` is in conjunction with the `import` function. E.g., + +```nix +with (import ./definitions.nix); ... +``` + +makes all attributes defined in the file `definitions.nix` available as +if they were defined locally in a `let`-expression. + +The bindings introduced by `with` do not shadow bindings introduced by +other means, e.g. + +```nix +let a = 3; in with { a = 1; }; let a = 4; in with { a = 2; }; ... +``` + +establishes the same scope as + +```nix +let a = 1; in let a = 2; in let a = 3; in let a = 4; in ... +``` + +Variables coming from outer `with` expressions *are* shadowed: + +```nix +with { a = "outer"; }; +with { a = "inner"; }; +a +``` + +Does evaluate to `"inner"`. + +## Comments + +- Inline comments start with `#` and run until the end of the line. + + > **Example** + > + > ```nix + > # A number + > 2 # Equals 1 + 1 + > ``` + > + > ```console + > 2 + > ``` + +- Block comments start with `/*` and run until the next occurrence of `*/`. + + > **Example** + > + > ```nix + > /* + > Block comments + > can span multiple lines. + > */ "hello" + > ``` + > + > ```console + > "hello" + > ``` + + This means that block comments cannot be nested. + + > **Example** + > + > ```nix + > /* /* nope */ */ 1 + > ``` + > + > ```console + > error: syntax error, unexpected '*' + > + > at «string»:1:15: + > + > 1| /* /* nope */ * + > | ^ + > ``` + + Consider escaping nested comments and unescaping them in post-processing. + + > **Example** + > + > ```nix + > /* /* nested *\/ */ 1 + > ``` + > + > ```console + > 1 + > ``` diff --git a/doc/manual/src/language/types.md b/doc/manual/src/language/types.md new file mode 100644 index 000000000..229756e6b --- /dev/null +++ b/doc/manual/src/language/types.md @@ -0,0 +1,120 @@ +# Data Types + +Every value in the Nix language has one of the following types: + +* [Integer](#type-int) +* [Float](#type-float) +* [Boolean](#type-bool) +* [String](#type-string) +* [Path](#type-path) +* [Null](#type-null) +* [Attribute set](#type-attrs) +* [List](#type-list) +* [Function](#type-function) +* [External](#type-external) + +## Primitives + +### Integer {#type-int} + +An _integer_ in the Nix language is a signed 64-bit integer. + +Non-negative integers can be expressed as [integer literals](syntax.md#number-literal). +Negative integers are created with the [arithmetic negation operator](./operators.md#arithmetic). +The function [`builtins.isInt`](builtins.md#builtins-isInt) can be used to determine if a value is an integer. + +### Float {#type-float} + +A _float_ in the Nix language is a 64-bit [IEEE 754](https://en.wikipedia.org/wiki/IEEE_754) floating-point number. + +Most non-negative floats can be expressed as [float literals](syntax.md#number-literal). +Negative floats are created with the [arithmetic negation operator](./operators.md#arithmetic). +The function [`builtins.isFloat`](builtins.md#builtins-isFloat) can be used to determine if a value is a float. + +### Boolean {#type-bool} + +A _boolean_ in the Nix language is one of _true_ or _false_. + + + +These values are available as attributes of [`builtins`](builtins.md#builtins-builtins) as [`builtins.true`](builtins.md#builtins-true) and [`builtins.false`](builtins.md#builtins-false). +The function [`builtins.isBool`](builtins.md#builtins-isBool) can be used to determine if a value is a boolean. + +### String {#type-string} + +A _string_ in the Nix language is an immutable, finite-length sequence of bytes, along with a [string context](string-context.md). +Nix does not assume or support working natively with character encodings. + +String values without string context can be expressed as [string literals](syntax.md#string-literal). +The function [`builtins.isString`](builtins.md#builtins-isString) can be used to determine if a value is a string. + +### Path {#type-path} + +A _path_ in the Nix language is an immutable, finite-length sequence of bytes starting with `/`, representing a POSIX-style, canonical file system path. +Path values are distinct from string values, even if they contain the same sequence of bytes. +Operations that produce paths will simplify the result as the standard C function [`realpath`] would, except that there is no symbolic link resolution. + +[`realpath`]: https://pubs.opengroup.org/onlinepubs/9699919799/functions/realpath.html + +Paths are suitable for referring to local files, and are often preferable over strings. +- Path values do not contain trailing or duplicate slashes, `.`, or `..`. +- Relative path literals are automatically resolved relative to their [base directory]. +- Tooling can recognize path literals and provide additional features, such as autocompletion, refactoring automation and jump-to-file. + +[base directory]: @docroot@/glossary.md#gloss-base-directory + +A file is not required to exist at a given path in order for that path value to be valid, but a path that is converted to a string with [string interpolation] or [string-and-path concatenation] must resolve to a readable file or directory which will be copied into the Nix store. +For instance, evaluating `"${./foo.txt}"` will cause `foo.txt` from the same directory to be copied into the Nix store and result in the string `"/nix/store/-foo.txt"`. +Operations such as [`import`] can also expect a path to resolve to a readable file or directory. + +[string interpolation]: string-interpolation.md#interpolated-expression +[string-and-path concatenation]: operators.md#string-and-path-concatenation +[`import`]: builtins.md#builtins-import + +> **Note** +> +> The Nix language assumes that all input files will remain _unchanged_ while evaluating a Nix expression. +> For example, assume you used a file path in an interpolated string during a `nix repl` session. +> Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new [store path], since Nix might not re-read the file contents. +> Use `:r` to reset the repl as needed. + +[store path]: @docroot@/store/store-path.md + +Path values can be expressed as [path literals](syntax.md#path-literal). +The function [`builtins.isPath`](builtins.md#builtins-isPath) can be used to determine if a value is a path. + +### Null {#type-null} + +There is a single value of type _null_ in the Nix language. + + + +This value is available as an attribute on the [`builtins`](builtins.md#builtins-builtins) attribute set as [`builtins.null`](builtins.md#builtins-null). + +## Compound values + +### Attribute set {#type-attrs} + + + +An attribute set can be constructed with an [attribute set literal](syntax.md#attrs-literal). +The function [`builtins.isAttrs`](builtins.md#builtins-isAttrs) can be used to determine if a value is an attribute set. + +### List {#type-list} + + + +A list can be constructed with a [list literal](syntax.md#list-literal). +The function [`builtins.isList`](builtins.md#builtins-isList) can be used to determine if a value is a list. + +## Function {#type-function} + + + +A function can be constructed with a [function expression](syntax.md#functions). +The function [`builtins.isFunction`](builtins.md#builtins-isFunction) can be used to determine if a value is a function. + +## External {#type-external} + +An _external_ value is an opaque value created by a Nix [plugin](../command-ref/conf-file.md#conf-plugin-files). +Such a value can be substituted in Nix expressions but only created and used by plugin code. diff --git a/doc/manual/src/language/values.md b/doc/manual/src/language/values.md deleted file mode 100644 index 74ffc7070..000000000 --- a/doc/manual/src/language/values.md +++ /dev/null @@ -1,269 +0,0 @@ -# Data Types - -## Primitives - -- String - - *Strings* can be written in three ways. - - The most common way is to enclose the string between double quotes, - e.g., `"foo bar"`. Strings can span multiple lines. The special - characters `"` and `\` and the character sequence `${` must be - escaped by prefixing them with a backslash (`\`). Newlines, carriage - returns and tabs can be written as `\n`, `\r` and `\t`, - respectively. - - You can include the results of other expressions into a string by enclosing them in `${ }`, a feature known as [string interpolation]. - - [string interpolation]: ./string-interpolation.md - - The second way to write string literals is as an *indented string*, - which is enclosed between pairs of *double single-quotes*, like so: - - ```nix - '' - This is the first line. - This is the second line. - This is the third line. - '' - ``` - - This kind of string literal intelligently strips indentation from - the start of each line. To be precise, it strips from each line a - number of spaces equal to the minimal indentation of the string as a - whole (disregarding the indentation of empty lines). For instance, - the first and second line are indented two spaces, while the third - line is indented four spaces. Thus, two spaces are stripped from - each line, so the resulting string is - - ```nix - "This is the first line.\nThis is the second line.\n This is the third line.\n" - ``` - - Note that the whitespace and newline following the opening `''` is - ignored if there is no non-whitespace text on the initial line. - - Indented strings support [string interpolation]. - - Since `${` and `''` have special meaning in indented strings, you - need a way to quote them. `$` can be escaped by prefixing it with - `''` (that is, two single quotes), i.e., `''$`. `''` can be escaped - by prefixing it with `'`, i.e., `'''`. `$` removes any special - meaning from the following `$`. Linefeed, carriage-return and tab - characters can be written as `''\n`, `''\r`, `''\t`, and `''\` - escapes any other character. - - Indented strings are primarily useful in that they allow multi-line - string literals to follow the indentation of the enclosing Nix - expression, and that less escaping is typically necessary for - strings representing languages such as shell scripts and - configuration files because `''` is much less common than `"`. - Example: - - ```nix - stdenv.mkDerivation { - ... - postInstall = - '' - mkdir $out/bin $out/etc - cp foo $out/bin - echo "Hello World" > $out/etc/foo.conf - ${if enableBar then "cp bar $out/bin" else ""} - ''; - ... - } - ``` - - Finally, as a convenience, *URIs* as defined in appendix B of - [RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as - is*, without quotes. For instance, the string - `"http://example.org/foo.tar.bz2"` can also be written as - `http://example.org/foo.tar.bz2`. - -- Number - - Numbers, which can be *integers* (like `123`) or *floating point* - (like `123.43` or `.27e13`). - - See [arithmetic] and [comparison] operators for semantics. - - [arithmetic]: ./operators.md#arithmetic - [comparison]: ./operators.md#comparison - -- Path - - *Paths*, e.g., `/bin/sh` or `./builder.sh`. A path must contain at - least one slash to be recognised as such. For instance, `builder.sh` - is not a path: it's parsed as an expression that selects the - attribute `sh` from the variable `builder`. If the file name is - relative, i.e., if it does not begin with a slash, it is made - absolute at parse time relative to the directory of the Nix - expression that contained it. For instance, if a Nix expression in - `/foo/bar/bla.nix` refers to `../xyzzy/fnord.nix`, the absolute path - is `/foo/xyzzy/fnord.nix`. - - If the first component of a path is a `~`, it is interpreted as if - the rest of the path were relative to the user's home directory. - e.g. `~/foo` would be equivalent to `/home/edolstra/foo` for a user - whose home directory is `/home/edolstra`. - - For instance, evaluating `"${./foo.txt}"` will cause `foo.txt` in the current directory to be copied into the Nix store and result in the string `"/nix/store/-foo.txt"`. - - Note that the Nix language assumes that all input files will remain _unchanged_ while evaluating a Nix expression. - For example, assume you used a file path in an interpolated string during a `nix repl` session. - Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new [store path], since Nix might not re-read the file contents. - - [store path]: ../glossary.md#gloss-store-path - - Paths can include [string interpolation] and can themselves be [interpolated in other expressions]. - - [interpolated in other expressions]: ./string-interpolation.md#interpolated-expressions - - At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path. - - `a.${foo}/b.${bar}` is a syntactically valid division operation. - `./a.${foo}/b.${bar}` is a path. - - [Lookup paths](./constructs/lookup-path.md) such as `` resolve to path values. - -- Boolean - - *Booleans* with values `true` and `false`. - -- Null - - The null value, denoted as `null`. - -## List - -Lists are formed by enclosing a whitespace-separated list of values -between square brackets. For example, - -```nix -[ 123 ./foo.nix "abc" (f { x = y; }) ] -``` - -defines a list of four elements, the last being the result of a call to -the function `f`. Note that function calls have to be enclosed in -parentheses. If they had been omitted, e.g., - -```nix -[ 123 ./foo.nix "abc" f { x = y; } ] -``` - -the result would be a list of five elements, the fourth one being a -function and the fifth being a set. - -Note that lists are only lazy in values, and they are strict in length. - -Elements in a list can be accessed using [`builtins.elemAt`](./builtins.md#builtins-elemAt). - -## Attribute Set - -An attribute set is a collection of name-value-pairs (called *attributes*) enclosed in curly brackets (`{ }`). - -An attribute name can be an identifier or a [string](#string). -An identifier must start with a letter (`a-z`, `A-Z`) or underscore (`_`), and can otherwise contain letters (`a-z`, `A-Z`), numbers (`0-9`), underscores (`_`), apostrophes (`'`), or dashes (`-`). - -> **Syntax** -> -> *name* = *identifier* | *string* \ -> *identifier* ~ `[a-zA-Z_][a-zA-Z0-9_'-]*` - -Names and values are separated by an equal sign (`=`). -Each value is an arbitrary expression terminated by a semicolon (`;`). - -> **Syntax** -> -> *attrset* = `{` [ *name* `=` *expr* `;` ]... `}` - -Attributes can appear in any order. -An attribute name may only occur once. - -Example: - -```nix -{ - x = 123; - text = "Hello"; - y = f { bla = 456; }; -} -``` - -This defines a set with attributes named `x`, `text`, `y`. - -Attributes can be accessed with the [`.` operator](./operators.md#attribute-selection). - -Example: - -```nix -{ a = "Foo"; b = "Bar"; }.a -``` - -This evaluates to `"Foo"`. - -It is possible to provide a default value in an attribute selection using the `or` keyword. - -Example: - -```nix -{ a = "Foo"; b = "Bar"; }.c or "Xyzzy" -``` - -```nix -{ a = "Foo"; b = "Bar"; }.c.d.e.f.g or "Xyzzy" -``` - -will both evaluate to `"Xyzzy"` because there is no `c` attribute in the set. - -You can use arbitrary double-quoted strings as attribute names: - -```nix -{ "$!@#?" = 123; }."$!@#?" -``` - -```nix -let bar = "bar"; in -{ "foo ${bar}" = 123; }."foo ${bar}" -``` - -Both will evaluate to `123`. - -Attribute names support [string interpolation]: - -```nix -let bar = "foo"; in -{ foo = 123; }.${bar} -``` - -```nix -let bar = "foo"; in -{ ${bar} = 123; }.foo -``` - -Both will evaluate to `123`. - -In the special case where an attribute name inside of a set declaration -evaluates to `null` (which is normally an error, as `null` cannot be coerced to -a string), that attribute is simply not added to the set: - -```nix -{ ${if foo then "bar" else null} = true; } -``` - -This will evaluate to `{}` if `foo` evaluates to `false`. - -A set that has a `__functor` attribute whose value is callable (i.e. is -itself a function or a set with a `__functor` attribute whose value is -callable) can be applied as if it were a function, with the set itself -passed in first , e.g., - -```nix -let add = { __functor = self: x: x + self.x; }; - inc = add // { x = 1; }; -in inc 1 -``` - -evaluates to `2`. This can be used to attach metadata to a function -without the caller needing to treat it specially, or to implement a form -of object-oriented programming, for example. diff --git a/doc/manual/src/language/variables.md b/doc/manual/src/language/variables.md new file mode 100644 index 000000000..af6aff8a2 --- /dev/null +++ b/doc/manual/src/language/variables.md @@ -0,0 +1,10 @@ +# Variables + +A *variable* is an [identifier](identifiers.md) used as an expression. + +> **Syntax** +> +> *expression* → *identifier* + +A variable must have the same name as a definition in the [scope](./scope.md) that encloses it. +The value of a variable is the value of the corresponding expression in the enclosing scope. diff --git a/doc/manual/src/package-management/copy-closure.md b/doc/manual/src/package-management/copy-closure.md deleted file mode 100644 index 14326298b..000000000 --- a/doc/manual/src/package-management/copy-closure.md +++ /dev/null @@ -1,34 +0,0 @@ -# Copying Closures via SSH - -The command `nix-copy-closure` copies a Nix store path along with all -its dependencies to or from another machine via the SSH protocol. It -doesn’t copy store paths that are already present on the target machine. -For example, the following command copies Firefox with all its -dependencies: - - $ nix-copy-closure --to alice@itchy.example.org $(type -p firefox) - -See the [manpage for `nix-copy-closure`](../command-ref/nix-copy-closure.md) for details. - -With `nix-store ---export` and `nix-store --import` you can write the closure of a store -path (that is, the path and all its dependencies) to a file, and then -unpack that file into another Nix store. For example, - - $ nix-store --export $(nix-store --query --requisites $(type -p firefox)) > firefox.closure - -writes the closure of Firefox to a file. You can then copy this file to -another machine and install the closure: - - $ nix-store --import < firefox.closure - -Any store paths in the closure that are already present in the target -store are ignored. It is also possible to pipe the export into another -command, e.g. to copy and install a closure directly to/on another -machine: - - $ nix-store --export $(nix-store --query --requisites $(type -p firefox)) | bzip2 | \ - ssh alice@itchy.example.org "bunzip2 | nix-store --import" - -However, `nix-copy-closure` is generally more efficient because it only -copies paths that are not already present in the target Nix store. diff --git a/doc/manual/src/protocols/derivation-aterm.md b/doc/manual/src/protocols/derivation-aterm.md index e58b602a3..1ba757ae0 100644 --- a/doc/manual/src/protocols/derivation-aterm.md +++ b/doc/manual/src/protocols/derivation-aterm.md @@ -14,6 +14,6 @@ Derivations are serialised in one of the following formats: DrvWithVersion(, ...) ``` - The only `version-string`s that are in use today are for [experimental features](@docroot@/contributing/experimental-features.md): + The only `version-string`s that are in use today are for [experimental features](@docroot@/development/experimental-features.md): - - `"xp-dyn-drv"` for the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. + - `"xp-dyn-drv"` for the [`dynamic-derivations`](@docroot@/development/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. diff --git a/doc/manual/src/protocols/json/derivation.md b/doc/manual/src/protocols/json/derivation.md index 649d543cc..2f85340d6 100644 --- a/doc/manual/src/protocols/json/derivation.md +++ b/doc/manual/src/protocols/json/derivation.md @@ -3,7 +3,7 @@ > **Warning** > > This JSON format is currently -> [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) +> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) > and subject to change. The JSON serialization of a @@ -18,10 +18,30 @@ is a JSON object with the following fields: Information about the output paths of the derivation. This is a JSON object with one member per output, where the key is the output name and the value is a JSON object with these fields: - * `path`: The output path. + * `path`: + The output path, if it is known in advanced. + Otherwise, `null`. + + + * `method`: + For an output which will be [content addresed], a string representing the [method](@docroot@/store/store-object/content-address.md) of content addressing that is chosen. + Valid method strings are: + + - [`flat`](@docroot@/store/store-object/content-address.md#method-flat) + - [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive) + - [`text`](@docroot@/store/store-object/content-address.md#method-text) + - [`git`](@docroot@/store/store-object/content-address.md#method-git) + + Otherwise, `null`. * `hashAlgo`: - For fixed-output derivations, the hashing algorithm (e.g. `sha256`), optionally prefixed by `r:` if `hash` denotes a NAR hash rather than a flat file hash. + For an output which will be [content addresed], the name of the hash algorithm used. + Valid algorithm strings are: + + - `md5` + - `sha1` + - `sha256` + - `sha512` * `hash`: For fixed-output derivations, the expected content hash in base-16. @@ -32,7 +52,8 @@ is a JSON object with the following fields: > "outputs": { > "out": { > "path": "/nix/store/2543j7c6jn75blc3drf4g5vhb1rhdq29-source", - > "hashAlgo": "r:sha256", + > "method": "nar", + > "hashAlgo": "sha256", > "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62" > } > } diff --git a/doc/manual/src/protocols/json/store-object-info.md b/doc/manual/src/protocols/json/store-object-info.md index ba4ab098f..6b4f48437 100644 --- a/doc/manual/src/protocols/json/store-object-info.md +++ b/doc/manual/src/protocols/json/store-object-info.md @@ -3,7 +3,7 @@ > **Warning** > > This JSON format is currently -> [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) +> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) > and subject to change. Info about a [store object]. @@ -24,41 +24,45 @@ Info about a [store object]. An array of [store paths][store path], possibly including this one. -* `ca` (optional): +* `ca`: - Content address of this store object's file system object, used to compute its store path. + If the store object is [content-addressed], + this is the content address of this store object's file system object, used to compute its store path. + Otherwise (i.e. if it is [input-addressed]), this is `null`. -[store path]: @docroot@/glossary.md#gloss-store-path +[store path]: @docroot@/store/store-path.md [file system object]: @docroot@/store/file-system-object.md -[Nix Archive]: @docroot@/glossary.md#gloss-nar +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive ## Impure fields These are not intrinsic properties of the store object. In other words, the same store object residing in different store could have different values for these properties. -* `deriver` (optional): +* `deriver`: - The path to the [derivation] from which this store object is produced. + If known, the path to the [derivation] from which this store object was produced. + Otherwise `null`. [derivation]: @docroot@/glossary.md#gloss-store-derivation * `registrationTime` (optional): - When this derivation was added to the store. + If known, when this derivation was added to the store. + Otherwise `null`. -* `ultimate` (optional): +* `ultimate`: Whether this store object is trusted because we built it ourselves, rather than substituted a build product from elsewhere. -* `signatures` (optional): +* `signatures`: Signatures claiming that this store object is what it claims to be. Not relevant for [content-addressed] store objects, but useful for [input-addressed] store objects. - [content-addressed]: @docroot@/glossary.md#gloss-content-addressed-store-object - [input-addressed]: @docroot@/glossary.md#gloss-input-addressed-store-object +[content-addressed]: @docroot@/store/store-object/content-address.md +[input-addressed]: @docroot@/glossary.md#gloss-input-addressed-store-object ### `.narinfo` extra fields @@ -83,7 +87,7 @@ This information is not intrinsic to the store object, but about how it is store ## Computed closure fields -These fields are not stored at all, but computed by traverising the other other fields across all the store objects in a [closure]. +These fields are not stored at all, but computed by traversing the other fields across all the store objects in a [closure]. * `closureSize`: diff --git a/doc/manual/src/protocols/nix-archive.md b/doc/manual/src/protocols/nix-archive.md new file mode 100644 index 000000000..640b527f1 --- /dev/null +++ b/doc/manual/src/protocols/nix-archive.md @@ -0,0 +1,43 @@ +# Nix Archive (NAR) format + +This is the complete specification of the [Nix Archive] format. +The Nix Archive format closely follows the abstract specification of a [file system object] tree, +because it is designed to serialize exactly that data structure. + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#nix-archive +[file system object]: @docroot@/store/file-system-object.md + +The format of this specification is close to [Extended Backus–Naur form](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form), with the exception of the `str(..)` function / parameterized rule, which length-prefixes and pads strings. +This makes the resulting binary format easier to parse. + +Regular users do *not* need to know this information. +But for those interested in exactly how Nix works, e.g. if they are reimplementing it, this information can be useful. + +```ebnf +nar = str("nix-archive-1"), nar-obj; + +nar-obj = str("("), nar-obj-inner, str(")"); + +nar-obj-inner + = str("type"), str("regular") regular + | str("type"), str("symlink") symlink + | str("type"), str("directory") directory + ; + +regular = [ str("executable"), str("") ], str("contents"), str(contents); + +symlink = str("target"), str(target); + +(* side condition: directory entries must be ordered by their names *) +directory = { directory-entry }; + +directory-entry = str("entry"), str("("), str("name"), str(name), str("node"), nar-obj, str(")"); +``` + +The `str` function / parameterized rule is defined as follows: + +- `str(s)` = `int(|s|), pad(s);` + +- `int(n)` = the 64-bit little endian representation of the number `n` + +- `pad(s)` = the byte sequence `s`, padded with 0s to a multiple of 8 byte diff --git a/doc/manual/src/protocols/store-path.md b/doc/manual/src/protocols/store-path.md index 565c4fa75..52352d358 100644 --- a/doc/manual/src/protocols/store-path.md +++ b/doc/manual/src/protocols/store-path.md @@ -1,12 +1,14 @@ # Complete Store Path Calculation -This is the complete specification for how store paths are calculated. +This is the complete specification for how [store path]s are calculated. The format of this specification is close to [Extended Backus–Naur form](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form), but must deviate for a few things such as hash functions which we treat as bidirectional for specification purposes. Regular users do *not* need to know this information --- store paths can be treated as black boxes computed from the properties of the store objects they refer to. But for those interested in exactly how Nix works, e.g. if they are reimplementing it, this information can be useful. +[store path](@docroot@/store/store-path.md) + ## Store path proper ```ebnf @@ -34,18 +36,23 @@ where - `type` = one of: - ```ebnf - | "text" ( ":" store-path )* + | "text" { ":" store-path } ``` - for encoded derivations written to the store. + This is for the + ["Text"](@docroot@/store/store-object/content-address.md#method-text) + method of content addressing store objects. The optional trailing store paths are the references of the store object. - ```ebnf - | "source" ( ":" store-path )* + | "source" { ":" store-path } [ ":self" ] ``` - For paths copied to the store and hashed via a [Nix Archive (NAR)] and [SHA-256][sha-256]. - Just like in the text case, we can have the store objects referenced by their paths. + This is for the + ["Nix Archive"](@docroot@/store/store-object/content-address.md#method-nix-archive) + method of content addressing store objects, + if the hash algorithm is [SHA-256]. + Just like in the "Text" case, we can have the store objects referenced by their paths. Additionally, we can have an optional `:self` label to denote self reference. - ```ebnf @@ -53,8 +60,12 @@ where ``` For either the outputs built from derivations, - paths copied to the store hashed that area single file hashed directly, or the via a hash algorithm other than [SHA-256][sha-256]. - (in that case "source" is used; this is only necessary for compatibility). + or content-addressed store objects that are not using one of the two above cases. + To be explicit about the latter, that is currently these methods: + + - ["Flat"](@docroot@/store/store-object/content-address.md#method-flat) + - ["Git"](@docroot@/store/store-object/content-address.md#method-git) + - ["Nix Archive"](@docroot@/store/store-object/content-address.md#method-nix-archive) if the hash algorithm is not [SHA-256]. `id` is the name of the output (usually, "out"). For content-addressed store objects, `id`, is always "out". @@ -113,8 +124,8 @@ where Note that `id` = `"out"`, regardless of the name part of the store path. Also note that NAR + SHA-256 must not use this case, and instead must use the `type` = `"source:" ...` case. -[Nix Archive (NAR)]: @docroot@/glossary.md#gloss-NAR -[sha-256]: https://en.m.wikipedia.org/wiki/SHA-256 +[Nix Archive (NAR)]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive +[SHA-256]: https://en.m.wikipedia.org/wiki/SHA-256 ### Historical Note diff --git a/doc/manual/src/protocols/tarball-fetcher.md b/doc/manual/src/protocols/tarball-fetcher.md index 274fa6d63..5cff05d66 100644 --- a/doc/manual/src/protocols/tarball-fetcher.md +++ b/doc/manual/src/protocols/tarball-fetcher.md @@ -22,7 +22,7 @@ Link: ; rel="immutable" *flakeref* must be a tarball flakeref. It can contain the tarball flake attributes `narHash`, `rev`, `revCount` and `lastModified`. If `narHash` is included, its -value must be the NAR hash of the unpacked tarball (as computed via +value must be the [NAR hash][Nix Archive] of the unpacked tarball (as computed via `nix hash path`). Nix checks the contents of the returned tarball against the `narHash` attribute. The `rev` and `revCount` attributes are useful when the tarball flake is a mirror of a fetcher type that @@ -40,3 +40,31 @@ Link: ///archive/.tar.gz +``` + +> **Example** +> +> +> ```nix +> # flake.nix +> { +> inputs = { +> foo.url = "https://gitea.example.org/some-person/some-flake/archive/main.tar.gz"; +> bar.url = "https://gitea.example.org/some-other-person/other-flake/archive/442793d9ec0584f6a6e82fa253850c8085bb150a.tar.gz"; +> qux = { +> url = "https://forgejo.example.org/another-person/some-non-flake-repo/archive/development.tar.gz"; +> flake = false; +> }; +> }; +> outputs = { foo, bar, qux }: { /* ... */ }; +> } +``` + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive diff --git a/doc/manual/src/quick-start.md b/doc/manual/src/quick-start.md index 75853ced7..9eb7a3265 100644 --- a/doc/manual/src/quick-start.md +++ b/doc/manual/src/quick-start.md @@ -34,7 +34,7 @@ For more in-depth information you are kindly referred to subsequent chapters. lolcat: command not found ``` -1. Search for more packages on to try them out. +1. Search for more packages on [search.nixos.org](https://search.nixos.org/) to try them out. 1. Free up storage space: diff --git a/doc/manual/src/release-notes/index.md b/doc/manual/src/release-notes/index.md index cc805e631..d4e6292a6 100644 --- a/doc/manual/src/release-notes/index.md +++ b/doc/manual/src/release-notes/index.md @@ -1,12 +1,13 @@ # Nix Release Notes +The Nix release cycle is calendar-based as follows: + Nix has a release cycle of roughly 6 weeks. Notable changes and additions are announced in the release notes for each version. -Bugfixes can be backported on request to previous Nix releases. -We typically backport only as far back as the Nix version used in the latest NixOS release, which is announced in the [NixOS release notes](https://nixos.org/manual/nixos/stable/release-notes.html#ch-release-notes). - -Backports never skip releases. -If a feature is backported to version `x.y`, it must also be available in version `x.(y+1)`. -This ensures that upgrading from an older version with backports is still safe and no backported functionality will go missing. +The supported Nix versions are: +- The latest release +- The version used in the stable NixOS release, which is announced in the [NixOS release notes](https://nixos.org/manual/nixos/stable/release-notes.html#ch-release-notes). +Bugfixes and security issues are backported to every supported version. +Patch releases are published as needed. diff --git a/doc/manual/src/release-notes/rl-2.15.md b/doc/manual/src/release-notes/rl-2.15.md index 133121999..e7e52631b 100644 --- a/doc/manual/src/release-notes/rl-2.15.md +++ b/doc/manual/src/release-notes/rl-2.15.md @@ -11,7 +11,7 @@ As the choice of hash formats is no longer binary, the `--base16` flag is also added to explicitly specify the Base16 format, which is still the default. -* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](../glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents. +* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](@docroot@/glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents. The new `^` syntax for store paths introduced in Nix 2.13 allows explicitly referencing output paths of a derivation. Using this is better and more clear than relying on the now-removed `.drv` special handling. diff --git a/doc/manual/src/release-notes/rl-2.18.md b/doc/manual/src/release-notes/rl-2.18.md index 4bbc52b50..eb26fc9e7 100644 --- a/doc/manual/src/release-notes/rl-2.18.md +++ b/doc/manual/src/release-notes/rl-2.18.md @@ -13,7 +13,7 @@ - The `discard-references` feature has been stabilized. This means that the - [unsafeDiscardReferences](@docroot@/contributing/experimental-features.md#xp-feature-discard-references) + [unsafeDiscardReferences](@docroot@/development/experimental-features.md#xp-feature-discard-references) attribute is no longer guarded by an experimental flag and can be used freely. @@ -21,7 +21,7 @@ This only affects `nix-build --json` when "building" non-derivation things like fetched sources, which is a no-op. - A new builtin [`outputOf`](@docroot@/language/builtins.md#builtins-outputOf) has been added. - It is part of the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. + It is part of the [`dynamic-derivations`](@docroot@/development/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. - Flake follow paths at depths greater than 2 are now handled correctly, preventing "follows a non-existent input" errors. diff --git a/doc/manual/src/release-notes/rl-2.19.md b/doc/manual/src/release-notes/rl-2.19.md index ba6eb9c64..e2e2f85cc 100644 --- a/doc/manual/src/release-notes/rl-2.19.md +++ b/doc/manual/src/release-notes/rl-2.19.md @@ -17,8 +17,8 @@ - `nix-shell` shebang lines now support single-quoted arguments. -- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). - This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). +- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/development/experimental-features.md#xp-fetch-tree). + This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/development/experimental-features.md#xp-fetch-tree). - The interface for creating and updating lock files has been overhauled: @@ -33,7 +33,7 @@ - The flake-specific flags `--recreate-lock-file` and `--update-input` have been removed from all commands operating on installables. They are superceded by `nix flake update`. -- Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). +- Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches). - [`nix path-info --json`](@docroot@/command-ref/new-cli/nix3-path-info.md) (experimental) now returns a JSON map rather than JSON list. diff --git a/doc/manual/src/release-notes/rl-2.20.md b/doc/manual/src/release-notes/rl-2.20.md index 8ede168a4..eb724f600 100644 --- a/doc/manual/src/release-notes/rl-2.20.md +++ b/doc/manual/src/release-notes/rl-2.20.md @@ -200,3 +200,9 @@ while performing various operations (including `nix develop`, `nix flake update`, and so on). With several fixes to Nix's signal handlers, Nix commands will now exit quickly after Ctrl-C is pressed. + +- `nix copy` to a `ssh-ng` store now needs `--substitute-on-destination` (a.k.a. `-s`) + in order to substitute paths on the remote store instead of copying them. + The behavior is consistent with `nix copy` to a different kind of remote store. + Previously this behavior was controlled by the + `builders-use-substitutes` setting and `--substitute-on-destination` was ignored. diff --git a/doc/manual/src/release-notes/rl-2.21.md b/doc/manual/src/release-notes/rl-2.21.md new file mode 100644 index 000000000..75114f117 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.21.md @@ -0,0 +1,302 @@ +# Release 2.21.0 (2024-03-11) + +- Fix a fixed-output derivation sandbox escape (CVE-2024-27297) + + Cooperating Nix derivations could send file descriptors to files in the Nix + store to each other via Unix domain sockets in the abstract namespace. This + allowed one derivation to modify the output of the other derivation, after Nix + has registered the path as "valid" and immutable in the Nix database. + In particular, this allowed the output of fixed-output derivations to be + modified from their expected content. + + This isn't the case any more. + +- CLI options `--arg-from-file` and `--arg-from-stdin` [#10122](https://github.com/NixOS/nix/pull/10122) + + The new CLI option `--arg-from-file` *name* *path* passes the contents + of file *path* as a string value via the function argument *name* to a + Nix expression. Similarly, the new option `--arg-from-stdin` *name* + reads the contents of the string from standard input. + +- Concise error printing in `nix repl` [#9928](https://github.com/NixOS/nix/pull/9928) + + Previously, if an element of a list or attribute set threw an error while + evaluating, `nix repl` would print the entire error (including source location + information) inline. This output was clumsy and difficult to parse: + + ``` + nix-repl> { err = builtins.throw "uh oh!"; } + { err = «error: + … while calling the 'throw' builtin + at «string»:1:9: + 1| { err = builtins.throw "uh oh!"; } + | ^ + + error: uh oh!»; } + ``` + + Now, only the error message is displayed, making the output much more readable. + ``` + nix-repl> { err = builtins.throw "uh oh!"; } + { err = «error: uh oh!»; } + ``` + + However, if the whole expression being evaluated throws an error, source + locations and (if applicable) a stack trace are printed, just like you'd expect: + + ``` + nix-repl> builtins.throw "uh oh!" + error: + … while calling the 'throw' builtin + at «string»:1:1: + 1| builtins.throw "uh oh!" + | ^ + + error: uh oh! + ``` + +- `--debugger` can now access bindings from `let` expressions [#8827](https://github.com/NixOS/nix/issues/8827) [#9918](https://github.com/NixOS/nix/pull/9918) + + Breakpoints and errors in the bindings of a `let` expression can now access + those bindings in the debugger. Previously, only the body of `let` expressions + could access those bindings. + +- Enter the `--debugger` when `builtins.trace` is called if `debugger-on-trace` is set [#9914](https://github.com/NixOS/nix/pull/9914) + + If the `debugger-on-trace` option is set and `--debugger` is given, + `builtins.trace` calls will behave similarly to `builtins.break` and will enter + the debug REPL. This is useful for determining where warnings are being emitted + from. + +- Debugger prints source position information [#9913](https://github.com/NixOS/nix/pull/9913) + + The `--debugger` now prints source location information, instead of the + pointers of source location information. Before: + + ``` + nix-repl> :bt + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + 0x600001522598 + ``` + + After: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + /nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27 + + 131| + 132| bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs; + | ^ + 133| in + ``` + +- The `--debugger` will start more reliably in `let` expressions and function calls [#6649](https://github.com/NixOS/nix/issues/6649) [#9917](https://github.com/NixOS/nix/pull/9917) + + Previously, if you attempted to evaluate this file with the debugger: + + ```nix + let + a = builtins.trace "before inner break" ( + builtins.break "hello" + ); + b = builtins.trace "before outer break" ( + builtins.break a + ); + in + b + ``` + + Nix would correctly enter the debugger at `builtins.break a`, but if you asked + it to `:continue`, it would skip over the `builtins.break "hello"` expression + entirely. + + Now, Nix will correctly enter the debugger at both breakpoints. + +- Nested debuggers are no longer supported [#9920](https://github.com/NixOS/nix/pull/9920) + + Previously, evaluating an expression that throws an error in the debugger would + enter a second, nested debugger: + + ``` + nix-repl> builtins.throw "what" + error: what + + + Starting REPL to allow you to inspect the current state of the evaluator. + + Welcome to Nix 2.18.1. Type :? for help. + + nix-repl> + ``` + + Now, it just prints the error message like `nix repl`: + + ``` + nix-repl> builtins.throw "what" + error: + … while calling the 'throw' builtin + at «string»:1:1: + 1| builtins.throw "what" + | ^ + + error: what + ``` + +- Consistent order of function arguments in printed expressions [#9874](https://github.com/NixOS/nix/pull/9874) + + Function arguments are now printed in lexicographic order rather than the internal, creation-time based symbol order. + +- Fix duplicate attribute error positions for `inherit` [#9874](https://github.com/NixOS/nix/pull/9874) + + When an `inherit` caused a duplicate attribute error the position of the error was not reported correctly, placing the error with the inherit itself or at the start of the bindings block instead of the offending attribute name. + +- `inherit (x) ...` evaluates `x` only once [#9847](https://github.com/NixOS/nix/pull/9847) + + `inherit (x) a b ...` now evaluates the expression `x` only once for all inherited attributes rather than once for each inherited attribute. + This does not usually have a measurable impact, but side-effects (such as `builtins.trace`) would be duplicated and expensive expressions (such as derivations) could cause a measurable slowdown. + +- Store paths are allowed to start with `.` [#912](https://github.com/NixOS/nix/issues/912) [#9091](https://github.com/NixOS/nix/pull/9091) [#9095](https://github.com/NixOS/nix/pull/9095) [#9120](https://github.com/NixOS/nix/pull/9120) [#9121](https://github.com/NixOS/nix/pull/9121) [#9122](https://github.com/NixOS/nix/pull/9122) [#9130](https://github.com/NixOS/nix/pull/9130) [#9219](https://github.com/NixOS/nix/pull/9219) [#9224](https://github.com/NixOS/nix/pull/9224) [#9867](https://github.com/NixOS/nix/pull/9867) + + Leading periods were allowed by accident in Nix 2.4. The Nix team has considered this to be a bug, but this behavior has since been relied on by users, leading to unnecessary difficulties. + From now on, leading periods are supported. The names `.` and `..` are disallowed, as well as those starting with `.-` or `..-`. + + Nix versions that denied leading periods are documented [in the issue](https://github.com/NixOS/nix/issues/912#issuecomment-1919583286). + +- `nix repl` pretty-prints values [#9931](https://github.com/NixOS/nix/pull/9931) + + `nix repl` will now pretty-print values: + + ``` + { + attrs = { + a = { + b = { + c = { }; + }; + }; + }; + list = [ 1 ]; + list' = [ + 1 + 2 + 3 + ]; + } + ``` + +- Introduction of `--regex` and `--all` in `nix profile remove` and `nix profile upgrade` [#10166](https://github.com/NixOS/nix/pull/10166) + + Previously the command-line arguments for `nix profile remove` and `nix profile upgrade` matched the package entries using regular expression. + For instance: + + ``` + nix profile remove '.*vim.*' + ``` + + This would remove all packages that contain `vim` in their name. + + In most cases, only singular package names were used to remove and upgrade packages. Mixing this with regular expressions sometimes lead to unintended behavior. For instance, `python3.1` could match `python311`. + + To avoid unintended behavior, the arguments are now only matching exact names. + + Matching using regular expressions is still possible by using the new `--regex` flag: + + ``` + nix profile remove --regex '.*vim.*' + ``` + + One of the most useful cases for using regular expressions was to upgrade all packages. This was previously accomplished by: + + ``` + nix profile upgrade '.*' + ``` + + With the introduction of the `--all` flag, this now becomes more straightforward: + + ``` + nix profile upgrade --all + ``` + +- Visual clutter in `--debugger` is reduced [#9919](https://github.com/NixOS/nix/pull/9919) + + Before: + ``` + info: breakpoint reached + + + Starting REPL to allow you to inspect the current state of the evaluator. + + Welcome to Nix 2.20.0pre20231222_dirty. Type :? for help. + + nix-repl> :continue + error: uh oh + + + Starting REPL to allow you to inspect the current state of the evaluator. + + Welcome to Nix 2.20.0pre20231222_dirty. Type :? for help. + + nix-repl> + ``` + + After: + + ``` + info: breakpoint reached + + Nix 2.20.0pre20231222_dirty debugger + Type :? for help. + nix-repl> :continue + error: uh oh + + nix-repl> + ``` + +- Cycle detection in `nix repl` is simpler and more reliable [#8672](https://github.com/NixOS/nix/issues/8672) [#9926](https://github.com/NixOS/nix/pull/9926) + + The cycle detection in `nix repl`, `nix eval`, `builtins.trace`, and everywhere + else values are printed is now simpler and matches the cycle detection in + `nix-instantiate --eval` output. + + Before: + + ``` + nix eval --expr 'let self = { inherit self; }; in self' + { self = { self = «repeated»; }; } + ``` + + After: + + ``` + { self = «repeated»; } + ``` + +- In the debugger, `while evaluating the attribute` errors now include position information [#9915](https://github.com/NixOS/nix/pull/9915) + + Before: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + 0x600001522598 + ``` + + After: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + /nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27 + + 131| + 132| bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs; + | ^ + 133| in + ``` + +- Stack size is increased on macOS [#9860](https://github.com/NixOS/nix/pull/9860) + + Previously, Nix would set the stack size to 64MiB on Linux, but would leave the + stack size set to the default (approximately 8KiB) on macOS. Now, the stack + size is correctly set to 64MiB on macOS as well, which should reduce stack + overflow segfaults in deeply-recursive Nix expressions. + diff --git a/doc/manual/src/release-notes/rl-2.22.md b/doc/manual/src/release-notes/rl-2.22.md new file mode 100644 index 000000000..c78d3d692 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.22.md @@ -0,0 +1,21 @@ +# Release 2.22.0 (2024-04-23) + +### Significant changes + +- Remove experimental repl-flake [#10103](https://github.com/NixOS/nix/issues/10103) [#10299](https://github.com/NixOS/nix/pull/10299) + + The `repl-flake` experimental feature has been removed. The `nix repl` command now works like the rest of the new CLI in that `nix repl {path}` now tries to load a flake at `{path}` (or fails if the `flakes` experimental feature isn't enabled). + +### Other changes + +- `nix eval` prints derivations as `.drv` paths [#10200](https://github.com/NixOS/nix/pull/10200) + + `nix eval` will now print derivations as their `.drv` paths, rather than as + attribute sets. This makes commands like `nix eval nixpkgs#bash` terminate + instead of infinitely looping into recursive self-referential attributes: + + ```ShellSession + $ nix eval nixpkgs#bash + «derivation /nix/store/m32cbgbd598f4w299g0hwyv7gbw6rqcg-bash-5.2p26.drv» + ``` + diff --git a/doc/manual/src/release-notes/rl-2.23.md b/doc/manual/src/release-notes/rl-2.23.md new file mode 100644 index 000000000..ac842fdc0 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.23.md @@ -0,0 +1,102 @@ +# Release 2.23.0 (2024-06-03) + +- New builtin: `builtins.warn` [#306026](https://github.com/NixOS/nix/issues/306026) [#10592](https://github.com/NixOS/nix/pull/10592) + + `builtins.warn` behaves like `builtins.trace "warning: ${msg}"`, has an accurate log level, and is controlled by the options + [`debugger-on-trace`](@docroot@/command-ref/conf-file.md#conf-debugger-on-trace), + [`debugger-on-warn`](@docroot@/command-ref/conf-file.md#conf-debugger-on-warn) and + [`abort-on-warn`](@docroot@/command-ref/conf-file.md#conf-abort-on-warn). + +- Make `nix build --keep-going` consistent with `nix-build --keep-going` + + This means that if e.g. multiple fixed-output derivations fail to + build, all hash mismatches are displayed. + +- Modify `nix derivation {add,show}` JSON format [#9866](https://github.com/NixOS/nix/issues/9866) [#10722](https://github.com/NixOS/nix/pull/10722) + + The JSON format for derivations has been slightly revised to better conform to our [JSON guidelines](@docroot@/development/cli-guideline.md#returning-future-proof-json). + In particular, the hash algorithm and content addressing method of content-addresed derivation outputs are now separated into two fields `hashAlgo` and `method`, + rather than one field with an arcane `:`-separated format. + + This JSON format is only used by the experimental `nix derivation` family of commands, at this time. + Future revisions are expected as the JSON format is still not entirely in compliance even after these changes. + +- Warn on unknown settings anywhere in the command line [#10701](https://github.com/NixOS/nix/pull/10701) + + All `nix` commands will now properly warn when an unknown option is specified anywhere in the command line. + + Before: + + ```console + $ nix-instantiate --option foobar baz --expr '{}' + warning: unknown setting 'foobar' + $ nix-instantiate '{}' --option foobar baz --expr + $ nix eval --expr '{}' --option foobar baz + { } + ``` + + After: + + ```console + $ nix-instantiate --option foobar baz --expr '{}' + warning: unknown setting 'foobar' + $ nix-instantiate '{}' --option foobar baz --expr + warning: unknown setting 'foobar' + $ nix eval --expr '{}' --option foobar baz + warning: unknown setting 'foobar' + { } + ``` + +- `nix env shell` is the new `nix shell`, and `nix shell` remains an accepted alias [#10504](https://github.com/NixOS/nix/issues/10504) [#10807](https://github.com/NixOS/nix/pull/10807) + + This is part of an effort to bring more structure to the CLI subcommands. + + `nix env` will be about the process environment. + Future commands may include `nix env run` and `nix env print-env`. + + It is also somewhat analogous to the [planned](https://github.com/NixOS/nix/issues/10504) `nix dev shell` (currently `nix develop`), which is less about environment variables, and more about running a development shell, which is a more powerful command, but also requires more setup. + +- Flake operations that expect derivations now print the failing value and its type [#10778](https://github.com/NixOS/nix/pull/10778) + + In errors like `flake output attribute 'nixosConfigurations.yuki.config' is not a derivation or path`, the message now includes the failing value and type. + + Before: + + ``` + error: flake output attribute 'nixosConfigurations.yuki.config' is not a derivation or path + ```` + + After: + + ``` + error: expected flake output attribute 'nixosConfigurations.yuki.config' to be a derivation or path but found a set: { appstream = «thunk»; assertions = «thunk»; boot = { bcache = «thunk»; binfmt = «thunk»; binfmtMiscRegistrations = «thunk»; blacklistedKernelModules = «thunk»; bootMount = «thunk»; bootspec = «thunk»; cleanTmpDir = «thunk»; consoleLogLevel = «thunk»; «43 attributes elided» }; «48 attributes elided» } + ``` + +- `fetchTree` now fetches Git repositories shallowly by default [#10028](https://github.com/NixOS/nix/pull/10028) + + `builtins.fetchTree` now clones Git repositories shallowly by default, which reduces network traffic and disk usage significantly in many cases. + + Previously, the default behavior was to clone the full history of a specific tag or branch (e.g. `ref`) and only afterwards extract the files of one specific revision. + + From now on, the `ref` and `allRefs` arguments will be ignored, except if shallow cloning is disabled by setting `shallow = false`. + + The defaults for `builtins.fetchGit` remain unchanged. Here, shallow cloning has to be enabled manually by passing `shallow = true`. + +- Store object info JSON format now uses `null` rather than omitting fields [#9995](https://github.com/NixOS/nix/pull/9995) + + The [store object info JSON format](@docroot@/protocols/json/store-object-info.md), used for e.g. `nix path-info`, no longer omits fields to indicate absent information, but instead includes the fields with a `null` value. + For example, `"ca": null` is used to to indicate a store object that isn't content-addressed rather than omitting the `ca` field entirely. + This makes records of this sort more self-describing, and easier to consume programmatically. + + We will follow this design principle going forward; + the [JSON guidelines](@docroot@/development/json-guideline.md) in the contributing section have been updated accordingly. + +- Large path warnings [#10661](https://github.com/NixOS/nix/pull/10661) + + Nix can now warn when evaluation of a Nix expression causes a large + path to be copied to the Nix store. The threshold for this warning can + be configured using [the `warn-large-path-threshold` + setting](@docroot@/command-ref/conf-file.md#warn-large-path-threshold), + e.g. `--warn-large-path-threshold 100M` will warn about paths larger + than 100 MiB. + diff --git a/doc/manual/src/release-notes/rl-2.24.md b/doc/manual/src/release-notes/rl-2.24.md new file mode 100644 index 000000000..5bcc1d79c --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.24.md @@ -0,0 +1,324 @@ +# Release 2.24.0 (2024-07-31) + +### Significant changes + +- Harden user sandboxing + + The build directory has been hardened against interference with the outside world by nesting it inside another directory owned by (and only readable by) the daemon user. + + This is a low severity security fix, [CVE-2024-38531](https://www.cve.org/CVERecord?id=CVE-2024-38531). + + Credit: [**@alois31**](https://github.com/alois31), [**Linus Heckemann (@lheckemann)**](https://github.com/lheckemann) + Co-authors: [**@edolstra**](https://github.com/edolstra) + +- `nix-shell ` looks for `shell.nix` [#496](https://github.com/NixOS/nix/issues/496) [#2279](https://github.com/NixOS/nix/issues/2279) [#4529](https://github.com/NixOS/nix/issues/4529) [#5431](https://github.com/NixOS/nix/issues/5431) [#11053](https://github.com/NixOS/nix/issues/11053) [#11057](https://github.com/NixOS/nix/pull/11057) + + `nix-shell $x` now looks for `$x/shell.nix` when `$x` resolves to a directory. + + Although this might be seen as a breaking change, its primarily interactive usage makes it a minor issue. + This adjustment addresses a commonly reported problem. + + This also applies to `nix-shell` shebang scripts. Consider the following example: + + ```shell + #!/usr/bin/env nix-shell + #!nix-shell -i bash + ``` + + This will now load `shell.nix` from the script's directory, if it exists; `default.nix` otherwise. + + The old behavior can be opted into by setting the option [`nix-shell-always-looks-for-shell-nix`](@docroot@/command-ref/conf-file.md#conf-nix-shell-always-looks-for-shell-nix) to `false`. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + +- `nix-repl`'s `:doc` shows documentation comments [#3904](https://github.com/NixOS/nix/issues/3904) [#10771](https://github.com/NixOS/nix/issues/10771) [#1652](https://github.com/NixOS/nix/pull/1652) [#9054](https://github.com/NixOS/nix/pull/9054) [#11072](https://github.com/NixOS/nix/pull/11072) + + `nix repl` has a `:doc` command that previously only rendered documentation for internally defined functions. + This feature has been extended to also render function documentation comments, in accordance with [RFC 145]. + + Example: + + ``` + nix-repl> :doc lib.toFunction + Function toFunction + … defined at /home/user/h/nixpkgs/lib/trivial.nix:1072:5 + + Turns any non-callable values into constant functions. Returns + callable values as is. + + Inputs + + v + + : Any value + + Examples + + :::{.example} + + ## lib.trivial.toFunction usage example + + | nix-repl> lib.toFunction 1 2 + | 1 + | + | nix-repl> lib.toFunction (x: x + 1) 2 + | 3 + + ::: + ``` + + Known limitations: + - It does not render documentation for "formals", such as `{ /** the value to return */ x, ... }: x`. + - Some extensions to markdown are not yet supported, as you can see in the example above. + + We'd like to acknowledge [Yingchi Long (@inclyc)](https://github.com/inclyc) for proposing a proof of concept for this functionality in [#9054](https://github.com/NixOS/nix/pull/9054), as well as [@sternenseemann](https://github.com/sternenseemann) and [Johannes Kirschbauer (@hsjobeki)](https://github.com/hsjobeki) for their contributions, proposals, and their work on [RFC 145]. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + + [RFC 145]: https://github.com/NixOS/rfcs/pull/145 + +### Other changes + +- Solve `cached failure of attribute X` [#9165](https://github.com/NixOS/nix/issues/9165) [#10513](https://github.com/NixOS/nix/issues/10513) [#10564](https://github.com/NixOS/nix/pull/10564) + + This eliminates all "cached failure of attribute X" messages by forcing evaluation of the original value when needed to show the exception to the user. This enhancement improves error reporting by providing the underlying message and stack trace. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Run the flake regressions test suite [#10603](https://github.com/NixOS/nix/pull/10603) + + This update introduces a GitHub action to run a subset of the [flake regressions test suite](https://github.com/NixOS/flake-regressions), which includes 259 flakes with their expected evaluation results. Currently, the action runs the first 25 flakes due to the full test suite's extensive runtime. A manually triggered action may be implemented later to run the entire test suite. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Support unit prefixes in configuration settings [#10668](https://github.com/NixOS/nix/pull/10668) + + Configuration settings in Nix now support unit prefixes, allowing for more intuitive and readable configurations. For example, you can now specify [`--min-free 1G`](@docroot@/command-ref/opt-common.md#opt-min-free) to set the minimum free space to 1 gigabyte. + + This enhancement was extracted from [#7851](https://github.com/NixOS/nix/pull/7851) and is also useful for PR [#10661](https://github.com/NixOS/nix/pull/10661). + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- `nix build`: show all FOD errors with `--keep-going` [#10734](https://github.com/NixOS/nix/pull/10734) + + The [`nix build`](@docroot@/command-ref/new-cli/nix3-build.md) command has been updated to improve the behavior of the [`--keep-going`] flag. Now, when `--keep-going` is used, all hash-mismatch errors of failing fixed-output derivations (FODs) are displayed, similar to the behavior for other build failures. This enhancement ensures that all relevant build errors are shown, making it easier for users to update multiple derivations at once or to diagnose and fix issues. + + Author: [**Jörg Thalheim (@Mic92)**](https://github.com/Mic92), [**Maximilian Bosch (@Ma27)**](https://github.com/Ma27) + + [`--keep-going`](@docroot@/command-ref/opt-common.md#opt-keep-going) + +- Build with Meson [#2503](https://github.com/NixOS/nix/issues/2503) [#10378](https://github.com/NixOS/nix/pull/10378) [#10855](https://github.com/NixOS/nix/pull/10855) [#10904](https://github.com/NixOS/nix/pull/10904) [#10908](https://github.com/NixOS/nix/pull/10908) [#10914](https://github.com/NixOS/nix/pull/10914) [#10933](https://github.com/NixOS/nix/pull/10933) [#10936](https://github.com/NixOS/nix/pull/10936) [#10954](https://github.com/NixOS/nix/pull/10954) [#10955](https://github.com/NixOS/nix/pull/10955) [#10963](https://github.com/NixOS/nix/pull/10963) [#10967](https://github.com/NixOS/nix/pull/10967) [#10973](https://github.com/NixOS/nix/pull/10973) [#11034](https://github.com/NixOS/nix/pull/11034) [#11054](https://github.com/NixOS/nix/pull/11054) [#11055](https://github.com/NixOS/nix/pull/11055) [#11060](https://github.com/NixOS/nix/pull/11060) [#11064](https://github.com/NixOS/nix/pull/11064) [#11155](https://github.com/NixOS/nix/pull/11155) + + These changes aim to replace the use of autotools and `make` with Meson for building various components of Nix. Additionally, each library is built in its own derivation, leveraging Meson's "subprojects" feature to allow a single development shell for building all libraries while also supporting separate builds. This approach aims to improve productivity and build modularity, compared to both make and a monolithic Meson-based derivation. + + Special thanks to everyone who has contributed to the Meson port, particularly [**@p01arst0rm**](https://github.com/p01arst0rm) and [**@Qyriad**](https://github.com/Qyriad). + + Authors: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314), [**Tom Bereknyei**](https://github.com/tomberek), [**Théophane Hufschmitt (@thufschmitt)**](https://github.com/thufschmitt), [**Valentin Gagarin (@fricklerhandwerk)**](https://github.com/fricklerhandwerk), [**Robert Hensing (@roberth)**](https://github.com/roberth) + Co-authors: [**@p01arst0rm**](https://github.com/p01arst0rm), [**@Qyriad**](https://github.com/Qyriad) + +- Evaluation cache: fix cache regressions [#10570](https://github.com/NixOS/nix/issues/10570) [#11086](https://github.com/NixOS/nix/pull/11086) + + This update addresses two bugs in the evaluation cache system: + + 1. Regression in #10570: The evaluation cache was not being persisted in `nix develop`. + 2. Nix could sometimes try to commit the evaluation cache SQLite transaction without there being an active transaction, resulting in non-error errors being printed. + + Author: [**Lexi Mattick (@kognise)**](https://github.com/kognise) + +- Introduce `libnixflake` [#9063](https://github.com/NixOS/nix/pull/9063) + + A new library, `libnixflake`, has been introduced to better separate the Flakes layer within Nix. This change refactors the codebase to encapsulate Flakes-specific functionality within its own library. + + See the commits in the pull request for detailed changes, with the only significant code modifications happening in the initial commit. + + This change was alluded to in [RFC 134](https://github.com/nixos/rfcs/blob/master/rfcs/0134-nix-store-layer.md) and is a step towards a more modular and maintainable codebase. + + Author: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314) + +- CLI options `--arg-from-file` and `--arg-from-stdin` [#9913](https://github.com/NixOS/nix/pull/9913) + +- The `--debugger` now prints source location information, instead of the + pointers of source location information. Before: + + ``` + nix-repl> :bt + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + 0x600001522598 + ``` + + After: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + /nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27 + + 131| + 132| bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs; + | ^ + 133| in + ``` + +- Stop vendoring `toml11` + + We don't apply any patches to it, and vendoring it locks users into + bugs (it hasn't been updated since its introduction in late 2021). + + Author: [**Winter (@winterqt)**](https://github.com/winterqt) + +- Rename hash format `base32` to `nix32` [#8678](https://github.com/NixOS/nix/pull/8678) + + Hash format `base32` was renamed to `nix32` since it used a special nix-specific character set for + [Base32](https://en.wikipedia.org/wiki/Base32). + + **Deprecation**: Use `nix32` instead of `base32` as `toHashFormat` + + For the builtin `convertHash`, the `toHashFormat` parameter now accepts the same hash formats as the `--to`/`--from` + parameters of the `nix hash conert` command: `"base16"`, `"nix32"`, `"base64"`, and `"sri"`. The former `"base32"` value + remains as a deprecated alias for `"nix32"`. Please convert your code from: + + ```nix + builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base32";} + ``` + + to + + ```nix + builtins.convertHash { inherit hash hashAlgo; toHashFormat = "nix32";} + ``` + +- Add `pipe-operators` experimental feature [#11131](https://github.com/NixOS/nix/pull/11131) + + This is a draft implementation of [RFC 0148](https://github.com/NixOS/rfcs/pull/148). + + The `pipe-operators` experimental feature adds [`<|` and `|>` operators][pipe operators] to the Nix language. + *a* `|>` *b* is equivalent to the function application *b* *a*, and + *a* `<|` *b* is equivalent to the function application *a* *b*. + + For example: + + ``` + nix-repl> 1 |> builtins.add 2 |> builtins.mul 3 + 9 + + nix-repl> builtins.add 1 <| builtins.mul 2 <| 3 + 7 + ``` + + `<|` and `|>` are right and left associative, respectively, and have lower precedence than any other operator. + These properties may change in future releases. + + See [the RFC](https://github.com/NixOS/rfcs/pull/148) for more examples and rationale. + + [pipe operators]: @docroot@/language/operators.md#pipe-operators + +- `nix-shell` shebang uses relative path [#4232](https://github.com/NixOS/nix/issues/4232) [#5088](https://github.com/NixOS/nix/pull/5088) [#11058](https://github.com/NixOS/nix/pull/11058) + + + Relative [path](@docroot@/language/types.md#type-path) literals in `nix-shell` shebang scripts' options are now resolved relative to the [script's location](@docroot@/glossary.md?highlight=base%20directory#gloss-base-directory). + Previously they were resolved relative to the current working directory. + + For example, consider the following script in `~/myproject/say-hi`: + + ```shell + #!/usr/bin/env nix-shell + #!nix-shell --expr 'import ./shell.nix' + #!nix-shell --arg toolset './greeting-tools.nix' + #!nix-shell -i bash + hello + ``` + + Older versions of `nix-shell` would resolve `shell.nix` relative to the current working directory, such as the user's home directory in this example: + + ```console + [hostname:~]$ ./myproject/say-hi + error: + … while calling the 'import' builtin + at «string»:1:2: + 1| (import ./shell.nix) + | ^ + + error: path '/home/user/shell.nix' does not exist + ``` + + Since this release, `nix-shell` resolves `shell.nix` relative to the script's location, and `~/myproject/shell.nix` is used. + + ```console + $ ./myproject/say-hi + Hello, world! + ``` + + **Opt-out** + + This is technically a breaking change, so we have added an option so you can adapt independently of your Nix update. + The old behavior can be opted into by setting the option [`nix-shell-shebang-arguments-relative-to-script`](@docroot@/command-ref/conf-file.md#conf-nix-shell-shebang-arguments-relative-to-script) to `false`. + This option will be removed in a future release. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + +- Improve handling of tarballs that don't consist of a single top-level directory [#11195](https://github.com/NixOS/nix/pull/11195) + + In previous Nix releases, the tarball fetcher (used by `builtins.fetchTarball`) erroneously merged top-level directories into a single directory, and silently discarded top-level files that are not directories. This is no longer the case. The new behaviour is that *only* if the tarball consists of a single directory, the top-level path component of the files in the tarball is removed (similar to `tar`'s `--strip-components=1`). + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Improve handling of tarballs that don't consist of a single top-level directory [#11195](https://github.com/NixOS/nix/pull/11195) + + In previous Nix releases, the tarball fetcher (used by `builtins.fetchTarball`) erroneously merged top-level directories into a single directory, and silently discarded top-level files that are not directories. This is no longer the case. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Setting to warn about large paths [#10778](https://github.com/NixOS/nix/pull/10778) + + Nix can now warn when evaluation of a Nix expression causes a large + path to be copied to the Nix store. The threshold for this warning can + be configured using the `warn-large-path-threshold` setting, + e.g. `--warn-large-path-threshold 100M`. + + +# Contributors + +This release was made possible by the following 43 contributors: + +- Andreas Rammhold [**(@andir)**](https://github.com/andir) +- Andrew Marshall [**(@amarshall)**](https://github.com/amarshall) +- Brian McKenna [**(@puffnfresh)**](https://github.com/puffnfresh) +- Cameron [**(@SkamDart)**](https://github.com/SkamDart) +- Cole Helbling [**(@cole-h)**](https://github.com/cole-h) +- Corbin Simpson [**(@MostAwesomeDude)**](https://github.com/MostAwesomeDude) +- Eelco Dolstra [**(@edolstra)**](https://github.com/edolstra) +- Emily [**(@emilazy)**](https://github.com/emilazy) +- Enno Richter [**(@elohmeier)**](https://github.com/elohmeier) +- Farid Zakaria [**(@fzakaria)**](https://github.com/fzakaria) +- HaeNoe [**(@haenoe)**](https://github.com/haenoe) +- Hamir Mahal [**(@hamirmahal)**](https://github.com/hamirmahal) +- Harmen [**(@alicebob)**](https://github.com/alicebob) +- Ivan Trubach [**(@tie)**](https://github.com/tie) +- Jared Baur [**(@jmbaur)**](https://github.com/jmbaur) +- John Ericson [**(@Ericson2314)**](https://github.com/Ericson2314) +- Jonathan De Troye [**(@detroyejr)**](https://github.com/detroyejr) +- Jörg Thalheim [**(@Mic92)**](https://github.com/Mic92) +- Klemens Nanni [**(@klemensn)**](https://github.com/klemensn) +- Las Safin [**(@L-as)**](https://github.com/L-as) +- Lexi Mattick [**(@kognise)**](https://github.com/kognise) +- Matthew Bauer [**(@matthewbauer)**](https://github.com/matthewbauer) +- Max “Goldstein†Siling [**(@GoldsteinE)**](https://github.com/GoldsteinE) +- Mingye Wang [**(@Artoria2e5)**](https://github.com/Artoria2e5) +- Philip Taron [**(@philiptaron)**](https://github.com/philiptaron) +- Pierre Bourdon [**(@delroth)**](https://github.com/delroth) +- Pino Toscano [**(@pinotree)**](https://github.com/pinotree) +- RTUnreal [**(@RTUnreal)**](https://github.com/RTUnreal) +- Robert Hensing [**(@roberth)**](https://github.com/roberth) +- Romain Neil [**(@romain-neil)**](https://github.com/romain-neil) +- Ryan Hendrickson [**(@rhendric)**](https://github.com/rhendric) +- Sergei Trofimovich [**(@trofi)**](https://github.com/trofi) +- Shogo Takata [**(@pineapplehunter)**](https://github.com/pineapplehunter) +- Siddhant Kumar [**(@siddhantk232)**](https://github.com/siddhantk232) +- Silvan Mosberger [**(@infinisil)**](https://github.com/infinisil) +- Théophane Hufschmitt [**(@thufschmitt)**](https://github.com/thufschmitt) +- Valentin Gagarin [**(@fricklerhandwerk)**](https://github.com/fricklerhandwerk) +- Winter [**(@winterqt)**](https://github.com/winterqt) +- jade [**(@lf-)**](https://github.com/lf-) +- kirillrdy [**(@kirillrdy)**](https://github.com/kirillrdy) +- pennae [**(@pennae)**](https://github.com/pennae) +- poweredbypie [**(@poweredbypie)**](https://github.com/poweredbypie) +- tomberek [**(@tomberek)**](https://github.com/tomberek) diff --git a/doc/manual/src/release-notes/rl-2.4.md b/doc/manual/src/release-notes/rl-2.4.md index 8b566fc7b..1201e53b6 100644 --- a/doc/manual/src/release-notes/rl-2.4.md +++ b/doc/manual/src/release-notes/rl-2.4.md @@ -23,7 +23,7 @@ more than 2800 commits from 195 contributors since release 2.3. * The **`nix` command** has seen a lot of work and is now almost at feature parity with the old command-line interface (the `nix-*` commands). It aims to be [more modern, consistent and pleasant to - use](../contributing/cli-guideline.md) than the old CLI. It is still + use](../development/cli-guideline.md) than the old CLI. It is still marked as experimental but its interface should not change much anymore in future releases. diff --git a/doc/manual/src/store/file-system-object/content-address.md b/doc/manual/src/store/file-system-object/content-address.md new file mode 100644 index 000000000..410d7fb7c --- /dev/null +++ b/doc/manual/src/store/file-system-object/content-address.md @@ -0,0 +1,85 @@ +# Content-Addressing File System Objects + +For many operations, Nix needs to calculate [a content addresses](@docroot@/glossary.md#gloss-content-address) of [a file system object][file system object]. +Usually this is needed as part of +[content addressing store objects](../store-object/content-address.md), +since store objects always have a root file system object. +But some command-line utilities also just work on "raw" file system objects, not part of any store object. + +Every content addressing scheme Nix uses ultimately involves feeding data into a [hash function](https://en.wikipedia.org/wiki/Hash_function), and getting back an opaque fixed-size digest which is deemed a content address. +The various *methods* of content addressing thus differ in how abstract data (in this case, a file system object and its descendents) are fed into the hash function. + +## Serialising File System Objects { #serial } + +The simplest method is to serialise the entire file system object tree into a single binary string, and then hash that binary string, yielding the content address. +In this section we describe the currently-supported methods of serialising file system objects. + +### Flat { #serial-flat } + +A single file object can just be hashed by its contents. +This is not enough information to encode the fact that the file system object is a file, +but if we *already* know that the FSO is a single non-executable file by other means, it is sufficient. + +Because the hashed data is just the raw file, as is, this choice is good for compatibility with other systems. +For example, Unix commands like `sha256sum` or `sha1sum` will produce hashes for single files that match this. + +### Nix Archive (NAR) { #serial-nix-archive } + +For the other cases of [file system objects][file system object], especially directories with arbitrary descendents, we need a more complex serialisation format. +Examples of such serialisations are the ZIP and TAR file formats. +However, for our purposes these formats have two problems: + +- They do not have a canonical serialisation, meaning that given an FSO, there can +be many different serialisations. + For instance, TAR files can have variable amounts of padding between archive members; + and some archive formats leave the order of directory entries undefined. + This would be bad because we use serialisation to compute cryptographic hashes over file system objects, and for those hashes to be useful as a content address or for integrity checking, uniqueness is crucial. + Otherwise, correct hashes would report false mismatches, and the store would fail to find the content. + +- They store more information than we have in our notion of FSOs, such as time stamps. + This can cause FSOs that Nix should consider equal to hash to different values on different machines, just because the dates differ. + +- As a practical consideration, the TAR format is the only truly universal format in the Unix environment. + It has many problems, such as an inability to deal with long file names and files larger than 2^33 bytes. + Current implementations such as GNU Tar work around these limitations in various ways. + +For these reasons, Nix has its very own archive format—the Nix Archive (NAR) format, +which is carefully designed to avoid the problems described above. + +The exact specification of the Nix Archive format is in `protocols/nix-archive.md` + +## Content addressing File System Objects beyond a single serialisation pass + +Serialising the entire tree and then hashing that binary string is not the only option for content addressing, however. +Another technique is that of a [Merkle graph](https://en.wikipedia.org/wiki/Merkle_tree), where previously computed hashes are included in subsequent byte strings to be hashed. + +In particular, the Merkle graphs can match the original graph structure of file system objects: +we can first hash (serialised) child file system objects, and then hash parent objects using the hashes of their children in the serialisation (to be hashed) of the parent file system objects. + +Currently, there is one such Merkle DAG content addressing method supported. + +### Git ([experimental][xp-feature-git-hashing]) { #git } + +> **Warning** +> +> This method is part of the [`git-hashing`][xp-feature-git-hashing] experimental feature. + +Git's file system model is very close to Nix's, and so Git's content addressing method is a pretty good fit. +Just as with regular Git, files and symlinks are hashed as git "blobs", and directories are hashed as git "trees". + +However, one difference between Nix's and Git's file system model needs special treatment. +Plain files, executable files, and symlinks are not differentiated as distinctly addressable objects, but by their context: by the directory entry that refers to them. +That means so long as the root object is a directory, there is no problem: +every non-directory object is owned by a parent directory, and the entry that refers to it provides the missing information. +However, if the root object is not a directory, then we have no way of knowing which one of an executable file, non-executable file, or symlink it is supposed to be. + +In response to this, we have decided to treat a bare file as non-executable file. +This is similar to do what we do with [flat serialisation](#serial-flat), which also lacks this information. +To avoid an address collision, attempts to hash a bare executable file or symlink will result in an error (just as would happen for flat serialisation also). +Thus, Git can encode some, but not all of Nix's "File System Objects", and this sort of content-addressing is likewise partial. + +In the future, we may support a Git-like hash for such file system objects, or we may adopt another Merkle DAG format which is capable of representing all Nix file system objects. + +[file system object]: ../file-system-object.md +[store object]: ../store-object.md +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/doc/manual/src/store/store-object/content-address.md b/doc/manual/src/store/store-object/content-address.md new file mode 100644 index 000000000..02dce2836 --- /dev/null +++ b/doc/manual/src/store/store-object/content-address.md @@ -0,0 +1,95 @@ +# Content-Addressing Store Objects + +Just [like][fso-ca] [File System Objects][File System Object], +[Store Objects][Store Object] can also be [content-addressed](@docroot@/glossary.md#gloss-content-addressed), +unless they are [input-addressed](@docroot@/glossary.md#gloss-input-addressed-store-object). + +For store objects, the content address we produce will take the form of a [Store Path] rather than regular hash. +In particular, the content-addressing scheme will ensure that the digest of the store path is solely computed from the + +- file system object graph (the root one and its children, if it has any) +- references +- [store directory](../store-path.md#store-directory) +- name + +of the store object, and not any other information, which would not be an intrinsic property of that store object. + +For the full specification of the algorithms involved, see the [specification of store path digests][sp-spec]. + +[File System Object]: ../file-system-object.md +[Store Object]: ../store-object.md +[Store Path]: ../store-path.md + +## Content addressing each part of a store object + +### File System Objects + +With all currently supported store object content addressing methods, the file system object is always [content-addressed][fso-ca] first, and then that hash is incorporated into content address computation for the store object. + +### References + +With all currently supported store object content addressing methods, +other objects are referred to by their regular (string-encoded-) [store paths][Store Path]. + +Self-references however cannot be referred to by their path, because we are in the midst of describing how to compute that path! + +> The alternative would require finding as hash function fixed point, i.e. the solution to an equation in the form +> ``` +> digest = hash(..... || digest || ....) +> ``` +> which is computationally infeasible. +> As far as we know, this is equivalent to finding a hash collision. + +Instead we just have a "has self reference" boolean, which will end up affecting the digest. + +### Name and Store Directory + +These two items affect the digest in a way that is standard for store path digest computations and not specific to content-addressing. +Consult the [specification of store path digests][sp-spec] for further details. + +## Content addressing Methods + +For historical reasons, we don't support all features in all combinations. +Each currently supported method of content addressing chooses a single method of file system object hashing, and may offer some restrictions on references. +The names and store directories are unrestricted however. + +### Flat { #method-flat } + +This uses the corresponding [Flat](../file-system-object/content-address.md#serial-flat) method of file system object content addressing. + +References are not supported: store objects with flat hashing *and* references can not be created. + +### Text { #method-text } + +This also uses the corresponding [Flat](../file-system-object/content-address.md#serial-flat) method of file system object content addressing. + +References to other store objects are supported, but self references are not. + +This is the only store-object content-addressing method that is not named identically with a corresponding file system object method. +It is somewhat obscure, mainly used for "drv files" +(derivations serialized as store objects in their ["ATerm" file format](@docroot@/protocols/derivation-aterm.md)). +Prefer another method if possible. + +### Nix Archive { #method-nix-archive } + +This uses the corresponding [Nix Archive](../file-system-object/content-address.md#serial-nix-archive) method of file system object content addressing. + +References (to other store objects and self references alike) are supported so long as the hash algorithm is SHA-256, but not (neither kind) otherwise. + +### Git { #method-git } + +> **Warning** +> +> This method is part of the [`git-hashing`][xp-feature-git-hashing] experimental feature. + +This uses the corresponding [Git](../file-system-object/content-address.md#serial-git) method of file system object content addressing. + +References are not supported. + +Only SHA-1 is supported at this time. +If [SHA-256-based Git](https://git-scm.com/docs/hash-function-transition) +becomes more widespread, this restriction will be revisited. + +[fso-ca]: ../file-system-object/content-address.md +[sp-spec]: @docroot@/protocols/store-path.md +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/doc/manual/src/store/store-path.md b/doc/manual/src/store/store-path.md index b5ad0c654..beec2389b 100644 --- a/doc/manual/src/store/store-path.md +++ b/doc/manual/src/store/store-path.md @@ -1,5 +1,11 @@ # Store Path +> **Example** +> +> `/nix/store/a040m110amc4h71lds2jmr8qrkj2jhxd-git-2.38.1` +> +> A rendered store path + Nix implements references to [store objects](./index.md#store-object) as *store paths*. Think of a store path as an [opaque], [unique identifier]: @@ -37,6 +43,10 @@ A store path is rendered to a file system path as the concatenation of > store directory digest name > ``` +Exactly how the digest is calculated depends on the type of store path. +Store path digests are *supposed* to be opaque, and so for most operations, it is not necessary to know the details. +That said, the manual has a full [specification of store path digests](@docroot@/protocols/store-path.md). + ## Store Directory Every [Nix store](./index.md) has a store directory. @@ -46,7 +56,7 @@ But if the store has a file system representation, the store directory contains [file system objects]: ./file-system-object.md -This means a store path is not just derived from the referenced store object itself, but depends on the store the store object is in. +This means a store path is not just derived from the referenced store object itself, but depends on the store that the store object is in. > **Note** > diff --git a/flake.lock b/flake.lock index bb2e400c0..2ac413a69 100644 --- a/flake.lock +++ b/flake.lock @@ -3,11 +3,11 @@ "flake-compat": { "flake": false, "locked": { - "lastModified": 1673956053, - "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", "owner": "edolstra", "repo": "flake-compat", - "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", "type": "github" }, "original": { @@ -16,38 +16,100 @@ "type": "github" } }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1719994518, + "narHash": "sha256-pQMhCCHyQGRzdfAkdJ4cIWiw+JNuWsTX7f0ZYSyz0VY=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "9227223f6d922fee3c7b190b2cc238a99527bbb7", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "git-hooks-nix": { + "inputs": { + "flake-compat": [], + "gitignore": [], + "nixpkgs": [ + "nixpkgs" + ], + "nixpkgs-stable": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1721042469, + "narHash": "sha256-6FPUl7HVtvRHCCBQne7Ylp4p+dpP3P/OYuzjztZ4s70=", + "owner": "cachix", + "repo": "git-hooks.nix", + "rev": "f451c19376071a90d8c58ab1a953c6e9840527fd", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "git-hooks.nix", + "type": "github" + } + }, "libgit2": { "flake": false, "locked": { - "lastModified": 1697646580, - "narHash": "sha256-oX4Z3S9WtJlwvj0uH9HlYcWv+x1hqp8mhXl7HsLu2f0=", + "lastModified": 1715853528, + "narHash": "sha256-J2rCxTecyLbbDdsyBWn9w7r3pbKRMkI9E7RvRgAqBdY=", "owner": "libgit2", "repo": "libgit2", - "rev": "45fd9ed7ae1a9b74b957ef4f337bc3c8b3df01b5", + "rev": "36f7e21ad757a3dacc58cf7944329da6bc1d6e96", "type": "github" }, "original": { "owner": "libgit2", + "ref": "v1.8.1", "repo": "libgit2", "type": "github" } }, "nixpkgs": { "locked": { - "lastModified": 1709083642, - "narHash": "sha256-7kkJQd4rZ+vFrzWu8sTRtta5D1kBG0LSRYAfhtmMlSo=", + "lastModified": 1721548954, + "narHash": "sha256-7cCC8+Tdq1+3OPyc3+gVo9dzUNkNIQfwSDJ2HSi2u3o=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "b550fe4b4776908ac2a861124307045f8e717c8e", + "rev": "63d37ccd2d178d54e7fb691d7ec76000740ea24a", "type": "github" }, "original": { "owner": "NixOS", - "ref": "release-23.11", + "ref": "nixos-24.05", "repo": "nixpkgs", "type": "github" } }, + "nixpkgs-23-11": { + "locked": { + "lastModified": 1717159533, + "narHash": "sha256-oamiKNfr2MS6yH64rUn99mIZjc45nGJlj9eGth/3Xuw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446", + "type": "github" + } + }, "nixpkgs-regression": { "locked": { "lastModified": 1643052045, @@ -67,8 +129,11 @@ "root": { "inputs": { "flake-compat": "flake-compat", + "flake-parts": "flake-parts", + "git-hooks-nix": "git-hooks-nix", "libgit2": "libgit2", "nixpkgs": "nixpkgs", + "nixpkgs-23-11": "nixpkgs-23-11", "nixpkgs-regression": "nixpkgs-regression" } } diff --git a/flake.nix b/flake.nix index 42aaace67..9e8592e3a 100644 --- a/flake.nix +++ b/flake.nix @@ -1,18 +1,28 @@ { description = "The purely functional package manager"; - # TODO switch to nixos-23.11-small - # https://nixpk.gs/pr-tracker.html?pr=291954 - inputs.nixpkgs.url = "github:NixOS/nixpkgs/release-23.11"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; + inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446"; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; - inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; }; + inputs.libgit2 = { url = "github:libgit2/libgit2/v1.8.1"; flake = false; }; + + # dev tooling + inputs.flake-parts.url = "github:hercules-ci/flake-parts"; + inputs.git-hooks-nix.url = "github:cachix/git-hooks.nix"; + # work around https://github.com/NixOS/nix/issues/7730 + inputs.flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; + inputs.git-hooks-nix.inputs.nixpkgs.follows = "nixpkgs"; + inputs.git-hooks-nix.inputs.nixpkgs-stable.follows = "nixpkgs"; + # work around 7730 and https://github.com/NixOS/nix/issues/7807 + inputs.git-hooks-nix.inputs.flake-compat.follows = ""; + inputs.git-hooks-nix.inputs.gitignore.follows = ""; + + outputs = inputs@{ self, nixpkgs, nixpkgs-regression, libgit2, ... }: - outputs = { self, nixpkgs, nixpkgs-regression, libgit2, ... }: let inherit (nixpkgs) lib; - inherit (lib) fileset; officialRelease = false; @@ -31,14 +41,9 @@ crossSystems = [ "armv6l-unknown-linux-gnueabihf" "armv7l-unknown-linux-gnueabihf" - "x86_64-unknown-freebsd13" + "riscv64-unknown-linux-gnu" "x86_64-unknown-netbsd" - ]; - - # Nix doesn't yet build on this platform, so we put it in a - # separate list. We just use this for `devShells` and - # `nixpkgsFor`, which this depends on. - shellCrossSystems = crossSystems ++ [ + "x86_64-unknown-freebsd" "x86_64-w64-mingw32" ]; @@ -50,6 +55,18 @@ "stdenv" ]; + /** + `flatMapAttrs attrs f` applies `f` to each attribute in `attrs` and + merges the results into a single attribute set. + + This can be nested to form a build matrix where all the attributes + generated by the innermost `f` are returned as is. + (Provided that the names are unique.) + + See https://nixos.org/manual/nixpkgs/stable/index.html#function-library-lib.attrsets.concatMapAttrs + */ + flatMapAttrs = attrs: f: lib.concatMapAttrs f attrs; + forAllSystems = lib.genAttrs systems; forAllCrossSystems = lib.genAttrs crossSystems; @@ -63,6 +80,17 @@ }) stdenvs); + + # We don't apply flake-parts to the whole flake so that non-development attributes + # load without fetching any development inputs. + devFlake = inputs.flake-parts.lib.mkFlake { inherit inputs; } { + imports = [ ./maintainers/flake-module.nix ]; + systems = lib.subtractLists crossSystems systems; + perSystem = { system, ... }: { + _module.args.pkgs = nixpkgsFor.${system}.native; + }; + }; + # Memoize nixpkgs for different platforms for efficiency. nixpkgsFor = forAllSystems (system: let @@ -84,31 +112,9 @@ in { inherit stdenvs native; static = native.pkgsStatic; - cross = lib.genAttrs shellCrossSystems (crossSystem: make-pkgs crossSystem "stdenv"); + cross = forAllCrossSystems (crossSystem: make-pkgs crossSystem "stdenv"); }); - installScriptFor = tarballs: - nixpkgsFor.x86_64-linux.native.callPackage ./scripts/installer.nix { - inherit tarballs; - }; - - testNixVersions = pkgs: client: daemon: - pkgs.callPackage ./package.nix { - pname = - "nix-tests" - + lib.optionalString - (lib.versionAtLeast daemon.version "2.4pre20211005" && - lib.versionAtLeast client.version "2.4pre20211005") - "-${client.version}-against-${daemon.version}"; - - inherit fileset; - - test-client = client; - test-daemon = daemon; - - doBuild = false; - }; - binaryTarball = nix: pkgs: pkgs.callPackage ./scripts/binary-tarball.nix { inherit nix; }; @@ -120,244 +126,130 @@ { nixStable = prev.nix; - default-busybox-sandbox-shell = final.busybox.override { - useMusl = true; - enableStatic = true; - enableMinimal = true; - extraConfig = '' - CONFIG_FEATURE_FANCY_ECHO y - CONFIG_FEATURE_SH_MATH y - CONFIG_FEATURE_SH_MATH_64 y + # A new scope, so that we can use `callPackage` to inject our own interdependencies + # without "polluting" the top level "`pkgs`" attrset. + # This also has the benefit of providing us with a distinct set of packages + # we can iterate over. + nixComponents = lib.makeScope final.nixDependencies.newScope (import ./packaging/components.nix); - CONFIG_ASH y - CONFIG_ASH_OPTIMIZE_FOR_SIZE y - - CONFIG_ASH_ALIAS y - CONFIG_ASH_BASH_COMPAT y - CONFIG_ASH_CMDCMD y - CONFIG_ASH_ECHO y - CONFIG_ASH_GETOPTS y - CONFIG_ASH_INTERNAL_GLOB y - CONFIG_ASH_JOB_CONTROL y - CONFIG_ASH_PRINTF y - CONFIG_ASH_TEST y - ''; - }; - - libgit2-nix = final.libgit2.overrideAttrs (attrs: { - src = libgit2; - version = libgit2.lastModifiedDate; - cmakeFlags = attrs.cmakeFlags or [] - ++ [ "-DUSE_SSH=exec" ]; + # The dependencies are in their own scope, so that they don't have to be + # in Nixpkgs top level `pkgs` or `nixComponents`. + nixDependencies = lib.makeScope final.newScope (import ./packaging/dependencies.nix { + inherit inputs stdenv versionSuffix; + pkgs = final; }); - boehmgc-nix = (final.boehmgc.override { - enableLargeConfig = true; - }).overrideAttrs(o: { - patches = (o.patches or []) ++ [ - ./dep-patches/boehmgc-coroutine-sp-fallback.diff + nix = final.nixComponents.nix; - # https://github.com/ivmai/bdwgc/pull/586 - ./dep-patches/boehmgc-traceable_allocator-public.diff - ]; - }); - - changelog-d-nix = final.buildPackages.callPackage ./misc/changelog-d.nix { }; - - nix = - let - officialRelease = false; - versionSuffix = - if officialRelease - then "" - else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}"; - - in final.callPackage ./package.nix { - inherit - fileset - stdenv - versionSuffix - ; - officialRelease = false; - boehmgc = final.boehmgc-nix; - libgit2 = final.libgit2-nix; - busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell; - } // { - # this is a proper separate downstream package, but put - # here also for back compat reasons. - perl-bindings = final.nix-perl-bindings; - }; - - nix-perl-bindings = final.callPackage ./perl { - inherit fileset stdenv; + nix_noTests = final.nix.override { + doInstallCheck = false; + doCheck = false; }; + # See https://github.com/NixOS/nixpkgs/pull/214409 + # Remove when fixed in this flake's nixpkgs + pre-commit = + if prev.stdenv.hostPlatform.system == "i686-linux" + then (prev.pre-commit.override (o: { dotnet-sdk = ""; })).overridePythonAttrs (o: { doCheck = false; }) + else prev.pre-commit; + }; in { # A Nixpkgs overlay that overrides the 'nix' and - # 'nix.perl-bindings' packages. + # 'nix-perl-bindings' packages. overlays.default = overlayFor (p: p.stdenv); - hydraJobs = { - - # Binary package for various platforms. - build = forAllSystems (system: self.packages.${system}.nix); - - shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation); - - buildStatic = lib.genAttrs linux64BitSystems (system: self.packages.${system}.nix-static); - - buildCross = forAllCrossSystems (crossSystem: - lib.genAttrs ["x86_64-linux"] (system: self.packages.${system}."nix-${crossSystem}")); - - buildNoGc = forAllSystems (system: - self.packages.${system}.nix.override { enableGC = false; } - ); - - buildNoTests = forAllSystems (system: - self.packages.${system}.nix.override { - doCheck = false; - doInstallCheck = false; - installUnitTests = false; - } - ); - - # Toggles some settings for better coverage. Windows needs these - # library combinations, and Debian build Nix with GNU readline too. - buildReadlineNoMarkdown = forAllSystems (system: - self.packages.${system}.nix.override { - enableMarkdown = false; - readlineFlavor = "readline"; - } - ); - - # Perl bindings for various platforms. - perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nix.perl-bindings); - - # Binary tarball for various platforms, containing a Nix store - # with the closure of 'nix' package, and the second half of - # the installation script. - binaryTarball = forAllSystems (system: binaryTarball nixpkgsFor.${system}.native.nix nixpkgsFor.${system}.native); - - binaryTarballCross = lib.genAttrs ["x86_64-linux"] (system: - forAllCrossSystems (crossSystem: - binaryTarball - self.packages.${system}."nix-${crossSystem}" - nixpkgsFor.${system}.cross.${crossSystem})); - - # The first half of the installation script. This is uploaded - # to https://nixos.org/nix/install. It downloads the binary - # tarball for the user's system and calls the second half of the - # installation script. - installerScript = installScriptFor [ - # Native - self.hydraJobs.binaryTarball."x86_64-linux" - self.hydraJobs.binaryTarball."i686-linux" - self.hydraJobs.binaryTarball."aarch64-linux" - self.hydraJobs.binaryTarball."x86_64-darwin" - self.hydraJobs.binaryTarball."aarch64-darwin" - # Cross - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" - ]; - installerScriptForGHA = installScriptFor [ - # Native - self.hydraJobs.binaryTarball."x86_64-linux" - self.hydraJobs.binaryTarball."x86_64-darwin" - # Cross - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" - ]; - - # docker image with Nix inside - dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); - - # Line coverage analysis. - coverage = nixpkgsFor.x86_64-linux.native.nix.override { - pname = "nix-coverage"; - withCoverageChecks = true; - }; - - # API docs for Nix's unstable internal C++ interfaces. - internal-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ./package.nix { - inherit fileset; - doBuild = false; - enableInternalAPIDocs = true; - }; - - # System tests. - tests = import ./tests/nixos { inherit lib nixpkgs nixpkgsFor; } // { - - # Make sure that nix-env still produces the exact same result - # on a particular version of Nixpkgs. - evalNixpkgs = - let - inherit (nixpkgsFor.x86_64-linux.native) runCommand nix; - in - runCommand "eval-nixos" { buildInputs = [ nix ]; } - '' - type -p nix-env - # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593. - time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages - [[ $(sha1sum < packages | cut -c1-40) = ff451c521e61e4fe72bdbe2d0ca5d1809affa733 ]] - mkdir $out - ''; - - nixpkgsLibTests = - forAllSystems (system: - import (nixpkgs + "/lib/tests/release.nix") - { pkgs = nixpkgsFor.${system}.native; - nixVersions = [ self.packages.${system}.nix ]; - } - ); - }; - - metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" { - pkgs = nixpkgsFor.x86_64-linux.native; - nixpkgs = nixpkgs-regression; - }; - - installTests = forAllSystems (system: - let pkgs = nixpkgsFor.${system}.native; in - pkgs.runCommand "install-tests" { - againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix; - againstCurrentUnstable = - # FIXME: temporarily disable this on macOS because of #3605. - if system == "x86_64-linux" - then testNixVersions pkgs pkgs.nix pkgs.nixUnstable - else null; - # Disabled because the latest stable version doesn't handle - # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work - # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable; - } "touch $out"); - - installerTests = import ./tests/installer { - binaryTarballs = self.hydraJobs.binaryTarball; - inherit nixpkgsFor; - }; - + hydraJobs = import ./packaging/hydra.nix { + inherit + inputs + binaryTarball + forAllCrossSystems + forAllSystems + lib + linux64BitSystems + nixpkgsFor + self + ; }; checks = forAllSystems (system: { binaryTarball = self.hydraJobs.binaryTarball.${system}; - perlBindings = self.hydraJobs.perlBindings.${system}; installTests = self.hydraJobs.installTests.${system}; nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system}; rl-next = let pkgs = nixpkgsFor.${system}.native; in pkgs.buildPackages.runCommand "test-rl-next-release-notes" { } '' - LANG=C.UTF-8 ${pkgs.changelog-d-nix}/bin/changelog-d ${./doc/manual/rl-next} >$out + LANG=C.UTF-8 ${pkgs.changelog-d}/bin/changelog-d ${./doc/manual/rl-next} >$out ''; + repl-completion = nixpkgsFor.${system}.native.callPackage ./tests/repl-completion.nix { }; } // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) { dockerImage = self.hydraJobs.dockerImage.${system}; - }); + } // (lib.optionalAttrs (!(builtins.elem system linux32BitSystems))) { + # Some perl dependencies are broken on i686-linux. + # Since the support is only best-effort there, disable the perl + # bindings - packages = forAllSystems (system: rec { - inherit (nixpkgsFor.${system}.native) nix changelog-d-nix; - default = nix; - } // (lib.optionalAttrs (builtins.elem system linux64BitSystems) { - nix-static = nixpkgsFor.${system}.static.nix; + # Temporarily disabled because GitHub Actions OOM issues. Once + # the old build system is gone and we are back to one build + # system, we should reenable this. + #perlBindings = self.hydraJobs.perlBindings.${system}; + } + # Add "passthru" tests + // flatMapAttrs ({ + "" = nixpkgsFor.${system}.native; + } // lib.optionalAttrs (! nixpkgsFor.${system}.native.stdenv.hostPlatform.isDarwin) { + # TODO: enable static builds for darwin, blocked on: + # https://github.com/NixOS/nixpkgs/issues/320448 + "static-" = nixpkgsFor.${system}.static; + }) + (nixpkgsPrefix: nixpkgs: + flatMapAttrs nixpkgs.nixComponents + (pkgName: pkg: + flatMapAttrs pkg.tests or {} + (testName: test: { + "${nixpkgsPrefix}${pkgName}-${testName}" = test; + }) + ) + ) + // devFlake.checks.${system} or {} + ); + + packages = forAllSystems (system: + { # Here we put attributes that map 1:1 into packages., ie + # for which we don't apply the full build matrix such as cross or static. + inherit (nixpkgsFor.${system}.native) + changelog-d; + default = self.packages.${system}.nix; + nix-internal-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-internal-api-docs; + nix-external-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-external-api-docs; + } + # We need to flatten recursive attribute sets of derivations to pass `flake check`. + // flatMapAttrs + { # Components we'll iterate over in the upcoming lambda + "nix" = { }; + # Temporarily disabled because GitHub Actions OOM issues. Once + # the old build system is gone and we are back to one build + # system, we should reenable these. + #"nix-util" = { }; + #"nix-store" = { }; + #"nix-fetchers" = { }; + } + (pkgName: {}: { + # These attributes go right into `packages.`. + "${pkgName}" = nixpkgsFor.${system}.native.nixComponents.${pkgName}; + "${pkgName}-static" = nixpkgsFor.${system}.static.nixComponents.${pkgName}; + } + // flatMapAttrs (lib.genAttrs crossSystems (_: { })) (crossSystem: {}: { + # These attributes go right into `packages.`. + "${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName}; + }) + // flatMapAttrs (lib.genAttrs stdenvs (_: { })) (stdenvName: {}: { + # These attributes go right into `packages.`. + "${pkgName}-${stdenvName}" = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nixComponents.${pkgName}; + }) + ) + // lib.optionalAttrs (builtins.elem system linux64BitSystems) { dockerImage = let pkgs = nixpkgsFor.${system}.native; @@ -372,21 +264,27 @@ ln -s ${image} $image echo "file binary-dist $image" >> $out/nix-support/hydra-build-products ''; - } // builtins.listToAttrs (map - (crossSystem: { - name = "nix-${crossSystem}"; - value = nixpkgsFor.${system}.cross.${crossSystem}.nix; - }) - crossSystems) - // builtins.listToAttrs (map - (stdenvName: { - name = "nix-${stdenvName}"; - value = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nix; - }) - stdenvs))); + }); devShells = let - makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; forDevShell = true; }).overrideAttrs (attrs: { + makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; forDevShell = true; }).overrideAttrs (attrs: + let + modular = devFlake.getSystem stdenv.buildPlatform.system; + transformFlag = prefix: flag: + assert builtins.isString flag; + let + rest = builtins.substring 2 (builtins.stringLength flag) flag; + in + "-D${prefix}:${rest}"; + havePerl = stdenv.buildPlatform == stdenv.hostPlatform && stdenv.hostPlatform.isUnix; + ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags; + in { + pname = "shell-for-" + attrs.pname; + + # Remove the version suffix to avoid unnecessary attempts to substitute in nix develop + version = lib.fileContents ./.version; + name = attrs.pname; + installFlags = "sysconfdir=$(out)/etc"; shellHook = '' PATH=$prefix/bin:$PATH @@ -397,11 +295,62 @@ XDG_DATA_DIRS+=:$out/share ''; + # We use this shell with the local checkout, not unpackPhase. + src = null; + + env = { + # Needed for Meson to find Boost. + # https://github.com/NixOS/nixpkgs/issues/86131. + BOOST_INCLUDEDIR = "${lib.getDev pkgs.nixDependencies.boost}/include"; + BOOST_LIBRARYDIR = "${lib.getLib pkgs.nixDependencies.boost}/lib"; + # For `make format`, to work without installing pre-commit + _NIX_PRE_COMMIT_HOOKS_CONFIG = + "${(pkgs.formats.yaml { }).generate "pre-commit-config.yaml" modular.pre-commit.settings.rawConfig}"; + }; + + mesonFlags = + map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents.nix-util.mesonFlags) + ++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents.nix-store.mesonFlags) + ++ map (transformFlag "libfetchers") (ignoreCrossFile pkgs.nixComponents.nix-fetchers.mesonFlags) + ++ lib.optionals havePerl (map (transformFlag "perl") (ignoreCrossFile pkgs.nixComponents.nix-perl-bindings.mesonFlags)) + ++ map (transformFlag "libexpr") (ignoreCrossFile pkgs.nixComponents.nix-expr.mesonFlags) + ++ map (transformFlag "libcmd") (ignoreCrossFile pkgs.nixComponents.nix-cmd.mesonFlags) + ; + nativeBuildInputs = attrs.nativeBuildInputs or [] + ++ pkgs.nixComponents.nix-util.nativeBuildInputs + ++ pkgs.nixComponents.nix-store.nativeBuildInputs + ++ pkgs.nixComponents.nix-fetchers.nativeBuildInputs + ++ lib.optionals havePerl pkgs.nixComponents.nix-perl-bindings.nativeBuildInputs + ++ pkgs.nixComponents.nix-internal-api-docs.nativeBuildInputs + ++ pkgs.nixComponents.nix-external-api-docs.nativeBuildInputs + ++ lib.optional + (!stdenv.buildPlatform.canExecute stdenv.hostPlatform + # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479 + && !(stdenv.hostPlatform.isWindows && stdenv.buildPlatform.isDarwin) + && stdenv.hostPlatform.emulatorAvailable pkgs.buildPackages + && lib.meta.availableOn stdenv.buildPlatform (stdenv.hostPlatform.emulator pkgs.buildPackages)) + pkgs.buildPackages.mesonEmulatorHook + ++ [ + pkgs.buildPackages.cmake + pkgs.buildPackages.shellcheck + pkgs.buildPackages.changelog-d + modular.pre-commit.settings.package + (pkgs.writeScriptBin "pre-commit-hooks-install" + modular.pre-commit.settings.installationScript) + ] # TODO: Remove the darwin check once # https://github.com/NixOS/nixpkgs/pull/291814 is available ++ lib.optional (stdenv.cc.isClang && !stdenv.buildPlatform.isDarwin) pkgs.buildPackages.bear ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) pkgs.buildPackages.clang-tools; + + buildInputs = attrs.buildInputs or [] + ++ [ + pkgs.gtest + pkgs.rapidcheck + ] + ++ lib.optional havePerl pkgs.perl + ; }); in forAllSystems (system: @@ -413,8 +362,8 @@ in (makeShells "native" nixpkgsFor.${system}.native) // (lib.optionalAttrs (!nixpkgsFor.${system}.native.stdenv.isDarwin) - (makeShells "static" nixpkgsFor.${system}.static)) // - (lib.genAttrs shellCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv)) // + (makeShells "static" nixpkgsFor.${system}.static) // + (forAllCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv))) // { default = self.devShells.${system}.native-stdenvPackages; } diff --git a/local.mk b/local.mk index 3f3abb9f0..b27c7031e 100644 --- a/local.mk +++ b/local.mk @@ -2,9 +2,14 @@ GLOBAL_CXXFLAGS += -Wno-deprecated-declarations -Werror=switch # Allow switch-enum to be overridden for files that do not support it, usually because of dependency headers. ERROR_SWITCH_ENUM = -Werror=switch-enum -$(foreach i, config.h $(wildcard src/lib*/*.hh), \ +$(foreach i, config.h $(wildcard src/lib*/*.hh) $(filter-out %_internal.h, $(wildcard src/lib*c/*.h)), \ $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) +ifdef HOST_UNIX + $(foreach i, $(wildcard src/lib*/unix/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) +endif + $(GCH): src/libutil/util.hh config.h -GCH_CXXFLAGS = -I src/libutil +GCH_CXXFLAGS = $(INCLUDE_libutil) diff --git a/m4/gcc_bug_80431.m4 b/m4/gcc_bug_80431.m4 index e42f01956..cdc4ddb40 100644 --- a/m4/gcc_bug_80431.m4 +++ b/m4/gcc_bug_80431.m4 @@ -46,11 +46,13 @@ AC_DEFUN([ENSURE_NO_GCC_BUG_80431], ]])], [status_80431=0], [status_80431=$?], - [ - # Assume we're bug-free when cross-compiling - ]) + [status_80431='']) AC_LANG_POP(C++) AS_CASE([$status_80431], + [''],[ + AC_MSG_RESULT(cannot check because cross compiling) + AC_MSG_NOTICE(assume we are bug free) + ], [0],[ AC_MSG_RESULT(yes) ], diff --git a/maintainers/README.md b/maintainers/README.md index fa321c7c0..b92833497 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -30,17 +30,18 @@ We aim to achieve this by improving the contributor experience and attracting mo ## Members - Eelco Dolstra (@edolstra) – Team lead -- Théophane Hufschmitt (@thufschmitt) - Valentin Gagarin (@fricklerhandwerk) - Thomas Bereknyei (@tomberek) - Robert Hensing (@roberth) - John Ericson (@Ericson2314) +The team is on Github as [@NixOS/nix-team](https://github.com/orgs/NixOS/teams/nix-team). + ## Meeting protocol -The team meets twice a week: +The team meets twice a week (times are denoted in the [Europe/Amsterdam](https://en.m.wikipedia.org/wiki/Time_in_the_Netherlands) time zone): -- Discussion meeting: [Fridays 13:00-14:00 CET](https://calendar.google.com/calendar/event?eid=MHNtOGVuNWtrZXNpZHR2bW1sM3QyN2ZjaGNfMjAyMjExMjVUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) +- Discussion meeting: [Wednesday 21:00-22:00 Europe/Amsterdam](https://www.google.com/calendar/event?eid=ZG5rZzNyajRjajducGV2NGY5aGkzYWIwdnJfMjAyNDA1MDhUMTkwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) 1. Triage issues and pull requests from the [No Status](#no-status) column (30 min) 2. Discuss issues and pull requests from the [To discuss](#to-discuss) column (30 min). @@ -49,15 +50,19 @@ The team meets twice a week: - mark it as draft if it is blocked on the contributor - escalate it back to the team by moving it to To discuss, and leaving a comment as to why the issue needs to be discussed again. -- Work meeting: [Mondays 13:00-15:00 CET](https://calendar.google.com/calendar/event?eid=NTM1MG1wNGJnOGpmOTZhYms3bTB1bnY5cWxfMjAyMjExMjFUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) +- Work meeting: [Mondays 14:00-16:00 Europe/Amsterdam](https://www.google.com/calendar/event?eid=Ym52NDdzYnRic2NzcDcybjZiNDhpNzhpa3NfMjAyNDA1MTNUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) 1. Code review on pull requests from [In review](#in-review). 2. Other chores and tasks. Meeting notes are collected on a [collaborative scratchpad](https://pad.lassul.us/Cv7FpYx-Ri-4VjUykQOLAw). -Notes on issues and pull requests are posted as comments and linked from the meeting notes, so they are easy to find from both places. +Notes on issues and pull requests are posted as comments and linked from the meeting notes, so they are can be found from both places. [All meeting notes](https://discourse.nixos.org/search?expanded=true&q=Nix%20team%20meeting%20minutes%20%23%20%23dev%3Anix%20in%3Atitle%20order%3Alatest_topic) are published on Discourse under the [Nix category](https://discourse.nixos.org/c/dev/nix/50). +Team meetings are generally open to anyone interested. +We can make exceptions to discuss sensitive issues, such as security incidents or people matters. +Contact any team member to get a calendar invite for reminders and updates. + ## Project board protocol The team uses a [GitHub project board](https://github.com/orgs/NixOS/projects/19/views/1) for tracking its work. diff --git a/maintainers/data/release-credits-email-to-handle.json b/maintainers/data/release-credits-email-to-handle.json new file mode 100644 index 000000000..cddc1a6e7 --- /dev/null +++ b/maintainers/data/release-credits-email-to-handle.json @@ -0,0 +1,52 @@ +{ + "bogus": "bogus", + "edolstra@gmail.com": "edolstra", + "roberth@users.noreply.github.com": "roberth", + "toscano.pino@tiscali.it": "pinotree", + "valentin@gagarin.work": "fricklerhandwerk", + "mr.trubach@icloud.com": "tie", + "robert@roberthensing.nl": "roberth", + "lix@jade.fyi": "lf-", + "cole.e.helbling@outlook.com": "cole-h", + "joerg@thalheim.io": "Mic92", + "John.Ericson@Obsidian.Systems": "Ericson2314", + "ryan.hendrickson@alum.mit.edu": "rhendric", + "67135060+poweredbypie@users.noreply.github.com": "poweredbypie", + "detroyejr@outlook.com": "detroyejr", + "silvan.mosberger@tweag.io": "infinisil", + "vcs@emily.moe": "emilazy", + "farid.m.zakaria@gmail.com": "fzakaria", + "22859658+RTUnreal@users.noreply.github.com": "RTUnreal", + "me@las.rs": "L-as", + "philip.taron@gmail.com": "philiptaron", + "root@goldstein.rs": "GoldsteinE", + "tomberek@users.noreply.github.com": "tomberek", + "lexi.mattick@neuralink.com": "kognise", + "andrew@johnandrewmarshall.com": "amarshall", + "contact@romain-neil.fr": "romain-neil", + "Mic92@users.noreply.github.com": "Mic92", + "valentin.gagarin@tweag.io": "fricklerhandwerk", + "siddhantk232@gmail.com": "siddhantk232", + "kn@openbsd.org": "klemensn", + "slyich@gmail.com": "trofi", + "theophane.hufschmitt@tweag.io": "thufschmitt", + "alicebob@lijzij.de": "alicebob", + "winter@winter.cafe": "winterqt", + "brian@brianmckenna.org": "puffnfresh", + "git@haenoe.party": "haenoe", + "peshogo@gmail.com": "pineapplehunter", + "poweredbypie@users.noreply.github.com": "poweredbypie", + "arthur200126@gmail.com": "Artoria2e5", + "tomberek@gmail.com": "tomberek", + "jaredbaur@fastmail.com": "jmbaur", + "andreas@rammhold.de": "andir", + "hamirmahal@gmail.com": "hamirmahal", + "git@JohnEricson.me": "Ericson2314", + "8763518+SkamDart@users.noreply.github.com": "SkamDart", + "kirillrdy@gmail.com": "kirillrdy", + "pennae@lix.systems": "pennae", + "delroth@gmail.com": "delroth", + "enno@nerdworks.de": "elohmeier", + "mjbauer95@gmail.com": "matthewbauer", + "MostAwesomeDude@gmail.com": "MostAwesomeDude" +} \ No newline at end of file diff --git a/maintainers/data/release-credits-handle-to-name.json b/maintainers/data/release-credits-handle-to-name.json new file mode 100644 index 000000000..abf9ed05b --- /dev/null +++ b/maintainers/data/release-credits-handle-to-name.json @@ -0,0 +1,45 @@ +{ + "fzakaria": "Farid Zakaria", + "kognise": "Lexi Mattick", + "L-as": "Las Safin", + "haenoe": "HaeNoe", + "andir": "Andreas Rammhold", + "matthewbauer": "Matthew Bauer", + "emilazy": "Emily", + "pineapplehunter": "Shogo Takata", + "RTUnreal": null, + "jmbaur": "Jared Baur", + "Ericson2314": "John Ericson", + "pinotree": "Pino Toscano", + "tie": "Ivan Trubach", + "poweredbypie": null, + "fricklerhandwerk": "Valentin Gagarin", + "Mic92": "J\u00f6rg Thalheim", + "alicebob": "Harmen", + "elohmeier": "Enno Richter", + "delroth": "Pierre Bourdon", + "kirillrdy": null, + "thufschmitt": "Th\u00e9ophane Hufschmitt", + "detroyejr": "Jonathan De Troye", + "klemensn": "Klemens Nanni", + "tomberek": null, + "rhendric": "Ryan Hendrickson", + "philiptaron": "Philip Taron", + "puffnfresh": "Brian McKenna", + "lf-": "jade", + "romain-neil": "Romain Neil", + "hamirmahal": "Hamir Mahal", + "edolstra": "Eelco Dolstra", + "Artoria2e5": "Mingye Wang", + "SkamDart": "Cameron", + "roberth": "Robert Hensing", + "amarshall": "Andrew Marshall", + "trofi": "Sergei Trofimovich", + "cole-h": "Cole Helbling", + "infinisil": "Silvan Mosberger", + "siddhantk232": "Siddhant Kumar", + "winterqt": "Winter", + "GoldsteinE": "Max \u201cGoldstein\u201d Siling", + "pennae": null, + "MostAwesomeDude": "Corbin Simpson" +} \ No newline at end of file diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix new file mode 100644 index 000000000..be91df536 --- /dev/null +++ b/maintainers/flake-module.nix @@ -0,0 +1,677 @@ +{ lib, getSystem, inputs, ... }: + +{ + imports = [ + inputs.git-hooks-nix.flakeModule + ]; + + perSystem = { config, pkgs, ... }: { + + # https://flake.parts/options/pre-commit-hooks-nix.html#options + pre-commit.settings = { + hooks = { + clang-format = { + enable = true; + excludes = [ + # We don't want to format test data + # ''tests/(?!nixos/).*\.nix'' + ''^tests/unit/[^/]*/data/.*$'' + + # Don't format vendored code + ''^doc/manual/redirects\.js$'' + ''^doc/manual/theme/highlight\.js$'' + + # We haven't applied formatting to these files yet + ''^doc/manual/redirects\.js$'' + ''^doc/manual/theme/highlight\.js$'' + ''^precompiled-headers\.h$'' + ''^src/build-remote/build-remote\.cc$'' + ''^src/libcmd/built-path\.cc$'' + ''^src/libcmd/built-path\.hh$'' + ''^src/libcmd/command\.cc$'' + ''^src/libcmd/command\.hh$'' + ''^src/libcmd/common-eval-args\.cc$'' + ''^src/libcmd/common-eval-args\.hh$'' + ''^src/libcmd/editor-for\.cc$'' + ''^src/libcmd/installable-attr-path\.cc$'' + ''^src/libcmd/installable-attr-path\.hh$'' + ''^src/libcmd/installable-derived-path\.cc$'' + ''^src/libcmd/installable-derived-path\.hh$'' + ''^src/libcmd/installable-flake\.cc$'' + ''^src/libcmd/installable-flake\.hh$'' + ''^src/libcmd/installable-value\.cc$'' + ''^src/libcmd/installable-value\.hh$'' + ''^src/libcmd/installables\.cc$'' + ''^src/libcmd/installables\.hh$'' + ''^src/libcmd/legacy\.hh$'' + ''^src/libcmd/markdown\.cc$'' + ''^src/libcmd/misc-store-flags\.cc$'' + ''^src/libcmd/repl-interacter\.cc$'' + ''^src/libcmd/repl-interacter\.hh$'' + ''^src/libcmd/repl\.cc$'' + ''^src/libcmd/repl\.hh$'' + ''^src/libexpr-c/nix_api_expr\.cc$'' + ''^src/libexpr-c/nix_api_external\.cc$'' + ''^src/libexpr/attr-path\.cc$'' + ''^src/libexpr/attr-path\.hh$'' + ''^src/libexpr/attr-set\.cc$'' + ''^src/libexpr/attr-set\.hh$'' + ''^src/libexpr/eval-cache\.cc$'' + ''^src/libexpr/eval-cache\.hh$'' + ''^src/libexpr/eval-error\.cc$'' + ''^src/libexpr/eval-inline\.hh$'' + ''^src/libexpr/eval-settings\.cc$'' + ''^src/libexpr/eval-settings\.hh$'' + ''^src/libexpr/eval\.cc$'' + ''^src/libexpr/eval\.hh$'' + ''^src/libexpr/function-trace\.cc$'' + ''^src/libexpr/gc-small-vector\.hh$'' + ''^src/libexpr/get-drvs\.cc$'' + ''^src/libexpr/get-drvs\.hh$'' + ''^src/libexpr/json-to-value\.cc$'' + ''^src/libexpr/nixexpr\.cc$'' + ''^src/libexpr/nixexpr\.hh$'' + ''^src/libexpr/parser-state\.hh$'' + ''^src/libexpr/pos-table\.hh$'' + ''^src/libexpr/primops\.cc$'' + ''^src/libexpr/primops\.hh$'' + ''^src/libexpr/primops/context\.cc$'' + ''^src/libexpr/primops/fetchClosure\.cc$'' + ''^src/libexpr/primops/fetchMercurial\.cc$'' + ''^src/libexpr/primops/fetchTree\.cc$'' + ''^src/libexpr/primops/fromTOML\.cc$'' + ''^src/libexpr/print-ambiguous\.cc$'' + ''^src/libexpr/print-ambiguous\.hh$'' + ''^src/libexpr/print-options\.hh$'' + ''^src/libexpr/print\.cc$'' + ''^src/libexpr/print\.hh$'' + ''^src/libexpr/search-path\.cc$'' + ''^src/libexpr/symbol-table\.hh$'' + ''^src/libexpr/value-to-json\.cc$'' + ''^src/libexpr/value-to-json\.hh$'' + ''^src/libexpr/value-to-xml\.cc$'' + ''^src/libexpr/value-to-xml\.hh$'' + ''^src/libexpr/value\.hh$'' + ''^src/libexpr/value/context\.cc$'' + ''^src/libexpr/value/context\.hh$'' + ''^src/libfetchers/attrs\.cc$'' + ''^src/libfetchers/cache\.cc$'' + ''^src/libfetchers/cache\.hh$'' + ''^src/libfetchers/fetch-settings\.cc$'' + ''^src/libfetchers/fetch-settings\.hh$'' + ''^src/libfetchers/fetch-to-store\.cc$'' + ''^src/libfetchers/fetchers\.cc$'' + ''^src/libfetchers/fetchers\.hh$'' + ''^src/libfetchers/filtering-source-accessor\.cc$'' + ''^src/libfetchers/filtering-source-accessor\.hh$'' + ''^src/libfetchers/fs-source-accessor\.cc$'' + ''^src/libfetchers/fs-source-accessor\.hh$'' + ''^src/libfetchers/git-utils\.cc$'' + ''^src/libfetchers/git-utils\.hh$'' + ''^src/libfetchers/github\.cc$'' + ''^src/libfetchers/indirect\.cc$'' + ''^src/libfetchers/memory-source-accessor\.cc$'' + ''^src/libfetchers/path\.cc$'' + ''^src/libfetchers/registry\.cc$'' + ''^src/libfetchers/registry\.hh$'' + ''^src/libfetchers/tarball\.cc$'' + ''^src/libfetchers/tarball\.hh$'' + ''^src/libfetchers/git\.cc$'' + ''^src/libfetchers/mercurial\.cc$'' + ''^src/libflake/flake/config\.cc$'' + ''^src/libflake/flake/flake\.cc$'' + ''^src/libflake/flake/flake\.hh$'' + ''^src/libflake/flake/flakeref\.cc$'' + ''^src/libflake/flake/flakeref\.hh$'' + ''^src/libflake/flake/lockfile\.cc$'' + ''^src/libflake/flake/lockfile\.hh$'' + ''^src/libflake/flake/url-name\.cc$'' + ''^src/libmain/common-args\.cc$'' + ''^src/libmain/common-args\.hh$'' + ''^src/libmain/loggers\.cc$'' + ''^src/libmain/loggers\.hh$'' + ''^src/libmain/progress-bar\.cc$'' + ''^src/libmain/shared\.cc$'' + ''^src/libmain/shared\.hh$'' + ''^src/libmain/unix/stack\.cc$'' + ''^src/libstore/binary-cache-store\.cc$'' + ''^src/libstore/binary-cache-store\.hh$'' + ''^src/libstore/build-result\.hh$'' + ''^src/libstore/builtins\.hh$'' + ''^src/libstore/builtins/buildenv\.cc$'' + ''^src/libstore/builtins/buildenv\.hh$'' + ''^src/libstore/common-protocol-impl\.hh$'' + ''^src/libstore/common-protocol\.cc$'' + ''^src/libstore/common-protocol\.hh$'' + ''^src/libstore/common-ssh-store-config\.hh$'' + ''^src/libstore/content-address\.cc$'' + ''^src/libstore/content-address\.hh$'' + ''^src/libstore/daemon\.cc$'' + ''^src/libstore/daemon\.hh$'' + ''^src/libstore/derivations\.cc$'' + ''^src/libstore/derivations\.hh$'' + ''^src/libstore/derived-path-map\.cc$'' + ''^src/libstore/derived-path-map\.hh$'' + ''^src/libstore/derived-path\.cc$'' + ''^src/libstore/derived-path\.hh$'' + ''^src/libstore/downstream-placeholder\.cc$'' + ''^src/libstore/downstream-placeholder\.hh$'' + ''^src/libstore/dummy-store\.cc$'' + ''^src/libstore/export-import\.cc$'' + ''^src/libstore/filetransfer\.cc$'' + ''^src/libstore/filetransfer\.hh$'' + ''^src/libstore/gc-store\.hh$'' + ''^src/libstore/globals\.cc$'' + ''^src/libstore/globals\.hh$'' + ''^src/libstore/http-binary-cache-store\.cc$'' + ''^src/libstore/legacy-ssh-store\.cc$'' + ''^src/libstore/legacy-ssh-store\.hh$'' + ''^src/libstore/length-prefixed-protocol-helper\.hh$'' + ''^src/libstore/linux/personality\.cc$'' + ''^src/libstore/linux/personality\.hh$'' + ''^src/libstore/local-binary-cache-store\.cc$'' + ''^src/libstore/local-fs-store\.cc$'' + ''^src/libstore/local-fs-store\.hh$'' + ''^src/libstore/log-store\.cc$'' + ''^src/libstore/log-store\.hh$'' + ''^src/libstore/machines\.cc$'' + ''^src/libstore/machines\.hh$'' + ''^src/libstore/make-content-addressed\.cc$'' + ''^src/libstore/make-content-addressed\.hh$'' + ''^src/libstore/misc\.cc$'' + ''^src/libstore/names\.cc$'' + ''^src/libstore/names\.hh$'' + ''^src/libstore/nar-accessor\.cc$'' + ''^src/libstore/nar-accessor\.hh$'' + ''^src/libstore/nar-info-disk-cache\.cc$'' + ''^src/libstore/nar-info-disk-cache\.hh$'' + ''^src/libstore/nar-info\.cc$'' + ''^src/libstore/nar-info\.hh$'' + ''^src/libstore/outputs-spec\.cc$'' + ''^src/libstore/outputs-spec\.hh$'' + ''^src/libstore/parsed-derivations\.cc$'' + ''^src/libstore/path-info\.cc$'' + ''^src/libstore/path-info\.hh$'' + ''^src/libstore/path-references\.cc$'' + ''^src/libstore/path-regex\.hh$'' + ''^src/libstore/path-with-outputs\.cc$'' + ''^src/libstore/path\.cc$'' + ''^src/libstore/path\.hh$'' + ''^src/libstore/pathlocks\.cc$'' + ''^src/libstore/pathlocks\.hh$'' + ''^src/libstore/profiles\.cc$'' + ''^src/libstore/profiles\.hh$'' + ''^src/libstore/realisation\.cc$'' + ''^src/libstore/realisation\.hh$'' + ''^src/libstore/remote-fs-accessor\.cc$'' + ''^src/libstore/remote-fs-accessor\.hh$'' + ''^src/libstore/remote-store-connection\.hh$'' + ''^src/libstore/remote-store\.cc$'' + ''^src/libstore/remote-store\.hh$'' + ''^src/libstore/s3-binary-cache-store\.cc$'' + ''^src/libstore/s3\.hh$'' + ''^src/libstore/serve-protocol-impl\.cc$'' + ''^src/libstore/serve-protocol-impl\.hh$'' + ''^src/libstore/serve-protocol\.cc$'' + ''^src/libstore/serve-protocol\.hh$'' + ''^src/libstore/sqlite\.cc$'' + ''^src/libstore/sqlite\.hh$'' + ''^src/libstore/ssh-store\.cc$'' + ''^src/libstore/ssh\.cc$'' + ''^src/libstore/ssh\.hh$'' + ''^src/libstore/store-api\.cc$'' + ''^src/libstore/store-api\.hh$'' + ''^src/libstore/store-dir-config\.hh$'' + ''^src/libstore/build/derivation-goal\.cc$'' + ''^src/libstore/build/derivation-goal\.hh$'' + ''^src/libstore/build/drv-output-substitution-goal\.cc$'' + ''^src/libstore/build/drv-output-substitution-goal\.hh$'' + ''^src/libstore/build/entry-points\.cc$'' + ''^src/libstore/build/goal\.cc$'' + ''^src/libstore/build/goal\.hh$'' + ''^src/libstore/unix/build/hook-instance\.cc$'' + ''^src/libstore/unix/build/local-derivation-goal\.cc$'' + ''^src/libstore/unix/build/local-derivation-goal\.hh$'' + ''^src/libstore/build/substitution-goal\.cc$'' + ''^src/libstore/build/substitution-goal\.hh$'' + ''^src/libstore/build/worker\.cc$'' + ''^src/libstore/build/worker\.hh$'' + ''^src/libstore/builtins/fetchurl\.cc$'' + ''^src/libstore/builtins/unpack-channel\.cc$'' + ''^src/libstore/gc\.cc$'' + ''^src/libstore/local-overlay-store\.cc$'' + ''^src/libstore/local-overlay-store\.hh$'' + ''^src/libstore/local-store\.cc$'' + ''^src/libstore/local-store\.hh$'' + ''^src/libstore/unix/user-lock\.cc$'' + ''^src/libstore/unix/user-lock\.hh$'' + ''^src/libstore/optimise-store\.cc$'' + ''^src/libstore/unix/pathlocks\.cc$'' + ''^src/libstore/posix-fs-canonicalise\.cc$'' + ''^src/libstore/posix-fs-canonicalise\.hh$'' + ''^src/libstore/uds-remote-store\.cc$'' + ''^src/libstore/uds-remote-store\.hh$'' + ''^src/libstore/windows/build\.cc$'' + ''^src/libstore/worker-protocol-impl\.hh$'' + ''^src/libstore/worker-protocol\.cc$'' + ''^src/libstore/worker-protocol\.hh$'' + ''^src/libutil-c/nix_api_util_internal\.h$'' + ''^src/libutil/archive\.cc$'' + ''^src/libutil/archive\.hh$'' + ''^src/libutil/args\.cc$'' + ''^src/libutil/args\.hh$'' + ''^src/libutil/args/root\.hh$'' + ''^src/libutil/callback\.hh$'' + ''^src/libutil/canon-path\.cc$'' + ''^src/libutil/canon-path\.hh$'' + ''^src/libutil/chunked-vector\.hh$'' + ''^src/libutil/closure\.hh$'' + ''^src/libutil/comparator\.hh$'' + ''^src/libutil/compute-levels\.cc$'' + ''^src/libutil/config-impl\.hh$'' + ''^src/libutil/config\.cc$'' + ''^src/libutil/config\.hh$'' + ''^src/libutil/current-process\.cc$'' + ''^src/libutil/current-process\.hh$'' + ''^src/libutil/english\.cc$'' + ''^src/libutil/english\.hh$'' + ''^src/libutil/environment-variables\.cc$'' + ''^src/libutil/error\.cc$'' + ''^src/libutil/error\.hh$'' + ''^src/libutil/exit\.hh$'' + ''^src/libutil/experimental-features\.cc$'' + ''^src/libutil/experimental-features\.hh$'' + ''^src/libutil/file-content-address\.cc$'' + ''^src/libutil/file-content-address\.hh$'' + ''^src/libutil/file-descriptor\.cc$'' + ''^src/libutil/file-descriptor\.hh$'' + ''^src/libutil/file-path-impl\.hh$'' + ''^src/libutil/file-path\.hh$'' + ''^src/libutil/file-system\.cc$'' + ''^src/libutil/file-system\.hh$'' + ''^src/libutil/finally\.hh$'' + ''^src/libutil/fmt\.hh$'' + ''^src/libutil/fs-sink\.cc$'' + ''^src/libutil/fs-sink\.hh$'' + ''^src/libutil/git\.cc$'' + ''^src/libutil/git\.hh$'' + ''^src/libutil/hash\.cc$'' + ''^src/libutil/hash\.hh$'' + ''^src/libutil/hilite\.cc$'' + ''^src/libutil/hilite\.hh$'' + ''^src/libutil/source-accessor\.hh$'' + ''^src/libutil/json-impls\.hh$'' + ''^src/libutil/json-utils\.cc$'' + ''^src/libutil/json-utils\.hh$'' + ''^src/libutil/linux/cgroup\.cc$'' + ''^src/libutil/linux/namespaces\.cc$'' + ''^src/libutil/logging\.cc$'' + ''^src/libutil/logging\.hh$'' + ''^src/libutil/lru-cache\.hh$'' + ''^src/libutil/memory-source-accessor\.cc$'' + ''^src/libutil/memory-source-accessor\.hh$'' + ''^src/libutil/pool\.hh$'' + ''^src/libutil/position\.cc$'' + ''^src/libutil/position\.hh$'' + ''^src/libutil/posix-source-accessor\.cc$'' + ''^src/libutil/posix-source-accessor\.hh$'' + ''^src/libutil/processes\.hh$'' + ''^src/libutil/ref\.hh$'' + ''^src/libutil/references\.cc$'' + ''^src/libutil/references\.hh$'' + ''^src/libutil/regex-combinators\.hh$'' + ''^src/libutil/serialise\.cc$'' + ''^src/libutil/serialise\.hh$'' + ''^src/libutil/signals\.hh$'' + ''^src/libutil/signature/local-keys\.cc$'' + ''^src/libutil/signature/local-keys\.hh$'' + ''^src/libutil/signature/signer\.cc$'' + ''^src/libutil/signature/signer\.hh$'' + ''^src/libutil/source-accessor\.cc$'' + ''^src/libutil/source-accessor\.hh$'' + ''^src/libutil/source-path\.cc$'' + ''^src/libutil/source-path\.hh$'' + ''^src/libutil/split\.hh$'' + ''^src/libutil/suggestions\.cc$'' + ''^src/libutil/suggestions\.hh$'' + ''^src/libutil/sync\.hh$'' + ''^src/libutil/terminal\.cc$'' + ''^src/libutil/terminal\.hh$'' + ''^src/libutil/thread-pool\.cc$'' + ''^src/libutil/thread-pool\.hh$'' + ''^src/libutil/topo-sort\.hh$'' + ''^src/libutil/types\.hh$'' + ''^src/libutil/unix/file-descriptor\.cc$'' + ''^src/libutil/unix/file-path\.cc$'' + ''^src/libutil/unix/monitor-fd\.hh$'' + ''^src/libutil/unix/processes\.cc$'' + ''^src/libutil/unix/signals-impl\.hh$'' + ''^src/libutil/unix/signals\.cc$'' + ''^src/libutil/unix-domain-socket\.cc$'' + ''^src/libutil/unix/users\.cc$'' + ''^src/libutil/url-parts\.hh$'' + ''^src/libutil/url\.cc$'' + ''^src/libutil/url\.hh$'' + ''^src/libutil/users\.cc$'' + ''^src/libutil/users\.hh$'' + ''^src/libutil/util\.cc$'' + ''^src/libutil/util\.hh$'' + ''^src/libutil/variant-wrapper\.hh$'' + ''^src/libutil/windows/environment-variables\.cc$'' + ''^src/libutil/windows/file-descriptor\.cc$'' + ''^src/libutil/windows/file-path\.cc$'' + ''^src/libutil/windows/processes\.cc$'' + ''^src/libutil/windows/users\.cc$'' + ''^src/libutil/windows/windows-error\.cc$'' + ''^src/libutil/windows/windows-error\.hh$'' + ''^src/libutil/xml-writer\.cc$'' + ''^src/libutil/xml-writer\.hh$'' + ''^src/nix-build/nix-build\.cc$'' + ''^src/nix-channel/nix-channel\.cc$'' + ''^src/nix-collect-garbage/nix-collect-garbage\.cc$'' + ''^src/nix-env/buildenv.nix$'' + ''^src/nix-env/nix-env\.cc$'' + ''^src/nix-env/user-env\.cc$'' + ''^src/nix-env/user-env\.hh$'' + ''^src/nix-instantiate/nix-instantiate\.cc$'' + ''^src/nix-store/dotgraph\.cc$'' + ''^src/nix-store/graphml\.cc$'' + ''^src/nix-store/nix-store\.cc$'' + ''^src/nix/add-to-store\.cc$'' + ''^src/nix/app\.cc$'' + ''^src/nix/build\.cc$'' + ''^src/nix/bundle\.cc$'' + ''^src/nix/cat\.cc$'' + ''^src/nix/config-check\.cc$'' + ''^src/nix/config\.cc$'' + ''^src/nix/copy\.cc$'' + ''^src/nix/derivation-add\.cc$'' + ''^src/nix/derivation-show\.cc$'' + ''^src/nix/derivation\.cc$'' + ''^src/nix/develop\.cc$'' + ''^src/nix/diff-closures\.cc$'' + ''^src/nix/dump-path\.cc$'' + ''^src/nix/edit\.cc$'' + ''^src/nix/eval\.cc$'' + ''^src/nix/flake\.cc$'' + ''^src/nix/fmt\.cc$'' + ''^src/nix/hash\.cc$'' + ''^src/nix/log\.cc$'' + ''^src/nix/ls\.cc$'' + ''^src/nix/main\.cc$'' + ''^src/nix/make-content-addressed\.cc$'' + ''^src/nix/nar\.cc$'' + ''^src/nix/optimise-store\.cc$'' + ''^src/nix/path-from-hash-part\.cc$'' + ''^src/nix/path-info\.cc$'' + ''^src/nix/prefetch\.cc$'' + ''^src/nix/profile\.cc$'' + ''^src/nix/realisation\.cc$'' + ''^src/nix/registry\.cc$'' + ''^src/nix/repl\.cc$'' + ''^src/nix/run\.cc$'' + ''^src/nix/run\.hh$'' + ''^src/nix/search\.cc$'' + ''^src/nix/sigs\.cc$'' + ''^src/nix/store-copy-log\.cc$'' + ''^src/nix/store-delete\.cc$'' + ''^src/nix/store-gc\.cc$'' + ''^src/nix/store-info\.cc$'' + ''^src/nix/store-repair\.cc$'' + ''^src/nix/store\.cc$'' + ''^src/nix/unix/daemon\.cc$'' + ''^src/nix/upgrade-nix\.cc$'' + ''^src/nix/verify\.cc$'' + ''^src/nix/why-depends\.cc$'' + + ''^tests/functional/plugins/plugintest\.cc'' + ''^tests/functional/test-libstoreconsumer/main\.cc'' + ''^tests/nixos/ca-fd-leak/sender\.c'' + ''^tests/nixos/ca-fd-leak/smuggler\.c'' + ''^tests/nixos/user-sandboxing/attacker\.c'' + ''^tests/unit/libexpr-support/tests/libexpr\.hh'' + ''^tests/unit/libexpr-support/tests/value/context\.cc'' + ''^tests/unit/libexpr-support/tests/value/context\.hh'' + ''^tests/unit/libexpr/derived-path\.cc'' + ''^tests/unit/libexpr/error_traces\.cc'' + ''^tests/unit/libexpr/eval\.cc'' + ''^tests/unit/libexpr/json\.cc'' + ''^tests/unit/libexpr/main\.cc'' + ''^tests/unit/libexpr/primops\.cc'' + ''^tests/unit/libexpr/search-path\.cc'' + ''^tests/unit/libexpr/trivial\.cc'' + ''^tests/unit/libexpr/value/context\.cc'' + ''^tests/unit/libexpr/value/print\.cc'' + ''^tests/unit/libfetchers/public-key\.cc'' + ''^tests/unit/libflake/flakeref\.cc'' + ''^tests/unit/libflake/url-name\.cc'' + ''^tests/unit/libstore-support/tests/derived-path\.cc'' + ''^tests/unit/libstore-support/tests/derived-path\.hh'' + ''^tests/unit/libstore-support/tests/nix_api_store\.hh'' + ''^tests/unit/libstore-support/tests/outputs-spec\.cc'' + ''^tests/unit/libstore-support/tests/outputs-spec\.hh'' + ''^tests/unit/libstore-support/tests/path\.cc'' + ''^tests/unit/libstore-support/tests/path\.hh'' + ''^tests/unit/libstore-support/tests/protocol\.hh'' + ''^tests/unit/libstore/common-protocol\.cc'' + ''^tests/unit/libstore/content-address\.cc'' + ''^tests/unit/libstore/derivation\.cc'' + ''^tests/unit/libstore/derived-path\.cc'' + ''^tests/unit/libstore/downstream-placeholder\.cc'' + ''^tests/unit/libstore/machines\.cc'' + ''^tests/unit/libstore/nar-info-disk-cache\.cc'' + ''^tests/unit/libstore/nar-info\.cc'' + ''^tests/unit/libstore/outputs-spec\.cc'' + ''^tests/unit/libstore/path-info\.cc'' + ''^tests/unit/libstore/path\.cc'' + ''^tests/unit/libstore/serve-protocol\.cc'' + ''^tests/unit/libstore/worker-protocol\.cc'' + ''^tests/unit/libutil-support/tests/characterization\.hh'' + ''^tests/unit/libutil-support/tests/hash\.cc'' + ''^tests/unit/libutil-support/tests/hash\.hh'' + ''^tests/unit/libutil/args\.cc'' + ''^tests/unit/libutil/canon-path\.cc'' + ''^tests/unit/libutil/chunked-vector\.cc'' + ''^tests/unit/libutil/closure\.cc'' + ''^tests/unit/libutil/compression\.cc'' + ''^tests/unit/libutil/config\.cc'' + ''^tests/unit/libutil/file-content-address\.cc'' + ''^tests/unit/libutil/git\.cc'' + ''^tests/unit/libutil/hash\.cc'' + ''^tests/unit/libutil/hilite\.cc'' + ''^tests/unit/libutil/json-utils\.cc'' + ''^tests/unit/libutil/logging\.cc'' + ''^tests/unit/libutil/lru-cache\.cc'' + ''^tests/unit/libutil/pool\.cc'' + ''^tests/unit/libutil/references\.cc'' + ''^tests/unit/libutil/suggestions\.cc'' + ''^tests/unit/libutil/tests\.cc'' + ''^tests/unit/libutil/url\.cc'' + ''^tests/unit/libutil/xml-writer\.cc'' + ]; + }; + shellcheck = { + enable = true; + excludes = [ + # We haven't linted these files yet + ''^config/install-sh$'' + ''^misc/bash/completion\.sh$'' + ''^misc/fish/completion\.fish$'' + ''^misc/zsh/completion\.zsh$'' + ''^scripts/create-darwin-volume\.sh$'' + ''^scripts/install-darwin-multi-user\.sh$'' + ''^scripts/install-multi-user\.sh$'' + ''^scripts/install-nix-from-closure\.sh$'' + ''^scripts/install-systemd-multi-user\.sh$'' + ''^src/nix/get-env\.sh$'' + ''^tests/functional/build\.sh$'' + ''^tests/functional/ca/build-dry\.sh$'' + ''^tests/functional/ca/build-with-garbage-path\.sh$'' + ''^tests/functional/ca/common\.sh$'' + ''^tests/functional/ca/concurrent-builds\.sh$'' + ''^tests/functional/ca/eval-store\.sh$'' + ''^tests/functional/ca/gc\.sh$'' + ''^tests/functional/ca/import-derivation\.sh$'' + ''^tests/functional/ca/new-build-cmd\.sh$'' + ''^tests/functional/ca/nix-shell\.sh$'' + ''^tests/functional/ca/post-hook\.sh$'' + ''^tests/functional/ca/recursive\.sh$'' + ''^tests/functional/ca/repl\.sh$'' + ''^tests/functional/ca/selfref-gc\.sh$'' + ''^tests/functional/ca/why-depends\.sh$'' + ''^tests/functional/characterisation-test-infra\.sh$'' + ''^tests/functional/check\.sh$'' + ''^tests/functional/common/vars-and-functions\.sh$'' + ''^tests/functional/completions\.sh$'' + ''^tests/functional/compute-levels\.sh$'' + ''^tests/functional/config\.sh$'' + ''^tests/functional/db-migration\.sh$'' + ''^tests/functional/debugger\.sh$'' + ''^tests/functional/dependencies\.builder0\.sh$'' + ''^tests/functional/dependencies\.sh$'' + ''^tests/functional/dump-db\.sh$'' + ''^tests/functional/dyn-drv/build-built-drv\.sh$'' + ''^tests/functional/dyn-drv/common\.sh$'' + ''^tests/functional/dyn-drv/dep-built-drv\.sh$'' + ''^tests/functional/dyn-drv/eval-outputOf\.sh$'' + ''^tests/functional/dyn-drv/old-daemon-error-hack\.sh$'' + ''^tests/functional/dyn-drv/recursive-mod-json\.sh$'' + ''^tests/functional/eval-store\.sh$'' + ''^tests/functional/eval\.sh$'' + ''^tests/functional/export-graph\.sh$'' + ''^tests/functional/export\.sh$'' + ''^tests/functional/extra-sandbox-profile\.sh$'' + ''^tests/functional/fetchClosure\.sh$'' + ''^tests/functional/fetchGit\.sh$'' + ''^tests/functional/fetchGitRefs\.sh$'' + ''^tests/functional/fetchGitSubmodules\.sh$'' + ''^tests/functional/fetchGitVerification\.sh$'' + ''^tests/functional/fetchMercurial\.sh$'' + ''^tests/functional/fetchurl\.sh$'' + ''^tests/functional/fixed\.builder1\.sh$'' + ''^tests/functional/fixed\.builder2\.sh$'' + ''^tests/functional/fixed\.sh$'' + ''^tests/functional/flakes/absolute-paths\.sh$'' + ''^tests/functional/flakes/check\.sh$'' + ''^tests/functional/flakes/common\.sh$'' + ''^tests/functional/flakes/config\.sh$'' + ''^tests/functional/flakes/develop\.sh$'' + ''^tests/functional/flakes/flakes\.sh$'' + ''^tests/functional/flakes/follow-paths\.sh$'' + ''^tests/functional/flakes/prefetch\.sh$'' + ''^tests/functional/flakes/run\.sh$'' + ''^tests/functional/flakes/show\.sh$'' + ''^tests/functional/fmt\.sh$'' + ''^tests/functional/fmt\.simple\.sh$'' + ''^tests/functional/gc-auto\.sh$'' + ''^tests/functional/gc-concurrent\.builder\.sh$'' + ''^tests/functional/gc-concurrent\.sh$'' + ''^tests/functional/gc-concurrent2\.builder\.sh$'' + ''^tests/functional/gc-non-blocking\.sh$'' + ''^tests/functional/gc\.sh$'' + ''^tests/functional/git-hashing/common\.sh$'' + ''^tests/functional/git-hashing/simple\.sh$'' + ''^tests/functional/hash-convert\.sh$'' + ''^tests/functional/help\.sh$'' + ''^tests/functional/impure-derivations\.sh$'' + ''^tests/functional/impure-env\.sh$'' + ''^tests/functional/impure-eval\.sh$'' + ''^tests/functional/install-darwin\.sh$'' + ''^tests/functional/lang\.sh$'' + ''^tests/functional/legacy-ssh-store\.sh$'' + ''^tests/functional/linux-sandbox\.sh$'' + ''^tests/functional/local-overlay-store/add-lower-inner\.sh$'' + ''^tests/functional/local-overlay-store/add-lower\.sh$'' + ''^tests/functional/local-overlay-store/bad-uris\.sh$'' + ''^tests/functional/local-overlay-store/build-inner\.sh$'' + ''^tests/functional/local-overlay-store/build\.sh$'' + ''^tests/functional/local-overlay-store/check-post-init-inner\.sh$'' + ''^tests/functional/local-overlay-store/check-post-init\.sh$'' + ''^tests/functional/local-overlay-store/common\.sh$'' + ''^tests/functional/local-overlay-store/delete-duplicate-inner\.sh$'' + ''^tests/functional/local-overlay-store/delete-duplicate\.sh$'' + ''^tests/functional/local-overlay-store/delete-refs-inner\.sh$'' + ''^tests/functional/local-overlay-store/delete-refs\.sh$'' + ''^tests/functional/local-overlay-store/gc-inner\.sh$'' + ''^tests/functional/local-overlay-store/gc\.sh$'' + ''^tests/functional/local-overlay-store/optimise-inner\.sh$'' + ''^tests/functional/local-overlay-store/optimise\.sh$'' + ''^tests/functional/local-overlay-store/redundant-add-inner\.sh$'' + ''^tests/functional/local-overlay-store/redundant-add\.sh$'' + ''^tests/functional/local-overlay-store/remount\.sh$'' + ''^tests/functional/local-overlay-store/stale-file-handle-inner\.sh$'' + ''^tests/functional/local-overlay-store/stale-file-handle\.sh$'' + ''^tests/functional/local-overlay-store/verify-inner\.sh$'' + ''^tests/functional/local-overlay-store/verify\.sh$'' + ''^tests/functional/logging\.sh$'' + ''^tests/functional/misc\.sh$'' + ''^tests/functional/multiple-outputs\.sh$'' + ''^tests/functional/nar-access\.sh$'' + ''^tests/functional/nested-sandboxing\.sh$'' + ''^tests/functional/nested-sandboxing/command\.sh$'' + ''^tests/functional/nix-build\.sh$'' + ''^tests/functional/nix-channel\.sh$'' + ''^tests/functional/nix-collect-garbage-d\.sh$'' + ''^tests/functional/nix-copy-ssh-common\.sh$'' + ''^tests/functional/nix-copy-ssh-ng\.sh$'' + ''^tests/functional/nix-copy-ssh\.sh$'' + ''^tests/functional/nix-daemon-untrusting\.sh$'' + ''^tests/functional/nix-profile\.sh$'' + ''^tests/functional/nix-shell\.sh$'' + ''^tests/functional/nix_path\.sh$'' + ''^tests/functional/optimise-store\.sh$'' + ''^tests/functional/output-normalization\.sh$'' + ''^tests/functional/parallel\.builder\.sh$'' + ''^tests/functional/parallel\.sh$'' + ''^tests/functional/pass-as-file\.sh$'' + ''^tests/functional/path-from-hash-part\.sh$'' + ''^tests/functional/path-info\.sh$'' + ''^tests/functional/placeholders\.sh$'' + ''^tests/functional/plugins\.sh$'' + ''^tests/functional/post-hook\.sh$'' + ''^tests/functional/pure-eval\.sh$'' + ''^tests/functional/push-to-store-old\.sh$'' + ''^tests/functional/push-to-store\.sh$'' + ''^tests/functional/read-only-store\.sh$'' + ''^tests/functional/readfile-context\.sh$'' + ''^tests/functional/recursive\.sh$'' + ''^tests/functional/referrers\.sh$'' + ''^tests/functional/remote-store\.sh$'' + ''^tests/functional/repair\.sh$'' + ''^tests/functional/restricted\.sh$'' + ''^tests/functional/search\.sh$'' + ''^tests/functional/secure-drv-outputs\.sh$'' + ''^tests/functional/selfref-gc\.sh$'' + ''^tests/functional/shell\.sh$'' + ''^tests/functional/shell\.shebang\.sh$'' + ''^tests/functional/signing\.sh$'' + ''^tests/functional/simple\.builder\.sh$'' + ''^tests/functional/simple\.sh$'' + ''^tests/functional/ssh-relay\.sh$'' + ''^tests/functional/store-info\.sh$'' + ''^tests/functional/structured-attrs\.sh$'' + ''^tests/functional/substitute-with-invalid-ca\.sh$'' + ''^tests/functional/suggestions\.sh$'' + ''^tests/functional/supplementary-groups\.sh$'' + ''^tests/functional/tarball\.sh$'' + ''^tests/functional/test-infra\.sh$'' + ''^tests/functional/test-libstoreconsumer\.sh$'' + ''^tests/functional/timeout\.sh$'' + ''^tests/functional/toString-path\.sh$'' + ''^tests/functional/user-envs-migration\.sh$'' + ''^tests/functional/user-envs-test-case\.sh$'' + ''^tests/functional/user-envs\.builder\.sh$'' + ''^tests/functional/user-envs\.sh$'' + ''^tests/functional/why-depends\.sh$'' + ''^tests/functional/zstd\.sh$'' + ''^tests/unit/libutil/data/git/check-data\.sh$'' + ]; + }; + # TODO: nixfmt, https://github.com/NixOS/nixfmt/issues/153 + }; + }; + }; + + # We'll be pulling from this in the main flake + flake.getSystem = getSystem; +} diff --git a/maintainers/local.mk b/maintainers/local.mk new file mode 100644 index 000000000..88d594d67 --- /dev/null +++ b/maintainers/local.mk @@ -0,0 +1,15 @@ + +.PHONY: format +print-top-help += echo ' format: Format source code' + +# This uses the cached .pre-commit-hooks.yaml file +format: + @if ! type -p pre-commit &>/dev/null; then \ + echo "make format: pre-commit not found. Please use \`nix develop\`."; \ + exit 1; \ + fi; \ + if test -z "$$_NIX_PRE_COMMIT_HOOKS_CONFIG"; then \ + echo "make format: _NIX_PRE_COMMIT_HOOKS_CONFIG not set. Please use \`nix develop\`."; \ + exit 1; \ + fi; \ + pre-commit run --config $$_NIX_PRE_COMMIT_HOOKS_CONFIG --all-files diff --git a/maintainers/release-credits b/maintainers/release-credits new file mode 100755 index 000000000..7a5c87d7d --- /dev/null +++ b/maintainers/release-credits @@ -0,0 +1,184 @@ +#!/usr/bin/env nix +# vim: set filetype=python: +#!nix develop --impure --expr +#!nix `` +#!nix let flake = builtins.getFlake ("git+file://" + toString ../.); +#!nix pkgs = flake.inputs.nixpkgs.legacyPackages.${builtins.currentSystem}; +#!nix in pkgs.mkShell { nativeBuildInputs = [ +#!nix (pkgs.python3.withPackages (ps: with ps; [ requests ])) +#!nix ]; } +#!nix `` --command python3 + +# This script lists out the contributors for a given release. +# It must be run from the root of the Nix repository. + +import os +import sys +import json +import requests + +github_token = os.environ.get("GITHUB_TOKEN") +if not github_token: + print("GITHUB_TOKEN is not set. If you hit the rate limit, set it", file=sys.stderr) + # Might be ok, as we have a cache. + # raise ValueError("GITHUB_TOKEN must be set") + +# 1. Read the current version in .version +version = os.environ.get("VERSION") +if not version: + version = open(".version").read().strip() + +print(f"Generating release credits for Nix {version}", file=sys.stderr) + +# 2. Compute previous version +vcomponents = version.split(".") +if len(vcomponents) >= 2: + prev_version = f"{vcomponents[0]}.{int(vcomponents[1])-1}.0" +else: + raise ValueError(".version must have at least two components") + +# For unreleased versions +endref = "HEAD" +# For older releases +# endref = version + +# 2. Find the merge base between the current version and the previous version +mergeBase = os.popen(f"git merge-base {prev_version} {endref}").read().strip() +print(f"Merge base between {prev_version} and {endref} is {mergeBase}", file=sys.stderr) + +# 3. Find the date of the merge base +mergeBaseDate = os.popen(f"git show -s --format=%ci {mergeBase}").read().strip()[0:10] +print(f"Merge base date is {mergeBaseDate}", file=sys.stderr) + +# 4. Get the commits between the merge base and the current version + +def get_commits(): + raw = os.popen(f"git log --pretty=format:'%H\t%an\t%ae' {mergeBase}..{endref}").read().strip() + lines = raw.split("\n") + return [ { "hash": items[0], "author": items[1], "email": items[2] } + for line in lines + for items in (line.split("\t"),) + ] + +def commits_to_first_commit_by_email(commits): + by_email = dict() + for commit in commits: + email = commit["email"] + if email not in by_email: + by_email[email] = commit + return by_email + + +samples = commits_to_first_commit_by_email(get_commits()) + +# For quick testing, only pick two samples from the dict +# samples = dict(list(samples.items())[:2]) + +# Query the GitHub API to get handle +def get_github_commit(commit): + url = f"https://api.github.com/repos/NixOS/nix/commits/{commit['hash']}" + headers = {'Authorization': f'token {github_token}'} + response = requests.get(url, headers=headers) + response.raise_for_status() + return response.json() + +class Cache: + def __init__(self, filename, require = True): + self.filename = filename + try: + with open(filename, "r") as f: + self.values = json.load(f) + except FileNotFoundError: + if require: + raise + self.values = dict() + def save(self): + with open(self.filename, "w") as f: + json.dump(self.values, f, indent=4) + print(f"Saved cache to {self.filename}", file=sys.stderr) + +# The email to handle cache maps email addresses to either +# - a handle (string) +# - None (if no handle was found) +email_to_handle_cache = Cache("maintainers/data/release-credits-email-to-handle.json") + +handles = set() +emails = dict() + +for sample in samples: + s = samples[sample] + email = s["email"] + if not email in email_to_handle_cache.values: + print(f"Querying GitHub API for {s['hash']}, to get handle for {s['email']}") + ghc = get_github_commit(samples[sample]) + gha = ghc["author"] + if gha and gha["login"]: + handle = gha["login"] + print(f"Handle: {handle}") + email_to_handle_cache.values[email] = handle + else: + print(f"Found no handle for {s['email']}") + email_to_handle_cache.values[email] = None + handle = email_to_handle_cache.values[email] + if handle is not None: + handles.add(handle) + else: + emails[email] = s["author"] + +# print(email_to_handle_cache.values) + +email_to_handle_cache.save() + +handle_to_name_cache = Cache("maintainers/data/release-credits-handle-to-name.json") + +print(f"Found {len(handles)} handles", file=sys.stderr) + +for handle in handles: + if not handle in handle_to_name_cache.values: + print(f"Querying GitHub API for {handle}, to get name", file=sys.stderr) + url = f"https://api.github.com/users/{handle}" + headers = {'Authorization': f'token {github_token}'} + response = requests.get(url, headers=headers) + response.raise_for_status() + user = response.json() + name = user["name"] + print(f"Name: {name}", file=sys.stderr) + handle_to_name_cache.values[handle] = name + +handle_to_name_cache.save() + +entries = list() + +def is_bot(handle): + return handle.endswith("[bot]") + +for handle in handles: + name = handle_to_name_cache.values[handle] + if name is None: + # This way it looks more regular + name = handle + + # Note to bots: I'm only doing this because it would distract humans, and + # therefore delay your creation. I'm sorry; it's for your benefit. + if not is_bot(handle): + entries += [ f"- {name} [**(@{handle})**](https://github.com/{handle})" ] + +def shuffle(entries): + salt = os.urandom(16) + return sorted(entries, key=lambda x: hash((x, salt))) + +# Fair ordering is undecidable +entries = shuffle(entries) + +# For a sanity check, we could sort the entries by handle instead. +# entries = sorted(entries) + +print("") +print(f"This release was made possible by the following {len(entries)} contributors:") +print("") + +for entry in entries: + print(entry) + +for email in emails: + print(f"- {emails[email]}") diff --git a/maintainers/release-notes b/maintainers/release-notes index 2d84485c1..c0c4ee734 100755 --- a/maintainers/release-notes +++ b/maintainers/release-notes @@ -1,5 +1,6 @@ #!/usr/bin/env nix -#!nix shell .#changelog-d-nix --command bash +# vim: set filetype=bash: +#!nix shell .#changelog-d --command bash # --- CONFIGURATION --- @@ -151,6 +152,13 @@ section_title="Release $version_full ($DATE)" echo "# $section_title" echo changelog-d doc/manual/rl-next | sed -e 's/ *$//' + + if ! $IS_PATCH; then + echo + echo "# Contributors" + echo + VERSION=$version_full ./maintainers/release-credits + fi ) | tee -a $file log "Wrote $file" diff --git a/maintainers/release-process.md b/maintainers/release-process.md index da6886ea9..7a2b3c0a7 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -39,6 +39,10 @@ release: * Proof-read / edit / rearrange the release notes if needed. Breaking changes and highlights should go to the top. +* Run `maintainers/release-credits` to make sure the credits script works + and produces a sensible output. Some emails might not automatically map to + a GitHub handle. + * Push. ```console diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 4e2c379f0..731988568 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -11,6 +11,8 @@ use JSON::PP; use LWP::UserAgent; use Net::Amazon::S3; +delete $ENV{'shell'}; # shut up a LWP::UserAgent.pm warning + my $evalId = $ARGV[0] or die "Usage: $0 EVAL-ID\n"; my $releasesBucketName = "nix-releases"; @@ -36,11 +38,11 @@ sub fetch { my $evalUrl = "https://hydra.nixos.org/eval/$evalId"; my $evalInfo = decode_json(fetch($evalUrl, 'application/json')); #print Dumper($evalInfo); -my $flakeUrl = $evalInfo->{flake} or die; -my $flakeInfo = decode_json(`nix flake metadata --json "$flakeUrl"` or die); -my $nixRev = $flakeInfo->{revision} or die; +my $flakeUrl = $evalInfo->{flake}; +my $flakeInfo = decode_json(`nix flake metadata --json "$flakeUrl"` or die) if $flakeUrl; +my $nixRev = ($flakeInfo ? $flakeInfo->{revision} : $evalInfo->{jobsetevalinputs}->{nix}->{revision}) or die; -my $buildInfo = decode_json(fetch("$evalUrl/job/build.x86_64-linux", 'application/json')); +my $buildInfo = decode_json(fetch("$evalUrl/job/build.nix.x86_64-linux", 'application/json')); #print Dumper($buildInfo); my $releaseName = $buildInfo->{nixname}; @@ -83,12 +85,19 @@ my $channelsBucket = $s3_us->bucket($channelsBucketName) or die; sub getStorePath { my ($jobName, $output) = @_; my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json')); - return $buildInfo->{buildoutputs}->{$output or "out"}->{path} or die "cannot get store path for '$jobName'"; + return $buildInfo->{buildoutputs}->{$output or "out"}->{path} // die "cannot get store path for '$jobName'"; } sub copyManual { - my $manual = getStorePath("build.x86_64-linux", "doc"); - print "$manual\n"; + my $manual; + eval { + $manual = getStorePath("build.nix.x86_64-linux", "doc"); + }; + if ($@) { + warn "$@"; + return; + } + print "Manual: $manual\n"; my $manualNar = "$tmpDir/$releaseName-manual.nar.xz"; print "$manualNar\n"; @@ -154,19 +163,34 @@ downloadFile("binaryTarball.x86_64-linux", "1"); downloadFile("binaryTarball.aarch64-linux", "1"); downloadFile("binaryTarball.x86_64-darwin", "1"); downloadFile("binaryTarball.aarch64-darwin", "1"); -downloadFile("binaryTarballCross.x86_64-linux.armv6l-unknown-linux-gnueabihf", "1"); -downloadFile("binaryTarballCross.x86_64-linux.armv7l-unknown-linux-gnueabihf", "1"); +eval { + downloadFile("binaryTarballCross.x86_64-linux.armv6l-unknown-linux-gnueabihf", "1"); +}; +warn "$@" if $@; +eval { + downloadFile("binaryTarballCross.x86_64-linux.armv7l-unknown-linux-gnueabihf", "1"); +}; +warn "$@" if $@; +eval { + downloadFile("binaryTarballCross.x86_64-linux.riscv64-unknown-linux-gnu", "1"); +}; +warn "$@" if $@; downloadFile("installerScript", "1"); # Upload docker images to dockerhub. my $dockerManifest = ""; my $dockerManifestLatest = ""; +my $haveDocker = 0; for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) { my $system = $platforms->[0]; my $dockerPlatform = $platforms->[1]; my $fn = "nix-$version-docker-image-$dockerPlatform.tar.gz"; - downloadFile("dockerImage.$system", "1", $fn); + eval { + downloadFile("dockerImage.$system", "1", $fn); + }; + die "$@" if $@; + $haveDocker = 1; print STDERR "loading docker image for $dockerPlatform...\n"; system("docker load -i $tmpDir/$fn") == 0 or die; @@ -194,31 +218,34 @@ for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) { $dockerManifestLatest .= " --amend $latestTag" } -print STDERR "creating multi-platform docker manifest...\n"; -system("docker manifest rm nixos/nix:$version"); -system("docker manifest create nixos/nix:$version $dockerManifest") == 0 or die; -if ($isLatest) { - print STDERR "creating latest multi-platform docker manifest...\n"; - system("docker manifest rm nixos/nix:latest"); - system("docker manifest create nixos/nix:latest $dockerManifestLatest") == 0 or die; -} +if ($haveDocker) { + print STDERR "creating multi-platform docker manifest...\n"; + system("docker manifest rm nixos/nix:$version"); + system("docker manifest create nixos/nix:$version $dockerManifest") == 0 or die; + if ($isLatest) { + print STDERR "creating latest multi-platform docker manifest...\n"; + system("docker manifest rm nixos/nix:latest"); + system("docker manifest create nixos/nix:latest $dockerManifestLatest") == 0 or die; + } -print STDERR "pushing multi-platform docker manifest...\n"; -system("docker manifest push nixos/nix:$version") == 0 or die; + print STDERR "pushing multi-platform docker manifest...\n"; + system("docker manifest push nixos/nix:$version") == 0 or die; -if ($isLatest) { - print STDERR "pushing latest multi-platform docker manifest...\n"; - system("docker manifest push nixos/nix:latest") == 0 or die; + if ($isLatest) { + print STDERR "pushing latest multi-platform docker manifest...\n"; + system("docker manifest push nixos/nix:latest") == 0 or die; + } } # Upload nix-fallback-paths.nix. write_file("$tmpDir/fallback-paths.nix", "{\n" . - " x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" . - " i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" . - " aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" . - " x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" . - " aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" . + " x86_64-linux = \"" . getStorePath("build.nix.x86_64-linux") . "\";\n" . + " i686-linux = \"" . getStorePath("build.nix.i686-linux") . "\";\n" . + " aarch64-linux = \"" . getStorePath("build.nix.aarch64-linux") . "\";\n" . + " riscv64-linux = \"" . getStorePath("buildCross.nix.riscv64-unknown-linux-gnu.x86_64-linux") . "\";\n" . + " x86_64-darwin = \"" . getStorePath("build.nix.x86_64-darwin") . "\";\n" . + " aarch64-darwin = \"" . getStorePath("build.nix.aarch64-darwin") . "\";\n" . "}\n"); # Upload release files to S3. diff --git a/meson.build b/meson.build new file mode 100644 index 000000000..1554244ab --- /dev/null +++ b/meson.build @@ -0,0 +1,44 @@ +# This is just a stub project to include all the others as subprojects +# for development shell purposes + +project('nix-dev-shell', 'cpp', + version : files('.version'), + subproject_dir : 'src', +) + +# Internal Libraries +subproject('libutil') +subproject('libstore') +subproject('libfetchers') +subproject('libexpr') +subproject('libflake') +subproject('libmain') +subproject('libcmd') + +# Executables +subproject('nix') + +# Docs +subproject('internal-api-docs') +subproject('external-api-docs') + +# External C wrapper libraries +subproject('libutil-c') +subproject('libstore-c') +subproject('libexpr-c') +subproject('libmain-c') + +# Language Bindings +if not meson.is_cross_build() + subproject('perl') +endif + +# Testing +subproject('nix-util-test-support') +subproject('nix-util-tests') +subproject('nix-store-test-support') +subproject('nix-store-tests') +subproject('nix-fetchers-tests') +subproject('nix-expr-test-support') +subproject('nix-expr-tests') +subproject('nix-flake-tests') diff --git a/misc/changelog-d.cabal.nix b/misc/changelog-d.cabal.nix deleted file mode 100644 index 76f9353cd..000000000 --- a/misc/changelog-d.cabal.nix +++ /dev/null @@ -1,31 +0,0 @@ -{ mkDerivation, aeson, base, bytestring, cabal-install-parsers -, Cabal-syntax, containers, directory, filepath, frontmatter -, generic-lens-lite, lib, mtl, optparse-applicative, parsec, pretty -, regex-applicative, text, pkgs -}: -let rev = "f30f6969e9cd8b56242309639d58acea21c99d06"; -in -mkDerivation { - pname = "changelog-d"; - version = "0.1"; - src = pkgs.fetchurl { - name = "changelog-d-${rev}.tar.gz"; - url = "https://codeberg.org/roberth/changelog-d/archive/${rev}.tar.gz"; - hash = "sha256-8a2+i5u7YoszAgd5OIEW0eYUcP8yfhtoOIhLJkylYJ4="; - } // { inherit rev; }; - isLibrary = false; - isExecutable = true; - libraryHaskellDepends = [ - aeson base bytestring cabal-install-parsers Cabal-syntax containers - directory filepath frontmatter generic-lens-lite mtl parsec pretty - regex-applicative text - ]; - executableHaskellDepends = [ - base bytestring Cabal-syntax directory filepath - optparse-applicative - ]; - doHaddock = false; - description = "Concatenate changelog entries into a single one"; - license = lib.licenses.gpl3Plus; - mainProgram = "changelog-d"; -} diff --git a/misc/changelog-d.nix b/misc/changelog-d.nix deleted file mode 100644 index 1b20f4596..000000000 --- a/misc/changelog-d.nix +++ /dev/null @@ -1,31 +0,0 @@ -# Taken temporarily from -{ - callPackage, - lib, - haskell, - haskellPackages, -}: - -let - hsPkg = haskellPackages.callPackage ./changelog-d.cabal.nix { }; - - addCompletions = haskellPackages.generateOptparseApplicativeCompletions ["changelog-d"]; - - haskellModifications = - lib.flip lib.pipe [ - addCompletions - haskell.lib.justStaticExecutables - ]; - - mkDerivationOverrides = finalAttrs: oldAttrs: { - - version = oldAttrs.version + "-git-${lib.strings.substring 0 7 oldAttrs.src.rev}"; - - meta = oldAttrs.meta // { - homepage = "https://codeberg.org/roberth/changelog-d"; - maintainers = [ lib.maintainers.roberth ]; - }; - - }; -in - (haskellModifications hsPkg).overrideAttrs mkDerivationOverrides diff --git a/misc/systemv/nix-daemon b/misc/systemv/nix-daemon index fea537167..e8326f947 100755 --- a/misc/systemv/nix-daemon +++ b/misc/systemv/nix-daemon @@ -34,6 +34,7 @@ else fi # Source function library. +# shellcheck source=/dev/null . /etc/init.d/functions LOCKFILE=/var/lock/subsys/nix-daemon @@ -41,14 +42,20 @@ RUNDIR=/var/run/nix PIDFILE=${RUNDIR}/nix-daemon.pid RETVAL=0 -base=${0##*/} +# https://www.shellcheck.net/wiki/SC3004 +# Check if gettext exists +if ! type gettext > /dev/null 2>&1 +then + # If not, create a dummy function that returns the input verbatim + gettext() { printf '%s' "$1"; } +fi start() { mkdir -p ${RUNDIR} chown ${NIX_DAEMON_USER}:${NIX_DAEMON_USER} ${RUNDIR} - echo -n $"Starting nix daemon... " + printf '%s' "$(gettext 'Starting nix daemon... ')" daemonize -u $NIX_DAEMON_USER -p ${PIDFILE} $NIX_DAEMON_BIN $NIX_DAEMON_OPTS RETVAL=$? @@ -58,7 +65,7 @@ start() { } stop() { - echo -n $"Shutting down nix daemon: " + printf '%s' "$(gettext 'Shutting down nix daemon: ')" killproc -p ${PIDFILE} $NIX_DAEMON_BIN RETVAL=$? [ $RETVAL -eq 0 ] && rm -f ${LOCKFILE} ${PIDFILE} @@ -67,7 +74,7 @@ stop() { } reload() { - echo -n $"Reloading nix daemon... " + printf '%s' "$(gettext 'Reloading nix daemon... ')" killproc -p ${PIDFILE} $NIX_DAEMON_BIN -HUP RETVAL=$? echo @@ -105,7 +112,7 @@ case "$1" in fi ;; *) - echo $"Usage: $0 {start|stop|status|restart|condrestart}" + printf '%s' "$(gettext "Usage: $0 {start|stop|status|restart|condrestart}")" exit 2 ;; esac diff --git a/mk/common-test.sh b/mk/common-test.sh index 2783d293b..817422c40 100644 --- a/mk/common-test.sh +++ b/mk/common-test.sh @@ -1,27 +1,23 @@ +# shellcheck shell=bash + # Remove overall test dir (at most one of the two should match) and # remove file extension. -test_name=$(echo -n "$test" | sed \ - -e "s|^tests/unit/[^/]*/data/||" \ + +test_name=$(echo -n "${test?must be defined by caller (test runner)}" | sed \ + -e "s|^src/[^/]*-test/data/||" \ -e "s|^tests/functional/||" \ -e "s|\.sh$||" \ ) +# shellcheck disable=SC2016 TESTS_ENVIRONMENT=( "TEST_NAME=$test_name" 'NIX_REMOTE=' 'PS4=+(${BASH_SOURCE[0]-$0}:$LINENO) ' ) -: ${BASH:=/usr/bin/env bash} +read -r -a bash <<< "${BASH:-/usr/bin/env bash}" run () { - cd "$(dirname $1)" && env "${TESTS_ENVIRONMENT[@]}" $BASH -x -e -u -o pipefail $(basename $1) -} - -init_test () { - run "$init" 2>/dev/null > /dev/null -} - -run_test_proper () { - run "$test" + cd "$(dirname "$1")" && env "${TESTS_ENVIRONMENT[@]}" "${bash[@]}" -x -e -u -o pipefail "$(basename "$1")" } diff --git a/mk/compilation-database.mk b/mk/compilation-database.mk new file mode 100644 index 000000000..f69dc0de0 --- /dev/null +++ b/mk/compilation-database.mk @@ -0,0 +1,11 @@ +compile-commands-json-files := + +define write-compile-commands + _srcs := $$(sort $$(foreach src, $$($(1)_SOURCES), $$(src))) + + $(1)_COMPILE_COMMANDS_JSON := $$(addprefix $(buildprefix), $$(addsuffix .compile_commands.json, $$(basename $$(_srcs)))) + + compile-commands-json-files += $$($(1)_COMPILE_COMMANDS_JSON) + + clean-files += $$($(1)_COMPILE_COMMANDS_JSON) +endef diff --git a/mk/cxx-big-literal.mk b/mk/cxx-big-literal.mk index 85365df8e..d64a171c8 100644 --- a/mk/cxx-big-literal.mk +++ b/mk/cxx-big-literal.mk @@ -1,5 +1,5 @@ %.gen.hh: % - @echo 'R"foo(' >> $@.tmp + @echo 'R"__NIX_STR(' >> $@.tmp $(trace-gen) cat $< >> $@.tmp - @echo ')foo"' >> $@.tmp + @echo ')__NIX_STR"' >> $@.tmp @mv $@.tmp $@ diff --git a/mk/debug-test.sh b/mk/debug-test.sh index 52482c01e..0dd4406c3 100755 --- a/mk/debug-test.sh +++ b/mk/debug-test.sh @@ -3,12 +3,8 @@ set -eu -o pipefail test=$1 -init=${2-} dir="$(dirname "${BASH_SOURCE[0]}")" source "$dir/common-test.sh" -if [ -n "$init" ]; then - (init_test) -fi -run_test_proper +run "$test" diff --git a/mk/lib.mk b/mk/lib.mk index fe0add1c9..1e7af6ad5 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -68,6 +68,7 @@ include mk/patterns.mk include mk/templates.mk include mk/cxx-big-literal.mk include mk/tests.mk +include mk/compilation-database.mk # Include all sub-Makefiles. @@ -86,17 +87,23 @@ $(foreach script, $(bin-scripts), $(eval $(call install-program-in,$(script),$(b $(foreach script, $(bin-scripts), $(eval programs-list += $(script))) $(foreach script, $(noinst-scripts), $(eval programs-list += $(script))) $(foreach template, $(template-files), $(eval $(call instantiate-template,$(template)))) -install_test_init=tests/functional/init.sh $(foreach test, $(install-tests), \ - $(eval $(call run-test,$(test),$(install_test_init))) \ + $(eval $(call run-test,$(test))) \ $(eval installcheck: $(test).test)) $(foreach test-group, $(install-tests-groups), \ - $(eval $(call run-test-group,$(test-group),$(install_test_init))) \ + $(eval $(call run-test-group,$(test-group))) \ $(eval installcheck: $(test-group).test-group) \ $(foreach test, $($(test-group)-tests), \ - $(eval $(call run-test,$(test),$(install_test_init))) \ + $(eval $(call run-test,$(test))) \ $(eval $(test-group).test-group: $(test).test))) +# Compilation database. +$(foreach lib, $(libraries), $(eval $(call write-compile-commands,$(lib)))) +$(foreach prog, $(programs), $(eval $(call write-compile-commands,$(prog)))) + +compile_commands.json: $(compile-commands-json-files) + @jq --slurp '.' $^ >$@ + # Include makefiles requiring built programs. $(foreach mf, $(makefiles-late), $(eval $(call include-sub-makefile,$(mf)))) diff --git a/mk/patterns.mk b/mk/patterns.mk index c81150260..4caa2039e 100644 --- a/mk/patterns.mk +++ b/mk/patterns.mk @@ -1,11 +1,41 @@ + +# These are the complete command lines we use to compile C and C++ files. +# - $< is the source file. +# - $1 is the object file to create. +CC_CMD=$(CC) -o $1 -c $< $(CPPFLAGS) $(GLOBAL_CFLAGS) $(CFLAGS) $($1_CFLAGS) -MMD -MF $(call filename-to-dep,$1) -MP +CXX_CMD=$(CXX) -o $1 -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($1_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep,$1) -MP + +# We use COMPILE_COMMANDS_JSON_CMD to turn a compilation command (like CC_CMD +# or CXX_CMD above) into a comple_commands.json file. We rely on bash native +# word splitting to define the positional arguments. +# - $< is the source file being compiled. +COMPILE_COMMANDS_JSON_CMD=jq --null-input '{ directory: $$ENV.PWD, file: "$<", arguments: $$ARGS.positional }' --args -- + + $(buildprefix)%.o: %.cc @mkdir -p "$(dir $@)" - $(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cxx) $(call CXX_CMD,$@) $(buildprefix)%.o: %.cpp @mkdir -p "$(dir $@)" - $(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cxx) $(call CXX_CMD,$@) $(buildprefix)%.o: %.c @mkdir -p "$(dir $@)" - $(trace-cc) $(CC) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CFLAGS) $(CFLAGS) $($@_CFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cc) $(call CC_CMD,$@) + +# In the following we need to replace the .compile_commands.json extension in $@ with .o +# to make the object file. This is needed because CC_CMD and CXX_CMD do further expansions +# based on the object file name (i.e. *_CXXFLAGS and filename-to-dep). + +$(buildprefix)%.compile_commands.json: %.cc + @mkdir -p "$(dir $@)" + $(trace-jq) $(COMPILE_COMMANDS_JSON_CMD) $(call CXX_CMD,$(@:.compile_commands.json=.o)) > $@ + +$(buildprefix)%.compile_commands.json: %.cpp + @mkdir -p "$(dir $@)" + $(trace-jq) $(COMPILE_COMMANDS_JSON_CMD) $(call CXX_CMD,$(@:.compile_commands.json=.o)) > $@ + +$(buildprefix)%.compile_commands.json: %.c + @mkdir -p "$(dir $@)" + $(trace-jq) $(COMPILE_COMMANDS_JSON_CMD) $(call CC_CMD,$(@:.compile_commands.json=.o)) > $@ diff --git a/mk/platform.mk b/mk/platform.mk index fe960dedf..22c114a20 100644 --- a/mk/platform.mk +++ b/mk/platform.mk @@ -29,4 +29,8 @@ ifdef HOST_OS HOST_SOLARIS = 1 HOST_UNIX = 1 endif + ifeq ($(HOST_KERNEL), gnu) + HOST_HURD = 1 + HOST_UNIX = 1 + endif endif diff --git a/mk/precompiled-headers.mk b/mk/precompiled-headers.mk index cdd3daecd..f2803eb79 100644 --- a/mk/precompiled-headers.mk +++ b/mk/precompiled-headers.mk @@ -8,7 +8,7 @@ GCH = $(buildprefix)precompiled-headers.h.gch $(GCH): precompiled-headers.h @rm -f $@ @mkdir -p "$(dir $@)" - $(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS) + $(trace-gen) $(CXX) -c -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS) clean-files += $(GCH) diff --git a/mk/run-test.sh b/mk/run-test.sh index da9c5a473..7f9f1d5f8 100755 --- a/mk/run-test.sh +++ b/mk/run-test.sh @@ -8,7 +8,6 @@ yellow="" normal="" test=$1 -init=${2-} dir="$(dirname "${BASH_SOURCE[0]}")" source "$dir/common-test.sh" @@ -22,20 +21,18 @@ if [ -t 1 ]; then fi run_test () { - if [ -n "$init" ]; then - (init_test 2>/dev/null > /dev/null) - fi - log="$(run_test_proper 2>&1)" && status=0 || status=$? + log="$(run "$test" 2>&1)" && status=0 || status=$? } run_test -if [ $status -eq 0 ]; then +if [[ "$status" = 0 ]]; then echo "$post_run_msg [${green}PASS$normal]" -elif [ $status -eq 99 ]; then +elif [[ "$status" = 77 ]]; then echo "$post_run_msg [${yellow}SKIP$normal]" else echo "$post_run_msg [${red}FAIL$normal]" + # shellcheck disable=SC2001 echo "$log" | sed 's/^/ /' exit "$status" fi diff --git a/mk/tests.mk b/mk/tests.mk index bac9b704a..0a10f6d3b 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -12,8 +12,8 @@ endef define run-test - $(eval $(call run-bash,$1.test,$1 $(test-deps),mk/run-test.sh $1 $2)) - $(eval $(call run-bash,$1.test-debug,$1 $(test-deps),mk/debug-test.sh $1 $2)) + $(eval $(call run-bash,$1.test,$1 $(test-deps),mk/run-test.sh $1)) + $(eval $(call run-bash,$1.test-debug,$1 $(test-deps),mk/debug-test.sh $1)) endef diff --git a/mk/tracing.mk b/mk/tracing.mk index 1fc5573d7..09db1e617 100644 --- a/mk/tracing.mk +++ b/mk/tracing.mk @@ -10,6 +10,8 @@ ifeq ($(V), 0) trace-install = @echo " INST " $@; trace-mkdir = @echo " MKDIR " $@; trace-test = @echo " TEST " $@; + trace-sh = @echo " SH " $@; + trace-jq = @echo " JQ " $@; suppress = @ diff --git a/package.nix b/package.nix index 20796a386..a7c8923e8 100644 --- a/package.nix +++ b/package.nix @@ -13,17 +13,16 @@ , curl , editline , readline -, fileset , flex , git , gtest , jq -, doxygen , libarchive , libcpuid , libgit2 , libseccomp , libsodium +, man , lowdown , mdbook , mdbook-linkcheck @@ -33,7 +32,8 @@ , pkg-config , rapidcheck , sqlite -, util-linux +, toml11 +, unixtools , xz , busybox-sandbox-shell ? null @@ -48,10 +48,8 @@ , pname ? "nix" , versionSuffix ? "" -, officialRelease ? false -# Whether to build Nix. Useful to skip for tasks like (a) just -# generating API docs or (b) testing existing pre-built versions of Nix +# Whether to build Nix. Useful to skip for tasks like testing existing pre-built versions of Nix , doBuild ? true # Run the unit tests as part of the build. See `installUnitTests` for an @@ -74,7 +72,10 @@ # sounds so long as evaluation just takes places within short-lived # processes. (When the process exits, the memory is reclaimed; it is # only leaked *within* the process.) -, enableGC ? true +# +# Temporarily disabled on Windows because the `GC_throw_bad_alloc` +# symbol is missing during linking. +, enableGC ? !stdenv.hostPlatform.isWindows # Whether to enable Markdown rendering in the Nix binary. , enableMarkdown ? !stdenv.hostPlatform.isWindows @@ -87,10 +88,6 @@ # - readline , readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline" -# Whether to build the internal API docs, can be done separately from -# everything else. -, enableInternalAPIDocs ? false - # Whether to install unit tests. This is useful when cross compiling # since we cannot run them natively during the build, but can do so # later. @@ -113,6 +110,8 @@ }: let + inherit (lib) fileset; + version = lib.fileContents ./.version + versionSuffix; # selected attributes with defaults, will be used to define some @@ -161,6 +160,8 @@ in { ./m4 # TODO: do we really need README.md? It doesn't seem used in the build. ./README.md + # This could be put behind a conditional + ./maintainers/local.mk # For make, regardless of what we are building ./local.mk ./Makefile @@ -171,17 +172,11 @@ in { ./doc ./misc ./precompiled-headers.h - ./src + (fileset.difference ./src ./src/perl) ./COPYING ./scripts/local.mk - ] ++ lib.optionals buildUnitTests [ + ] ++ lib.optionals enableManual [ ./doc/manual - ] ++ lib.optionals enableInternalAPIDocs [ - ./doc/internal-api - # Source might not be compiled, but still must be available - # for Doxygen to gather comments. - ./src - ./tests/unit ] ++ lib.optionals buildUnitTests [ ./tests/unit ] ++ lib.optionals doInstallCheck [ @@ -195,8 +190,10 @@ in { ++ lib.optional doBuild "dev" # If we are doing just build or just docs, the one thing will use # "out". We only need additional outputs if we are doing both. - ++ lib.optional (doBuild && (enableManual || enableInternalAPIDocs)) "doc" - ++ lib.optional installUnitTests "check"; + ++ lib.optional (doBuild && enableManual) "doc" + ++ lib.optional installUnitTests "check" + ++ lib.optional doCheck "testresults" + ; nativeBuildInputs = [ autoconf-archive @@ -213,14 +210,14 @@ in { git mercurial openssh + man # for testing `nix-* --help` ] ++ lib.optionals (doInstallCheck || enableManual) [ jq # Also for custom mdBook preprocessor. - ] ++ lib.optional stdenv.hostPlatform.isLinux util-linux - ++ lib.optional enableInternalAPIDocs doxygen + ] ++ lib.optional stdenv.hostPlatform.isStatic unixtools.hexdump ; - buildInputs = lib.optionals doBuild [ - boost + buildInputs = lib.optionals doBuild ( + [ brotli bzip2 curl @@ -229,6 +226,7 @@ in { libsodium openssl sqlite + toml11 xz ({ inherit readline editline; }.${readlineFlavor}) ] ++ lib.optionals enableMarkdown [ @@ -240,46 +238,22 @@ in { ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid # There have been issues building these dependencies ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin)) - (aws-sdk-cpp.override { - apis = ["s3" "transfer"]; - customMemoryManagement = false; - }) - ; + aws-sdk-cpp + ); - propagatedBuildInputs = [ + propagatedBuildInputs = lib.optionals doBuild ([ + boost nlohmann_json - ] ++ lib.optional enableGC boehmgc; + ] ++ lib.optional enableGC boehmgc + ); dontBuild = !attrs.doBuild; doCheck = attrs.doCheck; - disallowedReferences = [ boost ]; - - preConfigure = lib.optionalString (doBuild && ! stdenv.hostPlatform.isStatic) ( - '' - # Copy libboost_context so we don't get all of Boost in our closure. - # https://github.com/NixOS/nixpkgs/issues/45462 - mkdir -p $out/lib - cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib - rm -f $out/lib/*.a - '' + lib.optionalString stdenv.hostPlatform.isLinux '' - chmod u+w $out/lib/*.so.* - patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* - '' + lib.optionalString stdenv.hostPlatform.isDarwin '' - for LIB in $out/lib/*.dylib; do - chmod u+w $LIB - install_name_tool -id $LIB $LIB - install_name_tool -delete_rpath ${boost}/lib/ $LIB || true - done - install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib - '' - ); - configureFlags = [ (lib.enableFeature doBuild "build") (lib.enableFeature buildUnitTests "unit-tests") (lib.enableFeature doInstallCheck "functional-tests") - (lib.enableFeature enableInternalAPIDocs "internal-api-docs") (lib.enableFeature enableManual "doc-gen") (lib.enableFeature enableGC "gc") (lib.enableFeature enableMarkdown "markdown") @@ -303,8 +277,11 @@ in { makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1"; - installTargets = lib.optional doBuild "install" - ++ lib.optional enableInternalAPIDocs "internal-api-html"; + preCheck = '' + mkdir $testresults + ''; + + installTargets = lib.optional doBuild "install"; installFlags = "sysconfdir=$(out)/etc"; @@ -319,18 +296,16 @@ in { lib.optionalString stdenv.hostPlatform.isStatic '' mkdir -p $out/nix-support echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products - '' + lib.optionalString stdenv.isDarwin '' - install_name_tool \ - -change ${boost}/lib/libboost_context.dylib \ - $out/lib/libboost_context.dylib \ - $out/lib/libnixutil.dylib '' ) + lib.optionalString enableManual '' mkdir -p ''${!outputDoc}/nix-support echo "doc manual ''${!outputDoc}/share/doc/nix/manual" >> ''${!outputDoc}/nix-support/hydra-build-products - '' + lib.optionalString enableInternalAPIDocs '' - mkdir -p ''${!outputDoc}/nix-support - echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products + ''; + + # So the check output gets links for DLLs in the out output. + preFixup = lib.optionalString (stdenv.hostPlatform.isWindows && builtins.elem "check" finalAttrs.outputs) '' + ln -s "$check/lib/"*.dll "$check/bin" + ln -s "$out/bin/"*.dll "$check/bin" ''; doInstallCheck = attrs.doInstallCheck; @@ -341,20 +316,26 @@ in { # Work around weird bug where it doesn't think there is a Makefile. installCheckPhase = if (!doBuild && doInstallCheck) then '' + runHook preInstallCheck mkdir -p src/nix-channel make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES '' else null; # Needed for tests if we are not doing a build, but testing existing # built Nix. - preInstallCheck = lib.optionalString (! doBuild) '' - mkdir -p src/nix-channel - ''; + preInstallCheck = + lib.optionalString (! doBuild) '' + mkdir -p src/nix-channel + '' + # See https://github.com/NixOS/nix/issues/2523 + # Occurs often in tests since https://github.com/NixOS/nix/pull/9900 + + lib.optionalString stdenv.hostPlatform.isDarwin '' + export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES + ''; separateDebugInfo = !stdenv.hostPlatform.isStatic; - # TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated - # to work with `strictDeps`. + # TODO Always true after https://github.com/NixOS/nixpkgs/issues/318564 strictDeps = !withCoverageChecks; hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; diff --git a/packaging/components.nix b/packaging/components.nix new file mode 100644 index 000000000..870e9ae61 --- /dev/null +++ b/packaging/components.nix @@ -0,0 +1,43 @@ +scope: +let + inherit (scope) callPackage; +in + +# This becomes the pkgs.nixComponents attribute set +{ + nix = callPackage ../package.nix { }; + + nix-util = callPackage ../src/libutil/package.nix { }; + nix-util-c = callPackage ../src/libutil-c/package.nix { }; + nix-util-test-support = callPackage ../tests/unit/libutil-support/package.nix { }; + nix-util-tests = callPackage ../tests/unit/libutil/package.nix { }; + + nix-store = callPackage ../src/libstore/package.nix { }; + nix-store-c = callPackage ../src/libstore-c/package.nix { }; + nix-store-test-support = callPackage ../tests/unit/libstore-support/package.nix { }; + nix-store-tests = callPackage ../tests/unit/libstore/package.nix { }; + + nix-fetchers = callPackage ../src/libfetchers/package.nix { }; + nix-fetchers-tests = callPackage ../tests/unit/libfetchers/package.nix { }; + + nix-expr = callPackage ../src/libexpr/package.nix { }; + nix-expr-c = callPackage ../src/libexpr-c/package.nix { }; + nix-expr-test-support = callPackage ../tests/unit/libexpr-support/package.nix { }; + nix-expr-tests = callPackage ../tests/unit/libexpr/package.nix { }; + + nix-flake = callPackage ../src/libflake/package.nix { }; + nix-flake-tests = callPackage ../tests/unit/libflake/package.nix { }; + + nix-main = callPackage ../src/libmain/package.nix { }; + nix-main-c = callPackage ../src/libmain-c/package.nix { }; + + nix-cmd = callPackage ../src/libcmd/package.nix { }; + + # Will replace `nix` once the old build system is gone. + nix-ng = callPackage ../src/nix/package.nix { }; + + nix-internal-api-docs = callPackage ../src/internal-api-docs/package.nix { }; + nix-external-api-docs = callPackage ../src/external-api-docs/package.nix { }; + + nix-perl-bindings = callPackage ../src/perl/package.nix { }; +} diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix new file mode 100644 index 000000000..e0737593f --- /dev/null +++ b/packaging/dependencies.nix @@ -0,0 +1,165 @@ +# These overrides are applied to the dependencies of the Nix components. + +{ + # Flake inputs; used for sources + inputs, + + # The raw Nixpkgs, not affected by this scope + pkgs, + + stdenv, + versionSuffix, +}: + +let + prevStdenv = stdenv; +in + +let + inherit (pkgs) lib; + + root = ../.; + + stdenv = if prevStdenv.isDarwin && prevStdenv.isx86_64 + then darwinStdenv + else prevStdenv; + + # Fix the following error with the default x86_64-darwin SDK: + # + # error: aligned allocation function of type 'void *(std::size_t, std::align_val_t)' is only available on macOS 10.13 or newer + # + # Despite the use of the 10.13 deployment target here, the aligned + # allocation function Clang uses with this setting actually works + # all the way back to 10.6. + darwinStdenv = pkgs.overrideSDK prevStdenv { darwinMinVersion = "10.13"; }; + + # Nixpkgs implements this by returning a subpath into the fetched Nix sources. + resolvePath = p: p; + + # Indirection for Nixpkgs to override when package.nix files are vendored + filesetToSource = lib.fileset.toSource; + + localSourceLayer = finalAttrs: prevAttrs: + let + workDirPath = + # Ideally we'd pick finalAttrs.workDir, but for now `mkDerivation` has + # the requirement that everything except passthru and meta must be + # serialized by mkDerivation, which doesn't work for this. + prevAttrs.workDir; + + workDirSubpath = lib.path.removePrefix root workDirPath; + sources = assert prevAttrs.fileset._type == "fileset"; prevAttrs.fileset; + src = lib.fileset.toSource { fileset = sources; inherit root; }; + + in + { + sourceRoot = "${src.name}/" + workDirSubpath; + inherit src; + + # Clear what `derivation` can't/shouldn't serialize; see prevAttrs.workDir. + fileset = null; + workDir = null; + }; + + # Work around weird `--as-needed` linker behavior with BSD, see + # https://github.com/mesonbuild/meson/issues/3593 + bsdNoLinkAsNeeded = finalAttrs: prevAttrs: + lib.optionalAttrs stdenv.hostPlatform.isBSD { + mesonFlags = [ (lib.mesonBool "b_asneeded" false) ] ++ prevAttrs.mesonFlags or []; + }; + + miscGoodPractice = finalAttrs: prevAttrs: + { + strictDeps = prevAttrs.strictDeps or true; + enableParallelBuilding = true; + }; + +in +scope: { + inherit stdenv versionSuffix; + version = lib.fileContents ../.version + versionSuffix; + + aws-sdk-cpp = (pkgs.aws-sdk-cpp.override { + apis = [ "s3" "transfer" ]; + customMemoryManagement = false; + }).overrideAttrs { + # only a stripped down version is built, which takes a lot less resources + # to build, so we don't need a "big-parallel" machine. + requiredSystemFeatures = [ ]; + }; + + libseccomp = pkgs.libseccomp.overrideAttrs (_: rec { + version = "2.5.5"; + src = pkgs.fetchurl { + url = "https://github.com/seccomp/libseccomp/releases/download/v${version}/libseccomp-${version}.tar.gz"; + hash = "sha256-JIosik2bmFiqa69ScSw0r+/PnJ6Ut23OAsHJqiX7M3U="; + }; + }); + + boehmgc = pkgs.boehmgc.override { + enableLargeConfig = true; + }; + + # TODO Hack until https://github.com/NixOS/nixpkgs/issues/45462 is fixed. + boost = (pkgs.boost.override { + extraB2Args = [ + "--with-container" + "--with-context" + "--with-coroutine" + ]; + }).overrideAttrs (old: { + # Need to remove `--with-*` to use `--with-libraries=...` + buildPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.buildPhase; + installPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.installPhase; + }); + + libgit2 = pkgs.libgit2.overrideAttrs (attrs: { + src = inputs.libgit2; + version = inputs.libgit2.lastModifiedDate; + cmakeFlags = attrs.cmakeFlags or [] + ++ [ "-DUSE_SSH=exec" ]; + }); + + busybox-sandbox-shell = pkgs.busybox-sandbox-shell or (pkgs.busybox.override { + useMusl = true; + enableStatic = true; + enableMinimal = true; + extraConfig = '' + CONFIG_FEATURE_FANCY_ECHO y + CONFIG_FEATURE_SH_MATH y + CONFIG_FEATURE_SH_MATH_64 y + + CONFIG_ASH y + CONFIG_ASH_OPTIMIZE_FOR_SIZE y + + CONFIG_ASH_ALIAS y + CONFIG_ASH_BASH_COMPAT y + CONFIG_ASH_CMDCMD y + CONFIG_ASH_ECHO y + CONFIG_ASH_GETOPTS y + CONFIG_ASH_INTERNAL_GLOB y + CONFIG_ASH_JOB_CONTROL y + CONFIG_ASH_PRINTF y + CONFIG_ASH_TEST y + ''; + }); + + # TODO change in Nixpkgs, Windows works fine. First commit of + # https://github.com/NixOS/nixpkgs/pull/322977 backported will fix. + toml11 = pkgs.toml11.overrideAttrs (old: { + meta.platforms = lib.platforms.all; + }); + + inherit resolvePath filesetToSource; + + mkMesonDerivation = f: let + exts = [ + miscGoodPractice + bsdNoLinkAsNeeded + localSourceLayer + ]; + in stdenv.mkDerivation + (lib.extends + (lib.foldr lib.composeExtensions (_: _: {}) exts) + f); +} diff --git a/packaging/hydra.nix b/packaging/hydra.nix new file mode 100644 index 000000000..dbe992476 --- /dev/null +++ b/packaging/hydra.nix @@ -0,0 +1,200 @@ +{ inputs +, binaryTarball +, forAllCrossSystems +, forAllSystems +, lib +, linux64BitSystems +, nixpkgsFor +, self +}: +let + inherit (inputs) nixpkgs nixpkgs-regression; + + installScriptFor = tarballs: + nixpkgsFor.x86_64-linux.native.callPackage ../scripts/installer.nix { + inherit tarballs; + }; + + testNixVersions = pkgs: client: daemon: + pkgs.callPackage ../package.nix { + pname = + "nix-tests" + + lib.optionalString + (lib.versionAtLeast daemon.version "2.4pre20211005" && + lib.versionAtLeast client.version "2.4pre20211005") + "-${client.version}-against-${daemon.version}"; + + test-client = client; + test-daemon = daemon; + + doBuild = false; + }; + + # Technically we could just return `pkgs.nixComponents`, but for Hydra it's + # convention to transpose it, and to transpose it efficiently, we need to + # enumerate them manually, so that we don't evaluate unnecessary package sets. + forAllPackages = lib.genAttrs [ + "nix" + "nix-util" + "nix-util-c" + "nix-util-test-support" + "nix-util-tests" + "nix-store" + "nix-store-c" + "nix-store-test-support" + "nix-store-tests" + "nix-fetchers" + "nix-fetchers-tests" + "nix-expr" + "nix-expr-c" + "nix-expr-test-support" + "nix-expr-tests" + "nix-flake" + "nix-flake-tests" + "nix-main" + "nix-main-c" + "nix-cmd" + "nix-ng" + ]; +in +{ + # Binary package for various platforms. + build = forAllPackages (pkgName: + forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.${pkgName})); + + shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation); + + buildStatic = forAllPackages (pkgName: + lib.genAttrs linux64BitSystems (system: nixpkgsFor.${system}.static.nixComponents.${pkgName})); + + buildCross = forAllPackages (pkgName: + forAllCrossSystems (crossSystem: + lib.genAttrs [ "x86_64-linux" ] (system: nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName}))); + + buildNoGc = forAllSystems (system: + self.packages.${system}.nix.override { enableGC = false; } + ); + + buildNoTests = forAllSystems (system: nixpkgsFor.${system}.native.nix_noTests); + + # Toggles some settings for better coverage. Windows needs these + # library combinations, and Debian build Nix with GNU readline too. + buildReadlineNoMarkdown = forAllSystems (system: + self.packages.${system}.nix.override { + enableMarkdown = false; + readlineFlavor = "readline"; + } + ); + + # Perl bindings for various platforms. + perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.nix-perl-bindings); + + # Binary tarball for various platforms, containing a Nix store + # with the closure of 'nix' package, and the second half of + # the installation script. + binaryTarball = forAllSystems (system: binaryTarball nixpkgsFor.${system}.native.nix nixpkgsFor.${system}.native); + + binaryTarballCross = lib.genAttrs [ "x86_64-linux" ] (system: + forAllCrossSystems (crossSystem: + binaryTarball + nixpkgsFor.${system}.cross.${crossSystem}.nix + nixpkgsFor.${system}.cross.${crossSystem})); + + # The first half of the installation script. This is uploaded + # to https://nixos.org/nix/install. It downloads the binary + # tarball for the user's system and calls the second half of the + # installation script. + installerScript = installScriptFor [ + # Native + self.hydraJobs.binaryTarball."x86_64-linux" + self.hydraJobs.binaryTarball."i686-linux" + self.hydraJobs.binaryTarball."aarch64-linux" + self.hydraJobs.binaryTarball."x86_64-darwin" + self.hydraJobs.binaryTarball."aarch64-darwin" + # Cross + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu" + ]; + installerScriptForGHA = installScriptFor [ + # Native + self.hydraJobs.binaryTarball."x86_64-linux" + self.hydraJobs.binaryTarball."aarch64-darwin" + # Cross + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu" + ]; + + # docker image with Nix inside + dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); + + # Line coverage analysis. + coverage = nixpkgsFor.x86_64-linux.native.nix.override { + pname = "nix-coverage"; + withCoverageChecks = true; + }; + + # API docs for Nix's unstable internal C++ interfaces. + internal-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-internal-api-docs; + + # API docs for Nix's C bindings. + external-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-external-api-docs; + + # System tests. + tests = import ../tests/nixos { inherit lib nixpkgs nixpkgsFor self; } // { + + # Make sure that nix-env still produces the exact same result + # on a particular version of Nixpkgs. + evalNixpkgs = + let + inherit (nixpkgsFor.x86_64-linux.native) runCommand nix; + in + runCommand "eval-nixos" { buildInputs = [ nix ]; } + '' + type -p nix-env + # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593. + ( + set -x + time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages + [[ $(sha1sum < packages | cut -c1-40) = e01b031fc9785a572a38be6bc473957e3b6faad7 ]] + ) + mkdir $out + ''; + + nixpkgsLibTests = + forAllSystems (system: + import (nixpkgs + "/lib/tests/test-with-nix.nix") + { + lib = nixpkgsFor.${system}.native.lib; + nix = self.packages.${system}.nix; + pkgs = nixpkgsFor.${system}.native; + } + ); + }; + + metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" { + pkgs = nixpkgsFor.x86_64-linux.native; + nixpkgs = nixpkgs-regression; + }; + + installTests = forAllSystems (system: + let pkgs = nixpkgsFor.${system}.native; in + pkgs.runCommand "install-tests" + { + againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix; + againstCurrentLatest = + # FIXME: temporarily disable this on macOS because of #3605. + if system == "x86_64-linux" + then testNixVersions pkgs pkgs.nix pkgs.nixVersions.latest + else null; + # Disabled because the latest stable version doesn't handle + # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work + # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable; + } "touch $out"); + + installerTests = import ../tests/installer { + binaryTarballs = self.hydraJobs.binaryTarball; + inherit nixpkgsFor; + }; +} diff --git a/perl/.yath.rc b/perl/.yath.rc deleted file mode 100644 index 118bf80c8..000000000 --- a/perl/.yath.rc +++ /dev/null @@ -1,2 +0,0 @@ -[test] --I=rel(lib/Nix) diff --git a/perl/Makefile b/perl/Makefile deleted file mode 100644 index 832668dd1..000000000 --- a/perl/Makefile +++ /dev/null @@ -1,21 +0,0 @@ -makefiles = local.mk - -GLOBAL_CXXFLAGS += -g -Wall -std=c++2a - -# A convenience for concurrent development of Nix and its Perl bindings. -# Not needed in a standalone build of the Perl bindings. -ifneq ("$(wildcard ../src)", "") - GLOBAL_CXXFLAGS += -I ../src -endif - --include Makefile.config - -OPTIMIZE = 1 - -ifeq ($(OPTIMIZE), 1) - GLOBAL_CXXFLAGS += -O3 -else - GLOBAL_CXXFLAGS += -O0 -endif - -include mk/lib.mk diff --git a/perl/Makefile.config.in b/perl/Makefile.config.in deleted file mode 100644 index d856de3ad..000000000 --- a/perl/Makefile.config.in +++ /dev/null @@ -1,18 +0,0 @@ -HOST_OS = @host_os@ -CC = @CC@ -CFLAGS = @CFLAGS@ -CXX = @CXX@ -CXXFLAGS = @CXXFLAGS@ -PACKAGE_NAME = @PACKAGE_NAME@ -PACKAGE_VERSION = @PACKAGE_VERSION@ -SODIUM_LIBS = @SODIUM_LIBS@ -NIX_CFLAGS = @NIX_CFLAGS@ -NIX_LIBS = @NIX_LIBS@ -nixbindir = @nixbindir@ -curl = @curl@ -nixlibexecdir = @nixlibexecdir@ -nixlocalstatedir = @nixlocalstatedir@ -perl = @perl@ -perllibdir = @perllibdir@ -nixstoredir = @nixstoredir@ -nixsysconfdir = @nixsysconfdir@ diff --git a/perl/configure.ac b/perl/configure.ac deleted file mode 100644 index a02cb06c9..000000000 --- a/perl/configure.ac +++ /dev/null @@ -1,84 +0,0 @@ -AC_INIT(nix-perl, m4_esyscmd([bash -c "echo -n $(cat ../.version)$VERSION_SUFFIX"])) -AC_CONFIG_SRCDIR(MANIFEST) -AC_CONFIG_AUX_DIR(../config) - -CFLAGS= -CXXFLAGS= -AC_PROG_CC -AC_PROG_CXX - -AC_CANONICAL_HOST - -# Use 64-bit file system calls so that we can support files > 2 GiB. -AC_SYS_LARGEFILE - -AC_DEFUN([NEED_PROG], -[ -AC_PATH_PROG($1, $2) -if test -z "$$1"; then - AC_MSG_ERROR([$2 is required]) -fi -]) - -NEED_PROG(perl, perl) -NEED_PROG(curl, curl) -NEED_PROG(bzip2, bzip2) -NEED_PROG(xz, xz) - -# Test that Perl has the open/fork feature (Perl 5.8.0 and beyond). -AC_MSG_CHECKING([whether Perl is recent enough]) -if ! $perl -e 'open(FOO, "-|", "true"); while () { print; }; close FOO or die;'; then - AC_MSG_RESULT(no) - AC_MSG_ERROR([Your Perl version is too old. Nix requires Perl 5.8.0 or newer.]) -fi -AC_MSG_RESULT(yes) - - -# Figure out where to install Perl modules. -AC_MSG_CHECKING([for the Perl installation prefix]) -perlversion=$($perl -e 'use Config; print $Config{version};') -perlarchname=$($perl -e 'use Config; print $Config{archname};') -AC_SUBST(perllibdir, [${libdir}/perl5/site_perl/$perlversion/$perlarchname]) -AC_MSG_RESULT($perllibdir) - -# Look for libsodium. -PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"]) - -# Check for the required Perl dependencies (DBI and DBD::SQLite). -perlFlags="-I$perllibdir" - -AC_ARG_WITH(dbi, AC_HELP_STRING([--with-dbi=PATH], - [prefix of the Perl DBI library]), - perlFlags="$perlFlags -I$withval") - -AC_ARG_WITH(dbd-sqlite, AC_HELP_STRING([--with-dbd-sqlite=PATH], - [prefix of the Perl DBD::SQLite library]), - perlFlags="$perlFlags -I$withval") - -AC_MSG_CHECKING([whether DBD::SQLite works]) -if ! $perl $perlFlags -e 'use DBI; use DBD::SQLite;' 2>&5; then - AC_MSG_RESULT(no) - AC_MSG_FAILURE([The Perl modules DBI and/or DBD::SQLite are missing.]) -fi -AC_MSG_RESULT(yes) - -AC_SUBST(perlFlags) - -PKG_CHECK_MODULES([NIX], [nix-store]) - -NEED_PROG([NIX], [nix]) - -# Expand all variables in config.status. -test "$prefix" = NONE && prefix=$ac_default_prefix -test "$exec_prefix" = NONE && exec_prefix='${prefix}' -for name in $ac_subst_vars; do - declare $name="$(eval echo "${!name}")" - declare $name="$(eval echo "${!name}")" - declare $name="$(eval echo "${!name}")" -done - -rm -f Makefile.config -ln -sfn ../mk mk - -AC_CONFIG_FILES([]) -AC_OUTPUT diff --git a/perl/default.nix b/perl/default.nix deleted file mode 100644 index 7103574c9..000000000 --- a/perl/default.nix +++ /dev/null @@ -1,61 +0,0 @@ -{ lib, fileset -, stdenv -, perl, perlPackages -, autoconf-archive, autoreconfHook, pkg-config -, nix, curl, bzip2, xz, boost, libsodium, darwin -}: - -perl.pkgs.toPerlModule (stdenv.mkDerivation (finalAttrs: { - name = "nix-perl-${nix.version}"; - - src = fileset.toSource { - root = ../.; - fileset = fileset.unions ([ - ../.version - ../m4 - ../mk - ./MANIFEST - ./Makefile - ./Makefile.config.in - ./configure.ac - ./lib - ./local.mk - ] ++ lib.optionals finalAttrs.doCheck [ - ./.yath.rc - ./t - ]); - }; - - nativeBuildInputs = - [ autoconf-archive - autoreconfHook - pkg-config - ]; - - buildInputs = - [ nix - curl - bzip2 - xz - perl - boost - ] - ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium - ++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security; - - # `perlPackages.Test2Harness` is marked broken for Darwin - doCheck = !stdenv.isDarwin; - - nativeCheckInputs = [ - perlPackages.Test2Harness - ]; - - configureFlags = [ - "--with-dbi=${perlPackages.DBI}/${perl.libPrefix}" - "--with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix}" - ]; - - enableParallelBuilding = true; - - postUnpack = "sourceRoot=$sourceRoot/perl"; -})) diff --git a/perl/local.mk b/perl/local.mk deleted file mode 100644 index ed4764eb9..000000000 --- a/perl/local.mk +++ /dev/null @@ -1,46 +0,0 @@ -nix_perl_sources := \ - lib/Nix/Store.pm \ - lib/Nix/Manifest.pm \ - lib/Nix/SSH.pm \ - lib/Nix/CopyClosure.pm \ - lib/Nix/Config.pm.in \ - lib/Nix/Utils.pm - -nix_perl_modules := $(nix_perl_sources:.in=) - -$(foreach x, $(nix_perl_modules), $(eval $(call install-data-in, $(x), $(perllibdir)/Nix))) - -lib/Nix/Store.cc: lib/Nix/Store.xs - $(trace-gen) xsubpp $^ -output $@ - -libraries += Store - -Store_DIR := lib/Nix - -Store_SOURCES := $(Store_DIR)/Store.cc - -Store_CXXFLAGS = \ - $(NIX_CFLAGS) \ - -I$(shell perl -e 'use Config; print $$Config{archlibexp};')/CORE \ - -D_FILE_OFFSET_BITS=64 \ - -Wno-unknown-warning-option -Wno-unused-variable -Wno-literal-suffix \ - -Wno-reserved-user-defined-literal -Wno-duplicate-decl-specifier -Wno-pointer-bool-conversion - -Store_LDFLAGS := $(SODIUM_LIBS) $(NIX_LIBS) - -ifdef HOST_CYGWIN - archlib = $(shell perl -E 'use Config; print $$Config{archlib};') - libperl = $(shell perl -E 'use Config; print $$Config{libperl};') - Store_LDFLAGS += $(shell find ${archlib} -name ${libperl}) -endif - -Store_ALLOW_UNDEFINED = 1 - -Store_FORCE_INSTALL = 1 - -Store_INSTALL_DIR = $(perllibdir)/auto/Nix/Store - -clean-files += lib/Nix/Config.pm lib/Nix/Store.cc Makefile.config - -check: all - yath test diff --git a/precompiled-headers.h b/precompiled-headers.h index f52f1cab8..e1a3f8cc0 100644 --- a/precompiled-headers.h +++ b/precompiled-headers.h @@ -42,19 +42,22 @@ #include #include #include -#include -#include -#include #include -#include -#include -#include #include #include #include -#include -#include -#include #include +#ifndef _WIN32 +# include +# include +# include +# include +# include +# include +# include +# include +# include +#endif + #include diff --git a/scripts/bigsur-nixbld-user-migration.sh b/scripts/bigsur-nixbld-user-migration.sh index f1619fd56..0eb312e07 100755 --- a/scripts/bigsur-nixbld-user-migration.sh +++ b/scripts/bigsur-nixbld-user-migration.sh @@ -3,7 +3,7 @@ ((NEW_NIX_FIRST_BUILD_UID=301)) id_available(){ - dscl . list /Users UniqueID | grep -E '\b'$1'\b' >/dev/null + dscl . list /Users UniqueID | grep -E '\b'"$1"'\b' >/dev/null } change_nixbld_names_and_ids(){ @@ -26,18 +26,18 @@ change_nixbld_names_and_ids(){ fi done - if [[ $name == _* ]]; then + if [[ "$name" == _* ]]; then echo " It looks like $name has already been renamed--skipping." else # first 3 are cleanup, it's OK if they aren't here - sudo dscl . delete /Users/$name dsAttrTypeNative:_writers_passwd &>/dev/null || true - sudo dscl . change /Users/$name NFSHomeDirectory "/private/var/empty 1" "/var/empty" &>/dev/null || true + sudo dscl . delete "/Users/$name" dsAttrTypeNative:_writers_passwd &>/dev/null || true + sudo dscl . change "/Users/$name" NFSHomeDirectory "/private/var/empty 1" "/var/empty" &>/dev/null || true # remove existing user from group - sudo dseditgroup -o edit -t user -d $name nixbld || true - sudo dscl . change /Users/$name UniqueID $uid $next_id - sudo dscl . change /Users/$name RecordName $name _$name + sudo dseditgroup -o edit -t user -d "$name" nixbld || true + sudo dscl . change "/Users/$name" UniqueID "$uid" "$next_id" + sudo dscl . change "/Users/$name" RecordName "$name" "_$name" # add renamed user to group - sudo dseditgroup -o edit -t user -a _$name nixbld + sudo dseditgroup -o edit -t user -a "_$name" nixbld echo " $name migrated to _$name (uid: $next_id)" fi done < <(dscl . list /Users UniqueID | grep nixbld | sort -n -k2) diff --git a/scripts/check-hydra-status.sh b/scripts/check-hydra-status.sh deleted file mode 100644 index e62705e94..000000000 --- a/scripts/check-hydra-status.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail -# set -x - - -# mapfile BUILDS_FOR_LATEST_EVAL < <( -# curl -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master/evals | \ -# jq -r '.evals[0].builds[] | @sh') -BUILDS_FOR_LATEST_EVAL=$( -curl -sS -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master/evals | \ - jq -r '.evals[0].builds[]') - -someBuildFailed=0 - -for buildId in $BUILDS_FOR_LATEST_EVAL; do - buildInfo=$(curl --fail -sS -H 'Accept: application/json' "https://hydra.nixos.org/build/$buildId") - - finished=$(echo "$buildInfo" | jq -r '.finished') - - if [[ $finished = 0 ]]; then - continue - fi - - buildStatus=$(echo "$buildInfo" | jq -r '.buildstatus') - - if [[ $buildStatus != 0 ]]; then - someBuildFailed=1 - echo "Job “$(echo "$buildInfo" | jq -r '.job')†failed on hydra: $buildInfo" - fi -done - -exit "$someBuildFailed" diff --git a/scripts/flake-regressions.sh b/scripts/flake-regressions.sh new file mode 100755 index 000000000..d76531134 --- /dev/null +++ b/scripts/flake-regressions.sh @@ -0,0 +1,27 @@ +#! /usr/bin/env bash + +set -e + +echo "Nix version:" +nix --version + +cd flake-regressions + +status=0 + +flakes=$(find tests -mindepth 3 -maxdepth 3 -type d -not -path '*/.*' | sort | head -n25) + +echo "Running flake tests..." + +for flake in $flakes; do + + if ! REGENERATE=0 ./eval-flake.sh "$flake"; then + status=1 + echo "⌠$flake" + else + echo "✅ $flake" + fi + +done + +exit "$status" diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index ad3ee8881..6aee073e3 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -754,7 +754,7 @@ I will: (if it does, I will tell you how to clean them up.) - create local users (see the list above for the users I'll make) - create a local group ($NIX_BUILD_GROUP_NAME) - - install Nix in to $NIX_ROOT + - install Nix in $NIX_ROOT - create a configuration file in /etc/nix - set up the "default profile" by creating some Nix-related files in $ROOT_HOME diff --git a/scripts/install-systemd-multi-user.sh b/scripts/install-systemd-multi-user.sh index 202a9bb54..a62ed7e3a 100755 --- a/scripts/install-systemd-multi-user.sh +++ b/scripts/install-systemd-multi-user.sh @@ -35,7 +35,7 @@ escape_systemd_env() { # Gather all non-empty proxy environment variables into a string create_systemd_proxy_env() { - vars="http_proxy https_proxy ftp_proxy no_proxy HTTP_PROXY HTTPS_PROXY FTP_PROXY NO_PROXY" + vars="http_proxy https_proxy ftp_proxy all_proxy no_proxy HTTP_PROXY HTTPS_PROXY FTP_PROXY ALL_PROXY NO_PROXY" for v in $vars; do if [ "x${!v:-}" != "x" ]; then echo "Environment=${v}=$(escape_systemd_env ${!v})" diff --git a/scripts/install.in b/scripts/install.in index 7d2e52b26..b4e808d8e 100755 --- a/scripts/install.in +++ b/scripts/install.in @@ -50,6 +50,11 @@ case "$(uname -s).$(uname -m)" in path=@tarballPath_armv7l-linux@ system=armv7l-linux ;; + Linux.riscv64) + hash=@tarballHash_riscv64-linux@ + path=@tarballPath_riscv64-linux@ + system=riscv64-linux + ;; Darwin.x86_64) hash=@tarballHash_x86_64-darwin@ path=@tarballPath_x86_64-darwin@ diff --git a/scripts/nix-profile-daemon.fish.in b/scripts/nix-profile-daemon.fish.in index c23aa64f0..346dce5dd 100644 --- a/scripts/nix-profile-daemon.fish.in +++ b/scripts/nix-profile-daemon.fish.in @@ -28,7 +28,7 @@ else end # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. -if test -n "$NIX_SSH_CERT_FILE" +if test -n "$NIX_SSL_CERT_FILE" : # Allow users to override the NIX_SSL_CERT_FILE else if test -e /etc/ssl/certs/ca-certificates.crt # NixOS, Ubuntu, Debian, Gentoo, Arch set --export NIX_SSL_CERT_FILE /etc/ssl/certs/ca-certificates.crt @@ -44,7 +44,7 @@ else if test -e "$NIX_LINK/etc/ca-bundle.crt" # old cacert in Nix profile set --export NIX_SSL_CERT_FILE "$NIX_LINK/etc/ca-bundle.crt" else # Fall back to what is in the nix profiles, favouring whatever is defined last. - for i in $NIX_PROFILES + for i in (string split ' ' $NIX_PROFILES) if test -e "$i/etc/ssl/certs/ca-bundle.crt" set --export NIX_SSL_CERT_FILE "$i/etc/ssl/certs/ca-bundle.crt" end diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index d256b24ed..eb124c0b5 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -1,4 +1,5 @@ # Only execute this file once per shell. +# This file is tested by tests/installer/default.nix. if [ -n "${__ETC_PROFILE_NIX_SOURCED:-}" ]; then return; fi __ETC_PROFILE_NIX_SOURCED=1 @@ -9,11 +10,9 @@ else NIX_LINK_NEW=$HOME/.local/state/nix/profile fi if [ -e "$NIX_LINK_NEW" ]; then - NIX_LINK="$NIX_LINK_NEW" -else - if [ -t 2 ] && [ -e "$NIX_LINK_NEW" ]; then + if [ -t 2 ] && [ -e "$NIX_LINK" ]; then warning="\033[1;35mwarning:\033[0m" - printf "$warning Both %s and legacy %s exist; using the latter.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2 + printf "$warning Both %s and legacy %s exist; using the former.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2 if [ "$(realpath "$NIX_LINK")" = "$(realpath "$NIX_LINK_NEW")" ]; then printf " Since the profiles match, you can safely delete either of them.\n" 1>&2 else @@ -26,6 +25,7 @@ else printf "$warning Profiles do not match. You should manually migrate from %s to %s.\n" "$NIX_LINK" "$NIX_LINK_NEW" 1>&2 fi fi + NIX_LINK="$NIX_LINK_NEW" fi export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK" @@ -69,4 +69,4 @@ else fi export PATH="$NIX_LINK/bin:@localstatedir@/nix/profiles/default/bin:$PATH" -unset NIX_LINK +unset NIX_LINK NIX_LINK_NEW diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index 44bc96e89..e868399b1 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -1,3 +1,4 @@ +# This file is tested by tests/installer/default.nix. if [ -n "$HOME" ] && [ -n "$USER" ]; then # Set up the per-user profile. @@ -9,11 +10,9 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then NIX_LINK_NEW="$HOME/.local/state/nix/profile" fi if [ -e "$NIX_LINK_NEW" ]; then - NIX_LINK="$NIX_LINK_NEW" - else - if [ -t 2 ] && [ -e "$NIX_LINK_NEW" ]; then + if [ -t 2 ] && [ -e "$NIX_LINK" ]; then warning="\033[1;35mwarning:\033[0m" - printf "$warning Both %s and legacy %s exist; using the latter.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2 + printf "$warning Both %s and legacy %s exist; using the former.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2 if [ "$(realpath "$NIX_LINK")" = "$(realpath "$NIX_LINK_NEW")" ]; then printf " Since the profiles match, you can safely delete either of them.\n" 1>&2 else @@ -26,6 +25,7 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then printf "$warning Profiles do not match. You should manually migrate from %s to %s.\n" "$NIX_LINK" "$NIX_LINK_NEW" 1>&2 fi fi + NIX_LINK="$NIX_LINK_NEW" fi # Set up environment. diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 118468477..82ad7d862 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -11,11 +11,13 @@ #include "machines.hh" #include "shared.hh" +#include "plugin.hh" #include "pathlocks.hh" #include "globals.hh" #include "serialise.hh" #include "build-result.hh" #include "store-api.hh" +#include "strings.hh" #include "derivations.hh" #include "local-store.hh" #include "legacy.hh" @@ -37,7 +39,7 @@ static std::string currentLoad; static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot) { - return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true); + return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri.render()), slot), true); } static bool allSupportedLocally(Store & store, const std::set& requiredFeatures) { @@ -135,7 +137,7 @@ static int main_build_remote(int argc, char * * argv) Machine * bestMachine = nullptr; uint64_t bestLoad = 0; for (auto & m : machines) { - debug("considering building on remote machine '%s'", m.storeUri); + debug("considering building on remote machine '%s'", m.storeUri.render()); if (m.enabled && m.systemSupported(neededSystem) && @@ -202,7 +204,7 @@ static int main_build_remote(int argc, char * * argv) else drvstr = ""; - auto error = HintFmt(errorText); + auto error = HintFmt::fromFormatString(errorText); error % drvstr % neededSystem @@ -232,17 +234,16 @@ static int main_build_remote(int argc, char * * argv) lock = -1; try { + storeUri = bestMachine->storeUri.render(); - Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri)); + Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", storeUri)); sshStore = bestMachine->openStore(); sshStore->connect(); - storeUri = bestMachine->storeUri; - } catch (std::exception & e) { auto msg = chomp(drainFD(5, false)); printError("cannot build on '%s': %s%s", - bestMachine->storeUri, e.what(), + storeUri, e.what(), msg.empty() ? "" : ": " + msg); bestMachine->enabled = false; continue; @@ -262,7 +263,20 @@ connected: auto inputs = readStrings(source); auto wantedOutputs = readStrings(source); - AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true); + AutoCloseFD uploadLock; + { + auto setUpdateLock = [&](auto && fileName){ + uploadLock = openLockFile(currentLoad + "/" + escapeUri(fileName) + ".upload-lock", true); + }; + try { + setUpdateLock(storeUri); + } catch (SysError & e) { + if (e.errNo != ENAMETOOLONG) throw; + // Try again hashing the store URL so we have a shorter path + auto h = hashString(HashAlgorithm::MD5, storeUri); + setUpdateLock(h.to_string(HashFormat::Base64, false)); + } + } { Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri)); diff --git a/doc/internal-api/.gitignore b/src/external-api-docs/.gitignore similarity index 100% rename from doc/internal-api/.gitignore rename to src/external-api-docs/.gitignore diff --git a/src/external-api-docs/.version b/src/external-api-docs/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/external-api-docs/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/external-api-docs/README.md b/src/external-api-docs/README.md new file mode 100644 index 000000000..8760ac88b --- /dev/null +++ b/src/external-api-docs/README.md @@ -0,0 +1,121 @@ +# Getting started + +> **Warning** These bindings are **experimental**, which means they can change +> at any time or be removed outright; nevertheless the plan is to provide a +> stable external C API to the Nix language and the Nix store. + +The language library allows evaluating Nix expressions and interacting with Nix +language values. The Nix store API is still rudimentary, and only allows +initialising and connecting to a store for the Nix language evaluator to +interact with. + +Currently there are two ways to interface with the Nix language evaluator +programmatically: + +1. Embedding the evaluator +2. Writing language plug-ins + +Embedding means you link the Nix C libraries in your program and use them from +there. Adding a plug-in means you make a library that gets loaded by the Nix +language evaluator, specified through a configuration option. + +Many of the components and mechanisms involved are not yet documented, therefore +please refer to the [Nix source code](https://github.com/NixOS/nix/) for +details. Additions to in-code documentation and the reference manual are highly +appreciated. + +The following examples, for simplicity, don't include error handling. See the +[Handling errors](@ref errors) section for more information. + +# Embedding the Nix Evaluator{#nix_evaluator_example} + +In this example we programmatically start the Nix language evaluator with a +dummy store (that has no store paths and cannot be written to), and evaluate the +Nix expression `builtins.nixVersion`. + +**main.c:** + +```C +#include +#include +#include +#include +#include +#include + +// NOTE: This example lacks all error handling. Production code must check for +// errors, as some return values will be undefined. + +void my_get_string_cb(const char * start, unsigned int n, void * user_data) +{ + *((char **) user_data) = strdup(start); +} + +int main() +{ + nix_libexpr_init(NULL); + + Store * store = nix_store_open(NULL, "dummy://", NULL); + EvalState * state = nix_state_create(NULL, NULL, store); // empty search path (NIX_PATH) + Value * value = nix_alloc_value(NULL, state); + + nix_expr_eval_from_string(NULL, state, "builtins.nixVersion", ".", value); + nix_value_force(NULL, state, value); + + char * version; + nix_get_string(NULL, value, my_get_string_cb, &version); + printf("Nix version: %s\n", version); + + free(version); + nix_gc_decref(NULL, value); + nix_state_free(state); + nix_store_free(store); + return 0; +} +``` + +**Usage:** + +```ShellSession +$ gcc main.c $(pkg-config nix-expr-c --libs --cflags) -o main +$ ./main +Nix version: 2.17 +``` + +# Writing a Nix language plug-in + +In this example we add a custom primitive operation (_primop_) to `builtins`. It +will increment the argument if it is an integer and throw an error otherwise. + +**plugin.c:** + +```C +#include +#include +#include + +void increment(void* user_data, nix_c_context* ctx, EvalState* state, Value** args, Value* v) { + nix_value_force(NULL, state, args[0]); + if (nix_get_type(NULL, args[0]) == NIX_TYPE_INT) { + nix_init_int(NULL, v, nix_get_int(NULL, args[0]) + 1); + } else { + nix_set_err_msg(ctx, NIX_ERR_UNKNOWN, "First argument should be an integer."); + } +} + +void nix_plugin_entry() { + const char* args[] = {"n", NULL}; + PrimOp *p = nix_alloc_primop(NULL, increment, 1, "increment", args, "Example custom built-in function: increments an integer", NULL); + nix_register_primop(NULL, p); + nix_gc_decref(NULL, p); +} +``` + +**Usage:** + +```ShellSession +$ gcc plugin.c $(pkg-config nix-expr-c --libs --cflags) -shared -o plugin.so +$ nix --plugin-files ./plugin.so repl +nix-repl> builtins.increment 1 +2 +``` diff --git a/src/external-api-docs/doxygen.cfg.in b/src/external-api-docs/doxygen.cfg.in new file mode 100644 index 000000000..1be71d895 --- /dev/null +++ b/src/external-api-docs/doxygen.cfg.in @@ -0,0 +1,58 @@ +# Doxyfile 1.9.5 + +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by +# double-quotes, unless you are using Doxywizard) that should identify the +# project for which the documentation is generated. This name is used in the +# title of most generated pages and in a few other places. +# The default value is: My Project. + +PROJECT_NAME = "Nix" + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. This +# could be handy for archiving the generated documentation or if some version +# control system is used. + +PROJECT_NUMBER = @PROJECT_NUMBER@ + +OUTPUT_DIRECTORY = @OUTPUT_DIRECTORY@ + +# Using the PROJECT_BRIEF tag one can provide an optional one line description +# for a project that appears at the top of each page and should give viewer a +# quick idea about the purpose of the project. Keep the description short. + +PROJECT_BRIEF = "Nix, the purely functional package manager: C API (experimental)" + +# If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output. +# The default value is: YES. + +GENERATE_LATEX = NO + +# The INPUT tag is used to specify the files and/or directories that contain +# documented source files. You may enter file names like myfile.cpp or +# directories like /usr/src/myproject. Separate the files or directories with +# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING +# Note: If this tag is empty the current directory is searched. + +# FIXME Make this list more maintainable somehow. We could maybe generate this +# in the Makefile, but we would need to change how `.in` files are preprocessed +# so they can expand variables despite configure variables. + +INPUT = \ + @src@/src/libutil-c \ + @src@/src/libexpr-c \ + @src@/src/libstore-c \ + @src@/doc/external-api/README.md + +FILE_PATTERNS = nix_api_*.h *.md + +# The INCLUDE_PATH tag can be used to specify one or more directories that +# contain include files that are not input files but should be processed by the +# preprocessor. Note that the INCLUDE_PATH is not recursive, so the setting of +# RECURSIVE has no effect here. +# This tag requires that the tag SEARCH_INCLUDES is set to YES. + +EXCLUDE_PATTERNS = *_internal.h +GENERATE_TREEVIEW = YES +OPTIMIZE_OUTPUT_FOR_C = YES + +USE_MDFILE_AS_MAINPAGE = doc/external-api/README.md diff --git a/src/external-api-docs/meson.build b/src/external-api-docs/meson.build new file mode 100644 index 000000000..62474ffe4 --- /dev/null +++ b/src/external-api-docs/meson.build @@ -0,0 +1,31 @@ +project('nix-external-api-docs', + version : files('.version'), + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +fs = import('fs') + +doxygen_cfg = configure_file( + input : 'doxygen.cfg.in', + output : 'doxygen.cfg', + configuration : { + 'PROJECT_NUMBER': meson.project_version(), + 'OUTPUT_DIRECTORY' : meson.current_build_dir(), + 'src' : fs.parent(fs.parent(meson.project_source_root())), + }, +) + +doxygen = find_program('doxygen', native : true, required : true) + +custom_target( + 'external-api-docs', + command : [ doxygen , doxygen_cfg ], + input : [ + doxygen_cfg, + ], + output : 'html', + install : true, + install_dir : get_option('datadir') / 'doc/nix/external-api', + build_always_stale : true, +) diff --git a/src/external-api-docs/package.nix b/src/external-api-docs/package.nix new file mode 100644 index 000000000..743b3e9b7 --- /dev/null +++ b/src/external-api-docs/package.nix @@ -0,0 +1,59 @@ +{ lib +, mkMesonDerivation + +, meson +, ninja +, doxygen + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-external-api-docs"; + inherit version; + + workDir = ./.; + fileset = + let + cpp = fileset.fileFilter (file: file.hasExt "cc" || file.hasExt "h"); + in + fileset.unions [ + ./.version + ../../.version + ./meson.build + ./doxygen.cfg.in + ./README.md + # Source is not compiled, but still must be available for Doxygen + # to gather comments. + (cpp ../libexpr-c) + (cpp ../libstore-c) + (cpp ../libutil-c) + ]; + + nativeBuildInputs = [ + meson + ninja + doxygen + ]; + + preConfigure = + '' + chmod u+w ./.version + echo ${finalAttrs.version} > ./.version + ''; + + postInstall = '' + mkdir -p ''${!outputDoc}/nix-support + echo "doc external-api-docs $out/share/doc/nix/external-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products + ''; + + meta = { + platforms = lib.platforms.all; + }; +}) diff --git a/src/internal-api-docs/.gitignore b/src/internal-api-docs/.gitignore new file mode 100644 index 000000000..dab28b6b0 --- /dev/null +++ b/src/internal-api-docs/.gitignore @@ -0,0 +1,3 @@ +/doxygen.cfg +/html +/latex diff --git a/src/internal-api-docs/.version b/src/internal-api-docs/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/internal-api-docs/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/doc/internal-api/doxygen.cfg.in b/src/internal-api-docs/doxygen.cfg.in similarity index 84% rename from doc/internal-api/doxygen.cfg.in rename to src/internal-api-docs/doxygen.cfg.in index 6c6c325bd..f1ef75b38 100644 --- a/doc/internal-api/doxygen.cfg.in +++ b/src/internal-api-docs/doxygen.cfg.in @@ -12,7 +12,9 @@ PROJECT_NAME = "Nix" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = @PACKAGE_VERSION@ +PROJECT_NUMBER = @PROJECT_NUMBER@ + +OUTPUT_DIRECTORY = @OUTPUT_DIRECTORY@ # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a @@ -36,27 +38,27 @@ GENERATE_LATEX = NO # so they can expand variables despite configure variables. INPUT = \ - src/libcmd \ - src/libexpr \ - src/libexpr/flake \ - tests/unit/libexpr \ - tests/unit/libexpr/value \ - tests/unit/libexpr/test \ - tests/unit/libexpr/test/value \ - src/libexpr/value \ - src/libfetchers \ - src/libmain \ - src/libstore \ - src/libstore/build \ - src/libstore/builtins \ - tests/unit/libstore \ - tests/unit/libstore/test \ - src/libutil \ - tests/unit/libutil \ - tests/unit/libutil/test \ - src/nix \ - src/nix-env \ - src/nix-store + @src@/libcmd \ + @src@/libexpr \ + @src@/libexpr/flake \ + @src@/nix-expr-tests \ + @src@/nix-expr-tests/value \ + @src@/nix-expr-test-support/test \ + @src@/nix-expr-test-support/test/value \ + @src@/libexpr/value \ + @src@/libfetchers \ + @src@/libmain \ + @src@/libstore \ + @src@/libstore/build \ + @src@/libstore/builtins \ + @src@/nix-store-tests \ + @src@/nix-store-test-support/test \ + @src@/libutil \ + @src@/nix-util-tests \ + @src@/nix-util-test-support/test \ + @src@/nix \ + @src@/nix-env \ + @src@/nix-store # If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names # in the source code. If set to NO, only conditional compilation will be diff --git a/src/internal-api-docs/meson.build b/src/internal-api-docs/meson.build new file mode 100644 index 000000000..54eb7e5dd --- /dev/null +++ b/src/internal-api-docs/meson.build @@ -0,0 +1,31 @@ +project('nix-internal-api-docs', + version : files('.version'), + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +fs = import('fs') + +doxygen_cfg = configure_file( + input : 'doxygen.cfg.in', + output : 'doxygen.cfg', + configuration : { + 'PROJECT_NUMBER': meson.project_version(), + 'OUTPUT_DIRECTORY' : meson.current_build_dir(), + 'src' : fs.parent(fs.parent(meson.project_source_root())) / 'src', + }, +) + +doxygen = find_program('doxygen', native : true, required : true) + +custom_target( + 'internal-api-docs', + command : [ doxygen , doxygen_cfg ], + input : [ + doxygen_cfg, + ], + output : 'html', + install : true, + install_dir : get_option('datadir') / 'doc/nix/internal-api', + build_always_stale : true, +) diff --git a/src/internal-api-docs/package.nix b/src/internal-api-docs/package.nix new file mode 100644 index 000000000..07ca6d4d9 --- /dev/null +++ b/src/internal-api-docs/package.nix @@ -0,0 +1,54 @@ +{ lib +, mkMesonDerivation + +, meson +, ninja +, doxygen + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-internal-api-docs"; + inherit version; + + workDir = ./.; + fileset = let + cpp = fileset.fileFilter (file: file.hasExt "cc" || file.hasExt "hh"); + in fileset.unions [ + ./.version + ../../.version + ./meson.build + ./doxygen.cfg.in + # Source is not compiled, but still must be available for Doxygen + # to gather comments. + (cpp ../.) + ]; + + nativeBuildInputs = [ + meson + ninja + doxygen + ]; + + preConfigure = + '' + chmod u+w ./.version + echo ${finalAttrs.version} > ./.version + ''; + + postInstall = '' + mkdir -p ''${!outputDoc}/nix-support + echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products + ''; + + meta = { + platforms = lib.platforms.all; + }; +}) diff --git a/src/libcmd/.version b/src/libcmd/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libcmd/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libcmd/build-utils-meson b/src/libcmd/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libcmd/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc index c5eb93c5d..905e70f32 100644 --- a/src/libcmd/built-path.cc +++ b/src/libcmd/built-path.cc @@ -1,6 +1,7 @@ #include "built-path.hh" #include "derivations.hh" #include "store-api.hh" +#include "comparator.hh" #include @@ -8,30 +9,24 @@ namespace nix { -#define CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, COMPARATOR) \ - bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \ - { \ - const MY_TYPE* me = this; \ - auto fields1 = std::tie(*me->drvPath, me->FIELD); \ - me = &other; \ - auto fields2 = std::tie(*me->drvPath, me->FIELD); \ - return fields1 COMPARATOR fields2; \ - } -#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, ==) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <) +// Custom implementation to avoid `ref` ptr equality +GENERATE_CMP_EXT( + , + std::strong_ordering, + SingleBuiltPathBuilt, + *me->drvPath, + me->output); -#define FIELD_TYPE std::pair -CMP(SingleBuiltPath, SingleBuiltPathBuilt, output) -#undef FIELD_TYPE +// Custom implementation to avoid `ref` ptr equality -#define FIELD_TYPE std::map -CMP(SingleBuiltPath, BuiltPathBuilt, outputs) -#undef FIELD_TYPE - -#undef CMP -#undef CMP_ONE +// TODO no `GENERATE_CMP_EXT` because no `std::set::operator<=>` on +// Darwin, per header. +GENERATE_EQUAL( + , + BuiltPathBuilt ::, + BuiltPathBuilt, + *me->drvPath, + me->outputs); StorePath SingleBuiltPath::outPath() const { diff --git a/src/libcmd/built-path.hh b/src/libcmd/built-path.hh index 99917e0ee..dc78d3e59 100644 --- a/src/libcmd/built-path.hh +++ b/src/libcmd/built-path.hh @@ -18,7 +18,8 @@ struct SingleBuiltPathBuilt { static SingleBuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view); nlohmann::json toJSON(const StoreDirConfig & store) const; - DECLARE_CMP(SingleBuiltPathBuilt); + bool operator ==(const SingleBuiltPathBuilt &) const noexcept; + std::strong_ordering operator <=>(const SingleBuiltPathBuilt &) const noexcept; }; using _SingleBuiltPathRaw = std::variant< @@ -33,6 +34,9 @@ struct SingleBuiltPath : _SingleBuiltPathRaw { using Opaque = DerivedPathOpaque; using Built = SingleBuiltPathBuilt; + bool operator == (const SingleBuiltPath &) const = default; + auto operator <=> (const SingleBuiltPath &) const = default; + inline const Raw & raw() const { return static_cast(*this); } @@ -59,11 +63,13 @@ struct BuiltPathBuilt { ref drvPath; std::map outputs; + bool operator == (const BuiltPathBuilt &) const noexcept; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //std::strong_ordering operator <=> (const BuiltPathBuilt &) const noexcept; + std::string to_string(const StoreDirConfig & store) const; static BuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view); nlohmann::json toJSON(const StoreDirConfig & store) const; - - DECLARE_CMP(BuiltPathBuilt); }; using _BuiltPathRaw = std::variant< @@ -82,6 +88,10 @@ struct BuiltPath : _BuiltPathRaw { using Opaque = DerivedPathOpaque; using Built = BuiltPathBuilt; + bool operator == (const BuiltPath &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=> (const BuiltPath &) const = default; + inline const Raw & raw() const { return static_cast(*this); } diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 369fa6004..67fef1909 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -1,3 +1,5 @@ +#include + #include "command.hh" #include "markdown.hh" #include "store-api.hh" @@ -6,8 +8,7 @@ #include "nixexpr.hh" #include "profiles.hh" #include "repl.hh" - -#include +#include "strings.hh" extern char * * environ __attribute__((weak)); @@ -127,12 +128,12 @@ ref EvalCommand::getEvalState() if (!evalState) { evalState = #if HAVE_BOEHMGC - std::allocate_shared(traceable_allocator(), - searchPath, getEvalStore(), getStore()) + std::allocate_shared( + traceable_allocator(), #else std::make_shared( - searchPath, getEvalStore(), getStore()) #endif + lookupPath, getEvalStore(), fetchSettings, evalSettings, getStore()) ; evalState->repair = repair; @@ -148,7 +149,7 @@ MixOperateOnOptions::MixOperateOnOptions() { addFlag({ .longName = "derivation", - .description = "Operate on the [store derivation](../../glossary.md#gloss-store-derivation) rather than its outputs.", + .description = "Operate on the [store derivation](@docroot@/glossary.md#gloss-store-derivation) rather than its outputs.", .category = installablesCategory, .handler = {&operateOn, OperateOn::Derivation}, }); diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 444ff81c9..fcef92487 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -1,18 +1,57 @@ +#include "fetch-settings.hh" #include "eval-settings.hh" #include "common-eval-args.hh" #include "shared.hh" +#include "config-global.hh" #include "filetransfer.hh" #include "eval.hh" #include "fetchers.hh" #include "registry.hh" #include "flake/flakeref.hh" +#include "flake/settings.hh" #include "store-api.hh" #include "command.hh" #include "tarball.hh" #include "fetch-to-store.hh" +#include "compatibility-settings.hh" +#include "eval-settings.hh" namespace nix { +fetchers::Settings fetchSettings; + +static GlobalConfig::Register rFetchSettings(&fetchSettings); + +EvalSettings evalSettings { + settings.readOnlyMode, + { + { + "flake", + [](ref store, std::string_view rest) { + experimentalFeatureSettings.require(Xp::Flakes); + // FIXME `parseFlakeRef` should take a `std::string_view`. + auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false); + debug("fetching flake search path element '%s''", rest); + auto storePath = flakeRef.resolve(store).fetchTree(store).first; + return store->toRealPath(storePath); + }, + }, + }, +}; + +static GlobalConfig::Register rEvalSettings(&evalSettings); + + +flake::Settings flakeSettings; + +static GlobalConfig::Register rFlakeSettings(&flakeSettings); + + +CompatibilitySettings compatibilitySettings {}; + +static GlobalConfig::Register rCompatibilitySettings(&compatibilitySettings); + + MixEvalArgs::MixEvalArgs() { addFlag({ @@ -20,7 +59,7 @@ MixEvalArgs::MixEvalArgs() .description = "Pass the value *expr* as the argument *name* to Nix functions.", .category = category, .labels = {"name", "expr"}, - .handler = {[&](std::string name, std::string expr) { autoArgs[name] = 'E' + expr; }} + .handler = {[&](std::string name, std::string expr) { autoArgs.insert_or_assign(name, AutoArg{AutoArgExpr{expr}}); }} }); addFlag({ @@ -28,87 +67,40 @@ MixEvalArgs::MixEvalArgs() .description = "Pass the string *string* as the argument *name* to Nix functions.", .category = category, .labels = {"name", "string"}, - .handler = {[&](std::string name, std::string s) { autoArgs[name] = 'S' + s; }}, + .handler = {[&](std::string name, std::string s) { autoArgs.insert_or_assign(name, AutoArg{AutoArgString{s}}); }}, + }); + + addFlag({ + .longName = "arg-from-file", + .description = "Pass the contents of file *path* as the argument *name* to Nix functions.", + .category = category, + .labels = {"name", "path"}, + .handler = {[&](std::string name, std::string path) { autoArgs.insert_or_assign(name, AutoArg{AutoArgFile{path}}); }}, + .completer = completePath + }); + + addFlag({ + .longName = "arg-from-stdin", + .description = "Pass the contents of stdin as the argument *name* to Nix functions.", + .category = category, + .labels = {"name"}, + .handler = {[&](std::string name) { autoArgs.insert_or_assign(name, AutoArg{AutoArgStdin{}}); }}, }); addFlag({ .longName = "include", .shortName = 'I', .description = R"( - Add *path* to the Nix search path. The Nix search path is - initialized from the colon-separated [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) environment - variable, and is used to look up the location of Nix expressions using [paths](@docroot@/language/values.md#type-path) enclosed in angle - brackets (i.e., ``). + Add *path* to search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md) - For instance, passing + This option may be given multiple times. - ``` - -I /home/eelco/Dev - -I /etc/nixos - ``` - - will cause Nix to look for paths relative to `/home/eelco/Dev` and - `/etc/nixos`, in that order. This is equivalent to setting the - `NIX_PATH` environment variable to - - ``` - /home/eelco/Dev:/etc/nixos - ``` - - It is also possible to match paths against a prefix. For example, - passing - - ``` - -I nixpkgs=/home/eelco/Dev/nixpkgs-branch - -I /etc/nixos - ``` - - will cause Nix to search for `` in - `/home/eelco/Dev/nixpkgs-branch/path` and `/etc/nixos/nixpkgs/path`. - - If a path in the Nix search path starts with `http://` or `https://`, - it is interpreted as the URL of a tarball that will be downloaded and - unpacked to a temporary location. The tarball must consist of a single - top-level directory. For example, passing - - ``` - -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/master.tar.gz - ``` - - tells Nix to download and use the current contents of the `master` - branch in the `nixpkgs` repository. - - The URLs of the tarballs from the official `nixos.org` channels - (see [the manual page for `nix-channel`](../nix-channel.md)) can be - abbreviated as `channel:`. For instance, the - following two flags are equivalent: - - ``` - -I nixpkgs=channel:nixos-21.05 - -I nixpkgs=https://nixos.org/channels/nixos-21.05/nixexprs.tar.xz - ``` - - You can also fetch source trees using [flake URLs](./nix3-flake.md#url-like-syntax) and add them to the - search path. For instance, - - ``` - -I nixpkgs=flake:nixpkgs - ``` - - specifies that the prefix `nixpkgs` shall refer to the source tree - downloaded from the `nixpkgs` entry in the flake registry. Similarly, - - ``` - -I nixpkgs=flake:github:NixOS/nixpkgs/nixos-22.05 - ``` - - makes `` refer to a particular branch of the - `NixOS/nixpkgs` repository on GitHub. + Paths added through `-I` take precedence over the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path) and the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH). )", .category = category, .labels = {"path"}, .handler = {[&](std::string s) { - searchPath.elements.emplace_back(SearchPath::Elem::parse(s)); + lookupPath.elements.emplace_back(LookupPath::Elem::parse(s)); }} }); @@ -127,8 +119,8 @@ MixEvalArgs::MixEvalArgs() .category = category, .labels = {"original-ref", "resolved-ref"}, .handler = {[&](std::string _from, std::string _to) { - auto from = parseFlakeRef(_from, absPath(".")); - auto to = parseFlakeRef(_to, absPath(".")); + auto from = parseFlakeRef(fetchSettings, _from, absPath(".")); + auto to = parseFlakeRef(fetchSettings, _to, absPath(".")); fetchers::Attrs extraAttrs; if (to.subdir != "") extraAttrs["dir"] = to.subdir; fetchers::overrideRegistry(from.input, to.input, extraAttrs); @@ -154,13 +146,23 @@ MixEvalArgs::MixEvalArgs() Bindings * MixEvalArgs::getAutoArgs(EvalState & state) { auto res = state.buildBindings(autoArgs.size()); - for (auto & i : autoArgs) { + for (auto & [name, arg] : autoArgs) { auto v = state.allocValue(); - if (i.second[0] == 'E') - state.mkThunk_(*v, state.parseExprFromString(i.second.substr(1), state.rootPath("."))); - else - v->mkString(((std::string_view) i.second).substr(1)); - res.insert(state.symbols.create(i.first), v); + std::visit(overloaded { + [&](const AutoArgExpr & arg) { + state.mkThunk_(*v, state.parseExprFromString(arg.expr, compatibilitySettings.nixShellShebangArgumentsRelativeToScript ? state.rootPath(absPath(getCommandBaseDir())) : state.rootPath("."))); + }, + [&](const AutoArgString & arg) { + v->mkString(arg.s); + }, + [&](const AutoArgFile & arg) { + v->mkString(readFile(arg.path.string())); + }, + [&](const AutoArgStdin & arg) { + v->mkString(readFile(STDIN_FILENO)); + } + }, arg); + res.insert(state.symbols.create(name), v); } return res.finish(); } @@ -176,7 +178,7 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas else if (hasPrefix(s, "flake:")) { experimentalFeatureSettings.require(Xp::Flakes); - auto flakeRef = parseFlakeRef(std::string(s.substr(6)), {}, true, false); + auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false); auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first; return state.rootPath(CanonPath(state.store->toRealPath(storePath))); } diff --git a/src/libcmd/common-eval-args.hh b/src/libcmd/common-eval-args.hh index 2eb63e15d..c62365b32 100644 --- a/src/libcmd/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh @@ -6,13 +6,42 @@ #include "common-args.hh" #include "search-path.hh" +#include + namespace nix { class Store; + +namespace fetchers { struct Settings; } + class EvalState; +struct EvalSettings; +struct CompatibilitySettings; class Bindings; struct SourcePath; +namespace flake { struct Settings; } + +/** + * @todo Get rid of global setttings variables + */ +extern fetchers::Settings fetchSettings; + +/** + * @todo Get rid of global setttings variables + */ +extern EvalSettings evalSettings; + +/** + * @todo Get rid of global setttings variables + */ +extern flake::Settings flakeSettings; + +/** + * Settings that control behaviors that have changed since Nix 2.3. + */ +extern CompatibilitySettings compatibilitySettings; + struct MixEvalArgs : virtual Args, virtual MixRepair { static constexpr auto category = "Common evaluation options"; @@ -21,14 +50,24 @@ struct MixEvalArgs : virtual Args, virtual MixRepair Bindings * getAutoArgs(EvalState & state); - SearchPath searchPath; + LookupPath lookupPath; std::optional evalStoreUrl; private: - std::map autoArgs; + struct AutoArgExpr { std::string expr; }; + struct AutoArgString { std::string s; }; + struct AutoArgFile { std::filesystem::path path; }; + struct AutoArgStdin { }; + + using AutoArg = std::variant; + + std::map autoArgs; }; +/** + * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) + */ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * baseDir = nullptr); } diff --git a/src/libcmd/compatibility-settings.hh b/src/libcmd/compatibility-settings.hh new file mode 100644 index 000000000..a129a957a --- /dev/null +++ b/src/libcmd/compatibility-settings.hh @@ -0,0 +1,36 @@ +#pragma once +#include "config.hh" + +namespace nix { +struct CompatibilitySettings : public Config +{ + + CompatibilitySettings() = default; + + // Added in Nix 2.24, July 2024. + Setting nixShellAlwaysLooksForShellNix{this, true, "nix-shell-always-looks-for-shell-nix", R"( + Before Nix 2.24, [`nix-shell`](@docroot@/command-ref/nix-shell.md) would only look at `shell.nix` if it was in the working directory - when no file was specified. + + Since Nix 2.24, `nix-shell` always looks for a `shell.nix`, whether that's in the working directory, or in a directory that was passed as an argument. + + You may set this to `false` to temporarily revert to the behavior of Nix 2.23 and older. + + Using this setting is not recommended. + It will be deprecated and removed. + )"}; + + // Added in Nix 2.24, July 2024. + Setting nixShellShebangArgumentsRelativeToScript{ + this, true, "nix-shell-shebang-arguments-relative-to-script", R"( + Before Nix 2.24, relative file path expressions in arguments in a `nix-shell` shebang were resolved relative to the working directory. + + Since Nix 2.24, `nix-shell` resolves these paths in a manner that is relative to the [base directory](@docroot@/glossary.md#gloss-base-directory), defined as the script's directory. + + You may set this to `false` to temporarily revert to the behavior of Nix 2.23 and older. + + Using this setting is not recommended. + It will be deprecated and removed. + )"}; +}; + +}; diff --git a/src/libcmd/installable-attr-path.cc b/src/libcmd/installable-attr-path.cc index 3ec1c1614..8917e7a01 100644 --- a/src/libcmd/installable-attr-path.cc +++ b/src/libcmd/installable-attr-path.cc @@ -75,6 +75,8 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths() std::set outputsToInstall; for (auto & output : packageInfo.queryOutputs(false, true)) outputsToInstall.insert(output.first); + if (outputsToInstall.empty()) + outputsToInstall.insert("out"); return OutputsSpec::Names { std::move(outputsToInstall) }; }, [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec { diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index ddec7537b..8796ad5ba 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -43,20 +43,6 @@ std::vector InstallableFlake::getActualAttrPaths() return res; } -Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake) -{ - auto vFlake = state.allocValue(); - - callFlake(state, lockedFlake, *vFlake); - - auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); - assert(aOutputs); - - state.forceValue(*aOutputs->value, aOutputs->value->determinePos(noPos)); - - return aOutputs->value; -} - static std::string showAttrPaths(const std::vector & paths) { std::string s; @@ -106,9 +92,14 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() fmt("while evaluating the flake output attribute '%s'", attrPath))) { return { *derivedPathWithInfo }; + } else { + throw Error( + "expected flake output attribute '%s' to be a derivation or path but found %s: %s", + attrPath, + showType(v), + ValuePrinter(*this->state, v, errorPrintOptions) + ); } - else - throw Error("flake output attribute '%s' is not a derivation or path", attrPath); } auto drvPath = attr->forceDerivation(); @@ -205,7 +196,8 @@ std::shared_ptr InstallableFlake::getLockedFlake() const flake::LockFlags lockFlagsApplyConfig = lockFlags; // FIXME why this side effect? lockFlagsApplyConfig.applyNixConfig = true; - _lockedFlake = std::make_shared(lockFlake(*state, flakeRef, lockFlagsApplyConfig)); + _lockedFlake = std::make_shared(lockFlake( + flakeSettings, *state, flakeRef, lockFlagsApplyConfig)); } return _lockedFlake; } diff --git a/src/libcmd/installable-flake.hh b/src/libcmd/installable-flake.hh index 314918c14..8e0a232ef 100644 --- a/src/libcmd/installable-flake.hh +++ b/src/libcmd/installable-flake.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "common-eval-args.hh" #include "installable-value.hh" namespace nix { @@ -52,8 +53,6 @@ struct InstallableFlake : InstallableValue std::vector getActualAttrPaths(); - Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake); - DerivedPathsWithInfo toDerivedPaths() override; std::pair toValue(EvalState & state) override; @@ -80,7 +79,7 @@ struct InstallableFlake : InstallableValue */ static inline FlakeRef defaultNixpkgsFlakeRef() { - return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}}); + return FlakeRef::fromAttrs(fetchSettings, {{"type","indirect"}, {"id", "nixpkgs"}}); } ref openEvalCache( diff --git a/src/libcmd/installable-value.hh b/src/libcmd/installable-value.hh index f300d392b..798cb5e1a 100644 --- a/src/libcmd/installable-value.hh +++ b/src/libcmd/installable-value.hh @@ -66,7 +66,7 @@ struct ExtraPathInfoValue : ExtraPathInfo }; /** - * An Installable which corresponds a Nix langauge value, in addition to + * An Installable which corresponds a Nix language value, in addition to * a collection of \ref DerivedPath "derived paths". */ struct InstallableValue : Installable diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 16d25d3cf..0fe956ec0 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -27,6 +27,8 @@ #include +#include "strings-inline.hh" + namespace nix { void completeFlakeInputPath( @@ -129,7 +131,7 @@ MixFlakeOptions::MixFlakeOptions() lockFlags.writeLockFile = false; lockFlags.inputOverrides.insert_or_assign( flake::parseInputPath(inputPath), - parseFlakeRef(flakeRef, absPath(getCommandBaseDir()), true)); + parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir()), true)); }}, .completer = {[&](AddCompletions & completions, size_t n, std::string_view prefix) { if (n == 0) { @@ -146,7 +148,7 @@ MixFlakeOptions::MixFlakeOptions() .category = category, .labels = {"flake-lock-path"}, .handler = {[&](std::string lockFilePath) { - lockFlags.referenceLockFilePath = lockFilePath; + lockFlags.referenceLockFilePath = {getFSSourceAccessor(), CanonPath(absPath(lockFilePath))}; }}, .completer = completePath }); @@ -170,14 +172,15 @@ MixFlakeOptions::MixFlakeOptions() .handler = {[&](std::string flakeRef) { auto evalState = getEvalState(); auto flake = flake::lockFlake( + flakeSettings, *evalState, - parseFlakeRef(flakeRef, absPath(getCommandBaseDir())), + parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir())), { .writeLockFile = false }); for (auto & [inputName, input] : flake.lockFile.root->inputs) { auto input2 = flake.lockFile.findInput({inputName}); // resolve 'follows' nodes if (auto input3 = std::dynamic_pointer_cast(input2)) { overrideRegistry( - fetchers::Input::fromAttrs({{"type","indirect"}, {"id", inputName}}), + fetchers::Input::fromAttrs(fetchSettings, {{"type","indirect"}, {"id", inputName}}), input3->lockedRef.input, {}); } @@ -288,11 +291,11 @@ void SourceExprCommand::completeInstallable(AddCompletions & completions, std::s state->autoCallFunction(*autoArgs, v1, v2); if (v2.type() == nAttrs) { - for (auto & i : *v2.attrs) { - std::string name = state->symbols[i.name]; + for (auto & i : *v2.attrs()) { + std::string_view name = state->symbols[i.name]; if (name.find(searchWord) == 0) { if (prefix_ == "") - completions.add(name); + completions.add(std::string(name)); else completions.add(prefix_ + "." + name); } @@ -338,10 +341,11 @@ void completeFlakeRefWithFragment( auto flakeRefS = std::string(prefix.substr(0, hash)); // TODO: ideally this would use the command base directory instead of assuming ".". - auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath(".")); + auto flakeRef = parseFlakeRef(fetchSettings, expandTilde(flakeRefS), absPath(".")); auto evalCache = openEvalCache(*evalState, - std::make_shared(lockFlake(*evalState, flakeRef, lockFlags))); + std::make_shared(lockFlake( + flakeSettings, *evalState, flakeRef, lockFlags))); auto root = evalCache->getRoot(); @@ -372,6 +376,7 @@ void completeFlakeRefWithFragment( auto attrPath2 = (*attr)->getAttrPath(attr2); /* Strip the attrpath prefix. */ attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size()); + // FIXME: handle names with dots completions.add(flakeRefS + "#" + prefixRoot + concatStringsSep(".", evalState->symbols.resolve(attrPath2))); } } @@ -403,7 +408,7 @@ void completeFlakeRef(AddCompletions & completions, ref store, std::strin Args::completeDir(completions, 0, prefix); /* Look for registry entries that match the prefix. */ - for (auto & registry : fetchers::getRegistries(store)) { + for (auto & registry : fetchers::getRegistries(fetchSettings, store)) { for (auto & entry : registry->entries) { auto from = entry.from.to_string(); if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) { @@ -442,13 +447,10 @@ ref openEvalCache( EvalState & state, std::shared_ptr lockedFlake) { - auto fingerprint = lockedFlake->getFingerprint(); - return make_ref( - evalSettings.useEvalCache && evalSettings.pureEval - ? std::optional { std::cref(fingerprint) } - : std::nullopt, - state, - [&state, lockedFlake]() + auto fingerprint = evalSettings.useEvalCache && evalSettings.pureEval + ? lockedFlake->getFingerprint(state.store) + : std::nullopt; + auto rootLoader = [&state, lockedFlake]() { /* For testing whether the evaluation cache is complete. */ @@ -460,11 +462,21 @@ ref openEvalCache( state.forceAttrs(*vFlake, noPos, "while parsing cached flake data"); - auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); + auto aOutputs = vFlake->attrs()->get(state.symbols.create("outputs")); assert(aOutputs); return aOutputs->value; - }); + }; + + if (fingerprint) { + auto search = state.evalCaches.find(fingerprint.value()); + if (search == state.evalCaches.end()) { + search = state.evalCaches.emplace(fingerprint.value(), make_ref(fingerprint, state, rootLoader)).first; + } + return search->second; + } else { + return make_ref(std::nullopt, state, rootLoader); + } } Installables SourceExprCommand::parseInstallables( @@ -527,7 +539,8 @@ Installables SourceExprCommand::parseInstallables( } try { - auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, absPath(getCommandBaseDir())); + auto [flakeRef, fragment] = parseFlakeRefWithFragment( + fetchSettings, std::string { prefix }, absPath(getCommandBaseDir())); result.push_back(make_ref( this, getEvalState(), @@ -594,6 +607,37 @@ std::vector Installable::build( return res; } +static void throwBuildErrors( + std::vector & buildResults, + const Store & store) +{ + std::vector failed; + for (auto & buildResult : buildResults) { + if (!buildResult.success()) { + failed.push_back(buildResult); + } + } + + auto failedResult = failed.begin(); + if (failedResult != failed.end()) { + if (failed.size() == 1) { + failedResult->rethrow(); + } else { + StringSet failedPaths; + for (; failedResult != failed.end(); failedResult++) { + if (!failedResult->errorMsg.empty()) { + logError(ErrorInfo{ + .level = lvlError, + .msg = failedResult->errorMsg, + }); + } + failedPaths.insert(failedResult->path.to_string(store)); + } + throw Error("build of %s failed", concatStringsSep(", ", quoteStrings(failedPaths))); + } + } +} + std::vector, BuiltPathWithResult>> Installable::build2( ref evalStore, ref store, @@ -655,10 +699,9 @@ std::vector, BuiltPathWithResult>> Installable::build if (settings.printMissing) printMissing(store, pathsToBuild, lvlInfo); - for (auto & buildResult : store->buildPathsWithResults(pathsToBuild, bMode, evalStore)) { - if (!buildResult.success()) - buildResult.rethrow(); - + auto buildResults = store->buildPathsWithResults(pathsToBuild, bMode, evalStore); + throwBuildErrors(buildResults, *store); + for (auto & buildResult : buildResults) { for (auto & aux : backmap[buildResult.path]) { std::visit(overloaded { [&](const DerivedPath::Built & bfd) { @@ -814,6 +857,7 @@ std::vector RawInstallablesCommand::getFlakeRefsForCompletion() std::vector res; for (auto i : rawInstallables) res.push_back(parseFlakeRefWithFragment( + fetchSettings, expandTilde(i), absPath(getCommandBaseDir())).first); return res; @@ -836,6 +880,7 @@ std::vector InstallableCommand::getFlakeRefsForCompletion() { return { parseFlakeRefWithFragment( + fetchSettings, expandTilde(_installable), absPath(getCommandBaseDir())).first }; diff --git a/src/libcmd/local.mk b/src/libcmd/local.mk index abb7459a7..a270333f4 100644 --- a/src/libcmd/local.mk +++ b/src/libcmd/local.mk @@ -6,10 +6,10 @@ libcmd_DIR := $(d) libcmd_SOURCES := $(wildcard $(d)/*.cc) -libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers +libcmd_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) $(INCLUDE_libflake) $(INCLUDE_libmain) libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) $(THREAD_LDFLAGS) -libcmd_LIBS = libstore libutil libexpr libmain libfetchers +libcmd_LIBS = libutil libstore libfetchers libflake libexpr libmain $(eval $(call install-file-in, $(buildprefix)$(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644)) diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc index a4e3c5a77..6a0d05d9f 100644 --- a/src/libcmd/markdown.cc +++ b/src/libcmd/markdown.cc @@ -1,21 +1,23 @@ #include "markdown.hh" -#include "util.hh" +#include "environment-variables.hh" +#include "error.hh" #include "finally.hh" #include "terminal.hh" -#include #if HAVE_LOWDOWN -#include +# include +# include #endif namespace nix { -std::string renderMarkdownToTerminal(std::string_view markdown) -{ #if HAVE_LOWDOWN +static std::string doRenderMarkdownToTerminal(std::string_view markdown) +{ int windowWidth = getWindowSize().second; - struct lowdown_opts opts { + struct lowdown_opts opts + { .type = LOWDOWN_TERM, .maxdepth = 20, .cols = (size_t) std::max(windowWidth - 5, 60), @@ -50,10 +52,22 @@ std::string renderMarkdownToTerminal(std::string_view markdown) if (!rndr_res) throw Error("allocation error while rendering Markdown"); - return filterANSIEscapes(std::string(buf->data, buf->size), !shouldANSI()); -#else - return std::string(markdown); -#endif + return filterANSIEscapes(std::string(buf->data, buf->size), !isTTY()); } +std::string renderMarkdownToTerminal(std::string_view markdown) +{ + if (auto e = getEnv("_NIX_TEST_RAW_MARKDOWN"); e && *e == "1") + return std::string(markdown); + else + return doRenderMarkdownToTerminal(markdown); } + +#else +std::string renderMarkdownToTerminal(std::string_view markdown) +{ + return std::string(markdown); +} +#endif + +} // namespace nix diff --git a/src/libcmd/markdown.hh b/src/libcmd/markdown.hh index a04d32a4f..66db1736c 100644 --- a/src/libcmd/markdown.hh +++ b/src/libcmd/markdown.hh @@ -1,10 +1,17 @@ #pragma once ///@file -#include "types.hh" +#include namespace nix { +/** + * Render the given Markdown text to the terminal. + * + * If Nix is compiled without Markdown support, this function will return the input text as-is. + * + * The renderer takes into account the terminal width, and wraps text accordingly. + */ std::string renderMarkdownToTerminal(std::string_view markdown); } diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build new file mode 100644 index 000000000..c484cf998 --- /dev/null +++ b/src/libcmd/meson.build @@ -0,0 +1,130 @@ +project('nix-cmd', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ +] +deps_public_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), + dependency('nix-fetchers'), + dependency('nix-expr'), + dependency('nix-flake'), + dependency('nix-main'), +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') +deps_public += nlohmann_json + +lowdown = dependency('lowdown', version : '>= 0.9.0', required : get_option('markdown')) +deps_private += lowdown +configdata.set('HAVE_LOWDOWN', lowdown.found().to_int()) + +readline_flavor = get_option('readline-flavor') +if readline_flavor == 'editline' + editline = dependency('libeditline', 'editline', version : '>=1.14') + deps_private += editline +elif readline_flavor == 'readline' + readline = dependency('readline') + deps_private += readline + configdata.set( + 'USE_READLINE', + 1, + description: 'Use readline instead of editline', + ) +else + error('illegal editline flavor', readline_flavor) +endif + +config_h = configure_file( + configuration : configdata, + output : 'config-cmd.hh', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + # '-include', 'config-fetchers.h', + '-include', 'config-expr.hh', + '-include', 'config-main.hh', + '-include', 'config-cmd.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'built-path.cc', + 'command-installable-value.cc', + 'command.cc', + 'common-eval-args.cc', + 'editor-for.cc', + 'installable-attr-path.cc', + 'installable-derived-path.cc', + 'installable-flake.cc', + 'installable-value.cc', + 'installables.cc', + 'legacy.cc', + 'markdown.cc', + 'misc-store-flags.cc', + 'network-proxy.cc', + 'repl-interacter.cc', + 'repl.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'built-path.hh', + 'command-installable-value.hh', + 'command.hh', + 'common-eval-args.hh', + 'compatibility-settings.hh', + 'editor-for.hh', + 'installable-attr-path.hh', + 'installable-derived-path.hh', + 'installable-flake.hh', + 'installable-value.hh', + 'installables.hh', + 'legacy.hh', + 'markdown.hh', + 'misc-store-flags.hh', + 'network-proxy.hh', + 'repl-interacter.hh', + 'repl.hh', +) + +this_library = library( + 'nixcmd', + sources, + dependencies : deps_public + deps_private + deps_other, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libcmd/meson.options b/src/libcmd/meson.options new file mode 100644 index 000000000..79ae4fa55 --- /dev/null +++ b/src/libcmd/meson.options @@ -0,0 +1,15 @@ +# vim: filetype=meson + +option( + 'markdown', + type: 'feature', + description: 'Enable Markdown rendering in the Nix binary (requires lowdown)', +) + +option( + 'readline-flavor', + type : 'combo', + choices : ['editline', 'readline'], + value : 'editline', + description : 'Which library to use for nice line editing with the Nix language REPL', +) diff --git a/src/libcmd/misc-store-flags.cc b/src/libcmd/misc-store-flags.cc index e66d3f63b..06552c032 100644 --- a/src/libcmd/misc-store-flags.cc +++ b/src/libcmd/misc-store-flags.cc @@ -81,9 +81,15 @@ Args::Flag fileIngestionMethod(FileIngestionMethod * method) How to compute the hash of the input. One of: - - `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function. + - `nar` (the default): + Serialises the input as a + [Nix Archive](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) + and passes that to the hash function. - - `flat`: Assumes that the input is a single file and directly passes it to the hash function; + - `flat`: + Assumes that the input is a single file and + [directly passes](@docroot@/store/file-system-object/content-address.md#serial-flat) + it to the hash function. )", .labels = {"file-ingestion-method"}, .handler = {[method](std::string s) { @@ -101,15 +107,23 @@ Args::Flag contentAddressMethod(ContentAddressMethod * method) How to compute the content-address of the store object. One of: - - `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function. + - [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive) + (the default): + Serialises the input as a + [Nix Archive](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) + and passes that to the hash function. - - `flat`: Assumes that the input is a single file and directly passes it to the hash function; + - [`flat`](@docroot@/store/store-object/content-address.md#method-flat): + Assumes that the input is a single file and + [directly passes](@docroot@/store/file-system-object/content-address.md#serial-flat) + it to the hash function. - - `text`: Like `flat`, but used for - [derivations](@docroot@/glossary.md#store-derivation) serialized in store object and + - [`text`](@docroot@/store/store-object/content-address.md#method-text): + Like `flat`, but used for + [derivations](@docroot@/glossary.md#store-derivation) serialized in store object and [`builtins.toFile`](@docroot@/language/builtins.html#builtins-toFile). For advanced use-cases only; - for regular usage prefer `nar` and `flat. + for regular usage prefer `nar` and `flat`. )", .labels = {"content-address-method"}, .handler = {[method](std::string s) { diff --git a/src/libcmd/network-proxy.cc b/src/libcmd/network-proxy.cc new file mode 100644 index 000000000..738bf6147 --- /dev/null +++ b/src/libcmd/network-proxy.cc @@ -0,0 +1,50 @@ +#include "network-proxy.hh" + +#include + +#include "environment-variables.hh" + +namespace nix { + +static const StringSet lowercaseVariables{"http_proxy", "https_proxy", "ftp_proxy", "all_proxy", "no_proxy"}; + +static StringSet getAllVariables() +{ + StringSet variables = lowercaseVariables; + for (const auto & variable : lowercaseVariables) { + std::string upperVariable; + std::transform( + variable.begin(), variable.end(), upperVariable.begin(), [](unsigned char c) { return std::toupper(c); }); + variables.insert(std::move(upperVariable)); + } + return variables; +} + +const StringSet networkProxyVariables = getAllVariables(); + +static StringSet getExcludingNoProxyVariables() +{ + static const StringSet excludeVariables{"no_proxy", "NO_PROXY"}; + StringSet variables; + std::set_difference( + networkProxyVariables.begin(), + networkProxyVariables.end(), + excludeVariables.begin(), + excludeVariables.end(), + std::inserter(variables, variables.begin())); + return variables; +} + +static const StringSet excludingNoProxyVariables = getExcludingNoProxyVariables(); + +bool haveNetworkProxyConnection() +{ + for (const auto & variable : excludingNoProxyVariables) { + if (getEnv(variable).has_value()) { + return true; + } + } + return false; +} + +} diff --git a/src/libcmd/network-proxy.hh b/src/libcmd/network-proxy.hh new file mode 100644 index 000000000..0b6856acb --- /dev/null +++ b/src/libcmd/network-proxy.hh @@ -0,0 +1,22 @@ +#pragma once +///@file + +#include "types.hh" + +namespace nix { + +/** + * Environment variables relating to network proxying. These are used by + * a few misc commands. + * + * See the Environment section of https://curl.se/docs/manpage.html for details. + */ +extern const StringSet networkProxyVariables; + +/** + * Heuristically check if there is a proxy connection by checking for defined + * proxy variables. + */ +bool haveNetworkProxyConnection(); + +} diff --git a/src/libcmd/package.nix b/src/libcmd/package.nix new file mode 100644 index 000000000..cde494901 --- /dev/null +++ b/src/libcmd/package.nix @@ -0,0 +1,104 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-util +, nix-store +, nix-fetchers +, nix-expr +, nix-flake +, nix-main +, editline +, readline +, lowdown +, nlohmann_json + +# Configuration Options + +, version + +# Whether to enable Markdown rendering in the Nix binary. +, enableMarkdown ? !stdenv.hostPlatform.isWindows + +# Which interactive line editor library to use for Nix's repl. +# +# Currently supported choices are: +# +# - editline (default) +# - readline +, readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline" +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-cmd"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + ({ inherit editline readline; }.${readlineFlavor}) + ] ++ lib.optional enableMarkdown lowdown; + + propagatedBuildInputs = [ + nix-util + nix-store + nix-fetchers + nix-expr + nix-flake + nix-main + nlohmann_json + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + mesonFlags = [ + (lib.mesonEnable "markdown" enableMarkdown) + (lib.mesonOption "readline-flavor" readlineFlavor) + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libcmd/repl-interacter.cc b/src/libcmd/repl-interacter.cc new file mode 100644 index 000000000..187af46ea --- /dev/null +++ b/src/libcmd/repl-interacter.cc @@ -0,0 +1,206 @@ +#include + +#ifdef USE_READLINE +#include +#include +#else +// editline < 1.15.2 don't wrap their API for C++ usage +// (added in https://github.com/troglobit/editline/commit/91398ceb3427b730995357e9d120539fb9bb7461). +// This results in linker errors due to to name-mangling of editline C symbols. +// For compatibility with these versions, we wrap the API here +// (wrapping multiple times on newer versions is no problem). +extern "C" { +#include +} +#endif + +#include "signals.hh" +#include "finally.hh" +#include "repl-interacter.hh" +#include "file-system.hh" +#include "repl.hh" +#include "environment-variables.hh" + +namespace nix { + +namespace { +// Used to communicate to NixRepl::getLine whether a signal occurred in ::readline. +volatile sig_atomic_t g_signal_received = 0; + +void sigintHandler(int signo) +{ + g_signal_received = signo; +} +}; + +static detail::ReplCompleterMixin * curRepl; // ugly + +#ifndef USE_READLINE +static char * completionCallback(char * s, int * match) +{ + auto possible = curRepl->completePrefix(s); + if (possible.size() == 1) { + *match = 1; + auto * res = strdup(possible.begin()->c_str() + strlen(s)); + if (!res) + throw Error("allocation failure"); + return res; + } else if (possible.size() > 1) { + auto checkAllHaveSameAt = [&](size_t pos) { + auto & first = *possible.begin(); + for (auto & p : possible) { + if (p.size() <= pos || p[pos] != first[pos]) + return false; + } + return true; + }; + size_t start = strlen(s); + size_t len = 0; + while (checkAllHaveSameAt(start + len)) + ++len; + if (len > 0) { + *match = 1; + auto * res = strdup(std::string(*possible.begin(), start, len).c_str()); + if (!res) + throw Error("allocation failure"); + return res; + } + } + + *match = 0; + return nullptr; +} + +static int listPossibleCallback(char * s, char *** avp) +{ + auto possible = curRepl->completePrefix(s); + + if (possible.size() > (std::numeric_limits::max() / sizeof(char *))) + throw Error("too many completions"); + + int ac = 0; + char ** vp = nullptr; + + auto check = [&](auto * p) { + if (!p) { + if (vp) { + while (--ac >= 0) + free(vp[ac]); + free(vp); + } + throw Error("allocation failure"); + } + return p; + }; + + vp = check((char **) malloc(possible.size() * sizeof(char *))); + + for (auto & p : possible) + vp[ac++] = check(strdup(p.c_str())); + + *avp = vp; + + return ac; +} +#endif + +ReadlineLikeInteracter::Guard ReadlineLikeInteracter::init(detail::ReplCompleterMixin * repl) +{ + // Allow nix-repl specific settings in .inputrc + rl_readline_name = "nix-repl"; + try { + createDirs(dirOf(historyFile)); + } catch (SystemError & e) { + logWarning(e.info()); + } +#ifndef USE_READLINE + el_hist_size = 1000; +#endif + read_history(historyFile.c_str()); + auto oldRepl = curRepl; + curRepl = repl; + Guard restoreRepl([oldRepl] { curRepl = oldRepl; }); +#ifndef USE_READLINE + rl_set_complete_func(completionCallback); + rl_set_list_possib_func(listPossibleCallback); +#endif + return restoreRepl; +} + +static constexpr const char * promptForType(ReplPromptType promptType) +{ + switch (promptType) { + case ReplPromptType::ReplPrompt: + return "nix-repl> "; + case ReplPromptType::ContinuationPrompt: + return " "; + } + assert(false); +} + +bool ReadlineLikeInteracter::getLine(std::string & input, ReplPromptType promptType) +{ +#ifndef _WIN32 // TODO use more signals.hh for this + struct sigaction act, old; + sigset_t savedSignalMask, set; + + auto setupSignals = [&]() { + act.sa_handler = sigintHandler; + sigfillset(&act.sa_mask); + act.sa_flags = 0; + if (sigaction(SIGINT, &act, &old)) + throw SysError("installing handler for SIGINT"); + + sigemptyset(&set); + sigaddset(&set, SIGINT); + if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask)) + throw SysError("unblocking SIGINT"); + }; + auto restoreSignals = [&]() { + if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) + throw SysError("restoring signals"); + + if (sigaction(SIGINT, &old, 0)) + throw SysError("restoring handler for SIGINT"); + }; + + setupSignals(); +#endif + char * s = readline(promptForType(promptType)); + Finally doFree([&]() { free(s); }); +#ifndef _WIN32 // TODO use more signals.hh for this + restoreSignals(); +#endif + + if (g_signal_received) { + g_signal_received = 0; + input.clear(); + return true; + } + + // editline doesn't echo the input to the output when non-interactive, unlike readline + // this results in a different behavior when running tests. The echoing is + // quite useful for reading the test output, so we add it here. + if (auto e = getEnv("_NIX_TEST_REPL_ECHO"); s && e && *e == "1") + { +#ifndef USE_READLINE + // This is probably not right for multi-line input, but we don't use that + // in the characterisation tests, so it's fine. + std::cout << promptForType(promptType) << s << std::endl; +#endif + } + + if (!s) + return false; + input += s; + input += '\n'; + + return true; +} + +ReadlineLikeInteracter::~ReadlineLikeInteracter() +{ + write_history(historyFile.c_str()); +} + +}; diff --git a/src/libcmd/repl-interacter.hh b/src/libcmd/repl-interacter.hh new file mode 100644 index 000000000..cc70efd07 --- /dev/null +++ b/src/libcmd/repl-interacter.hh @@ -0,0 +1,48 @@ +#pragma once +/// @file + +#include "finally.hh" +#include "types.hh" +#include +#include + +namespace nix { + +namespace detail { +/** Provides the completion hooks for the repl, without exposing its complete + * internals. */ +struct ReplCompleterMixin { + virtual StringSet completePrefix(const std::string & prefix) = 0; +}; +}; + +enum class ReplPromptType { + ReplPrompt, + ContinuationPrompt, +}; + +class ReplInteracter +{ +public: + using Guard = Finally>; + + virtual Guard init(detail::ReplCompleterMixin * repl) = 0; + /** Returns a boolean of whether the interacter got EOF */ + virtual bool getLine(std::string & input, ReplPromptType promptType) = 0; + virtual ~ReplInteracter(){}; +}; + +class ReadlineLikeInteracter : public virtual ReplInteracter +{ + std::string historyFile; +public: + ReadlineLikeInteracter(std::string historyFile) + : historyFile(historyFile) + { + } + virtual Guard init(detail::ReplCompleterMixin * repl) override; + virtual bool getLine(std::string & input, ReplPromptType promptType) override; + virtual ~ReadlineLikeInteracter() override; +}; + +}; diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 8b83608fa..efc04b029 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -1,34 +1,17 @@ #include #include #include -#include - -#include - -#ifdef USE_READLINE -#include -#include -#else -// editline < 1.15.2 don't wrap their API for C++ usage -// (added in https://github.com/troglobit/editline/commit/91398ceb3427b730995357e9d120539fb9bb7461). -// This results in linker errors due to to name-mangling of editline C symbols. -// For compatibility with these versions, we wrap the API here -// (wrapping multiple times on newer versions is no problem). -extern "C" { -#include -} -#endif +#include "repl-interacter.hh" #include "repl.hh" #include "ansicolor.hh" -#include "signals.hh" #include "shared.hh" +#include "config-global.hh" #include "eval.hh" -#include "eval-cache.hh" -#include "eval-inline.hh" #include "eval-settings.hh" #include "attr-path.hh" +#include "signals.hh" #include "store-api.hh" #include "log-store.hh" #include "common-eval-args.hh" @@ -38,18 +21,21 @@ extern "C" { #include "flake/flake.hh" #include "flake/lockfile.hh" #include "users.hh" -#include "terminal.hh" #include "editor-for.hh" #include "finally.hh" #include "markdown.hh" #include "local-fs-store.hh" #include "print.hh" +#include "ref.hh" +#include "value.hh" #if HAVE_BOEHMGC #define GC_INCLUDE_NEW #include #endif +#include "strings.hh" + namespace nix { /** @@ -75,6 +61,7 @@ enum class ProcessLineResult { struct NixRepl : AbstractNixRepl + , detail::ReplCompleterMixin #if HAVE_BOEHMGC , gc #endif @@ -90,17 +77,16 @@ struct NixRepl int displ; StringSet varNames; - const Path historyFile; + std::unique_ptr interacter; - NixRepl(const SearchPath & searchPath, nix::ref store,ref state, + NixRepl(const LookupPath & lookupPath, nix::ref store,ref state, std::function getValues); - virtual ~NixRepl(); + virtual ~NixRepl() = default; ReplExitStatus mainLoop() override; void initEnv() override; - StringSet completePrefix(const std::string & prefix); - bool getLine(std::string & input, const std::string & prompt); + virtual StringSet completePrefix(const std::string & prefix) override; StorePath getDerivationPath(Value & v); ProcessLineResult processLine(std::string line); @@ -123,7 +109,8 @@ struct NixRepl .force = true, .derivationPaths = true, .maxDepth = maxDepth, - .prettyIndent = 2 + .prettyIndent = 2, + .errors = ErrorPrintBehavior::ThrowTopLevel, }); } }; @@ -137,111 +124,33 @@ std::string removeWhitespace(std::string s) } -NixRepl::NixRepl(const SearchPath & searchPath, nix::ref store, ref state, +NixRepl::NixRepl(const LookupPath & lookupPath, nix::ref store, ref state, std::function getValues) : AbstractNixRepl(state) , debugTraceIndex(0) , getValues(getValues) , staticEnv(new StaticEnv(nullptr, state->staticBaseEnv.get())) - , historyFile(getDataDir() + "/nix/repl-history") + , interacter(make_unique(getDataDir() + "/nix/repl-history")) { } - -NixRepl::~NixRepl() -{ - write_history(historyFile.c_str()); -} - void runNix(Path program, const Strings & args, const std::optional & input = {}) { auto subprocessEnv = getEnv(); subprocessEnv["NIX_CONFIG"] = globalConfig.toKeyValue(); - + //isInteractive avoid grabling interactive commands runProgram2(RunOptions { .program = settings.nixBinDir+ "/" + program, .args = args, .environment = subprocessEnv, .input = input, + .isInteractive = true, }); return; } -static NixRepl * curRepl; // ugly - -static char * completionCallback(char * s, int *match) { - auto possible = curRepl->completePrefix(s); - if (possible.size() == 1) { - *match = 1; - auto *res = strdup(possible.begin()->c_str() + strlen(s)); - if (!res) throw Error("allocation failure"); - return res; - } else if (possible.size() > 1) { - auto checkAllHaveSameAt = [&](size_t pos) { - auto &first = *possible.begin(); - for (auto &p : possible) { - if (p.size() <= pos || p[pos] != first[pos]) - return false; - } - return true; - }; - size_t start = strlen(s); - size_t len = 0; - while (checkAllHaveSameAt(start + len)) ++len; - if (len > 0) { - *match = 1; - auto *res = strdup(std::string(*possible.begin(), start, len).c_str()); - if (!res) throw Error("allocation failure"); - return res; - } - } - - *match = 0; - return nullptr; -} - -static int listPossibleCallback(char *s, char ***avp) { - auto possible = curRepl->completePrefix(s); - - if (possible.size() > (INT_MAX / sizeof(char*))) - throw Error("too many completions"); - - int ac = 0; - char **vp = nullptr; - - auto check = [&](auto *p) { - if (!p) { - if (vp) { - while (--ac >= 0) - free(vp[ac]); - free(vp); - } - throw Error("allocation failure"); - } - return p; - }; - - vp = check((char **)malloc(possible.size() * sizeof(char*))); - - for (auto & p : possible) - vp[ac++] = check(strdup(p.c_str())); - - *avp = vp; - - return ac; -} - -namespace { - // Used to communicate to NixRepl::getLine whether a signal occurred in ::readline. - volatile sig_atomic_t g_signal_received = 0; - - void sigintHandler(int signo) { - g_signal_received = signo; - } -} - static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positions, const DebugTrace & dt) { if (dt.isError) @@ -281,24 +190,7 @@ ReplExitStatus NixRepl::mainLoop() loadFiles(); - // Allow nix-repl specific settings in .inputrc - rl_readline_name = "nix-repl"; - try { - createDirs(dirOf(historyFile)); - } catch (SystemError & e) { - logWarning(e.info()); - } -#ifndef USE_READLINE - el_hist_size = 1000; -#endif - read_history(historyFile.c_str()); - auto oldRepl = curRepl; - curRepl = this; - Finally restoreRepl([&] { curRepl = oldRepl; }); -#ifndef USE_READLINE - rl_set_complete_func(completionCallback); - rl_set_list_possib_func(listPossibleCallback); -#endif + auto _guard = interacter->init(static_cast(this)); std::string input; @@ -307,7 +199,7 @@ ReplExitStatus NixRepl::mainLoop() logger->pause(); // When continuing input from previous lines, don't print a prompt, just align to the same // number of chars as the prompt. - if (!getLine(input, input.empty() ? "nix-repl> " : " ")) { + if (!interacter->getLine(input, input.empty() ? ReplPromptType::ReplPrompt : ReplPromptType::ContinuationPrompt)) { // Ctrl-D should exit the debugger. state->debugStop = false; logger->cout(""); @@ -325,7 +217,7 @@ ReplExitStatus NixRepl::mainLoop() case ProcessLineResult::PromptAgain: break; default: - abort(); + unreachable(); } } catch (ParseError & e) { if (e.msg().find("unexpected end of file") != std::string::npos) { @@ -336,13 +228,7 @@ ReplExitStatus NixRepl::mainLoop() printMsg(lvlError, e.msg()); } } catch (EvalError & e) { - // in debugger mode, an EvalError should trigger another repl session. - // when that session returns the exception will land here. No need to show it again; - // show the error for this repl session instead. - if (state->debugRepl && !state->debugTraces.empty()) - showDebugTrace(std::cout, state->positions, state->debugTraces.front()); - else - printMsg(lvlError, e.msg()); + printMsg(lvlError, e.msg()); } catch (Error & e) { printMsg(lvlError, e.msg()); } catch (Interrupted & e) { @@ -356,51 +242,6 @@ ReplExitStatus NixRepl::mainLoop() } } - -bool NixRepl::getLine(std::string & input, const std::string & prompt) -{ - struct sigaction act, old; - sigset_t savedSignalMask, set; - - auto setupSignals = [&]() { - act.sa_handler = sigintHandler; - sigfillset(&act.sa_mask); - act.sa_flags = 0; - if (sigaction(SIGINT, &act, &old)) - throw SysError("installing handler for SIGINT"); - - sigemptyset(&set); - sigaddset(&set, SIGINT); - if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask)) - throw SysError("unblocking SIGINT"); - }; - auto restoreSignals = [&]() { - if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) - throw SysError("restoring signals"); - - if (sigaction(SIGINT, &old, 0)) - throw SysError("restoring handler for SIGINT"); - }; - - setupSignals(); - char * s = readline(prompt.c_str()); - Finally doFree([&]() { free(s); }); - restoreSignals(); - - if (g_signal_received) { - g_signal_received = 0; - input.clear(); - return true; - } - - if (!s) - return false; - input += s; - input += '\n'; - return true; -} - - StringSet NixRepl::completePrefix(const std::string & prefix) { StringSet completions; @@ -421,11 +262,14 @@ StringSet NixRepl::completePrefix(const std::string & prefix) try { auto dir = std::string(cur, 0, slash); auto prefix2 = std::string(cur, slash + 1); - for (auto & entry : readDirectory(dir == "" ? "/" : dir)) { - if (entry.name[0] != '.' && hasPrefix(entry.name, prefix2)) - completions.insert(prev + dir + "/" + entry.name); + for (auto & entry : std::filesystem::directory_iterator{dir == "" ? "/" : dir}) { + checkInterrupt(); + auto name = entry.path().filename().string(); + if (name[0] != '.' && hasPrefix(name, prefix2)) + completions.insert(prev + entry.path().string()); } } catch (Error &) { + } catch (std::filesystem::filesystem_error &) { } } else if ((dot = cur.rfind('.')) == std::string::npos) { /* This is a variable name; look it up in the current scope. */ @@ -452,7 +296,7 @@ StringSet NixRepl::completePrefix(const std::string & prefix) e->eval(*state, *env, v); state->forceAttrs(v, noPos, "while evaluating an attrset for the purpose of completion (this error should not be displayed; file an issue?)"); - for (auto & i : *v.attrs) { + for (auto & i : *v.attrs()) { std::string_view name = state->symbols[i.name]; if (name.substr(0, cur2.size()) != cur2) continue; completions.insert(concatStrings(prev, expr, ".", name)); @@ -464,6 +308,8 @@ StringSet NixRepl::completePrefix(const std::string & prefix) // Quietly ignore evaluation errors. } catch (BadURL & e) { // Quietly ignore BadURL flake-related errors. + } catch (FileNotFound & e) { + // Quietly ignore non-existent file beeing `import`-ed. } } @@ -519,7 +365,7 @@ ProcessLineResult NixRepl::processLine(std::string line) if (line.empty()) return ProcessLineResult::PromptAgain; - _isInterrupted = false; + setInterrupted(false); std::string command, arg; @@ -548,6 +394,7 @@ ProcessLineResult NixRepl::processLine(std::string line) << " :l, :load Load Nix expression and add it to scope\n" << " :lf, :load-flake Load Nix flake and add it to scope\n" << " :p, :print Evaluate and print expression recursively\n" + << " Strings are printed directly, without escaping.\n" << " :q, :quit Exit nix-repl\n" << " :r, :reload Reload all files\n" << " :sh Build dependencies of derivation, then start\n" @@ -651,7 +498,7 @@ ProcessLineResult NixRepl::processLine(std::string line) auto path = state->coerceToPath(noPos, v, context, "while evaluating the filename to edit"); return {path, 0}; } else if (v.isLambda()) { - auto pos = state->positions[v.lambda.fun->pos]; + auto pos = state->positions[v.payload.lambda.fun->pos]; if (auto path = std::get_if(&pos.origin)) return {*path, pos.line}; else @@ -669,7 +516,7 @@ ProcessLineResult NixRepl::processLine(std::string line) // runProgram redirects stdout to a StringSink, // using runProgram2 to allow editors to display their UI - runProgram2(RunOptions { .program = editor, .searchPath = true, .args = args }); + runProgram2(RunOptions { .program = editor, .lookupPath = true, .args = args , .isInteractive = true }); // Reload right after exiting the editor state->resetFileCache(); @@ -755,7 +602,11 @@ ProcessLineResult NixRepl::processLine(std::string line) else if (command == ":p" || command == ":print") { Value v; evalString(arg, v); - printValue(std::cout, v); + if (v.type() == nString) { + std::cout << v.string_view(); + } else { + printValue(std::cout, v); + } std::cout << std::endl; } @@ -766,6 +617,35 @@ ProcessLineResult NixRepl::processLine(std::string line) else if (command == ":doc") { Value v; + + auto expr = parseString(arg); + std::string fallbackName; + PosIdx fallbackPos; + DocComment fallbackDoc; + if (auto select = dynamic_cast(expr)) { + Value vAttrs; + auto name = select->evalExceptFinalSelect(*state, *env, vAttrs); + fallbackName = state->symbols[name]; + + state->forceAttrs(vAttrs, noPos, "while evaluating an attribute set to look for documentation"); + auto attrs = vAttrs.attrs(); + assert(attrs); + auto attr = attrs->get(name); + if (!attr) { + // When missing, trigger the normal exception + // e.g. :doc builtins.foo + // behaves like + // nix-repl> builtins.foo + // error: attribute 'foo' missing + evalString(arg, v); + assert(false); + } + if (attr->pos) { + fallbackPos = attr->pos; + fallbackDoc = state->getDocCommentForPos(fallbackPos); + } + } + evalString(arg, v); if (auto doc = state->getDoc(v)) { std::string markdown; @@ -783,6 +663,19 @@ ProcessLineResult NixRepl::processLine(std::string line) markdown += stripIndentation(doc->doc); logger->cout(trim(renderMarkdownToTerminal(markdown))); + } else if (fallbackPos) { + std::stringstream ss; + ss << "Attribute `" << fallbackName << "`\n\n"; + ss << " … defined at " << state->positions[fallbackPos] << "\n\n"; + if (fallbackDoc) { + ss << fallbackDoc.getInnerText(state->positions); + } else { + ss << "No documentation found.\n\n"; + } + + auto markdown = ss.str(); + logger->cout(trim(renderMarkdownToTerminal(markdown))); + } else throw Error("value does not have documentation"); } @@ -840,14 +733,14 @@ void NixRepl::loadFlake(const std::string & flakeRefS) if (flakeRefS.empty()) throw Error("cannot use ':load-flake' without a path specified. (Use '.' for the current working directory.)"); - auto flakeRef = parseFlakeRef(flakeRefS, absPath("."), true); + auto flakeRef = parseFlakeRef(fetchSettings, flakeRefS, absPath("."), true); if (evalSettings.pureEval && !flakeRef.input.isLocked()) throw Error("cannot use ':load-flake' on locked flake reference '%s' (use --impure to override)", flakeRefS); Value v; flake::callFlake(*state, - flake::lockFlake(*state, flakeRef, + flake::lockFlake(flakeSettings, *state, flakeRef, flake::LockFlags { .updateLockFile = false, .useRegistries = !evalSettings.pureEval, @@ -899,17 +792,17 @@ void NixRepl::loadFiles() void NixRepl::addAttrsToScope(Value & attrs) { state->forceAttrs(attrs, [&]() { return attrs.determinePos(noPos); }, "while evaluating an attribute set to be merged in the global scope"); - if (displ + attrs.attrs->size() >= envSize) + if (displ + attrs.attrs()->size() >= envSize) throw Error("environment full; cannot add more variables"); - for (auto & i : *attrs.attrs) { + for (auto & i : *attrs.attrs()) { staticEnv->vars.emplace_back(i.name, displ); env->values[displ++] = i.value; varNames.emplace(state->symbols[i.name]); } staticEnv->sort(); staticEnv->deduplicate(); - notice("Added %1% variables.", attrs.attrs->size()); + notice("Added %1% variables.", attrs.attrs()->size()); } @@ -941,11 +834,11 @@ void NixRepl::evalString(std::string s, Value & v) std::unique_ptr AbstractNixRepl::create( - const SearchPath & searchPath, nix::ref store, ref state, + const LookupPath & lookupPath, nix::ref store, ref state, std::function getValues) { return std::make_unique( - searchPath, + lookupPath, openStore(), state, getValues @@ -961,9 +854,9 @@ ReplExitStatus AbstractNixRepl::runSimple( NixRepl::AnnotatedValues values; return values; }; - SearchPath searchPath = {}; + LookupPath lookupPath = {}; auto repl = std::make_unique( - searchPath, + lookupPath, openStore(), evalState, getValues diff --git a/src/libcmd/repl.hh b/src/libcmd/repl.hh index 21aa8bfc7..3fd4b2c39 100644 --- a/src/libcmd/repl.hh +++ b/src/libcmd/repl.hh @@ -3,11 +3,6 @@ #include "eval.hh" -#if HAVE_BOEHMGC -#define GC_INCLUDE_NEW -#include -#endif - namespace nix { struct AbstractNixRepl @@ -25,7 +20,7 @@ struct AbstractNixRepl typedef std::vector> AnnotatedValues; static std::unique_ptr create( - const SearchPath & searchPath, nix::ref store, ref state, + const LookupPath & lookupPath, nix::ref store, ref state, std::function getValues); static ReplExitStatus runSimple( diff --git a/src/libexpr-c/.version b/src/libexpr-c/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libexpr-c/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libexpr-c/build-utils-meson b/src/libexpr-c/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libexpr-c/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libexpr-c/local.mk b/src/libexpr-c/local.mk new file mode 100644 index 000000000..227a4095b --- /dev/null +++ b/src/libexpr-c/local.mk @@ -0,0 +1,25 @@ +libraries += libexprc + +libexprc_NAME = libnixexprc + +libexprc_DIR := $(d) + +libexprc_SOURCES := \ + $(wildcard $(d)/*.cc) \ + +# Not just for this library itself, but also for downstream libraries using this library + +INCLUDE_libexprc := -I $(d) +libexprc_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libutilc) \ + $(INCLUDE_libfetchers) \ + $(INCLUDE_libstore) $(INCLUDE_libstorec) \ + $(INCLUDE_libexpr) $(INCLUDE_libexprc) + +libexprc_LIBS = libutil libutilc libstore libstorec libfetchers libexpr + +libexprc_LDFLAGS += $(THREAD_LDFLAGS) + +$(eval $(call install-file-in, $(d)/nix-expr-c.pc, $(libdir)/pkgconfig, 0644)) + +libexprc_FORCE_INSTALL := 1 + diff --git a/src/libexpr-c/meson.build b/src/libexpr-c/meson.build new file mode 100644 index 000000000..6db5b83b8 --- /dev/null +++ b/src/libexpr-c/meson.build @@ -0,0 +1,93 @@ +project('nix-expr-c', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), + dependency('nix-expr'), +] +deps_public_maybe_subproject = [ + dependency('nix-util-c'), + dependency('nix-store-c'), +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +# TODO rename, because it will conflict with downstream projects +configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) + +config_h = configure_file( + configuration : configdata, + output : 'config-expr.h', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + + # From C++ libraries, only for internals + '-include', 'config-util.hh', + '-include', 'config-store.hh', + '-include', 'config-expr.hh', + + # From C libraries, for our public, installed headers too + '-include', 'config-util.h', + '-include', 'config-store.h', + '-include', 'config-expr.h', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'nix_api_expr.cc', + 'nix_api_external.cc', + 'nix_api_value.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'nix_api_expr.h', + 'nix_api_external.h', + 'nix_api_value.h', +) + +# TODO move this header to libexpr, maybe don't use it in tests? +headers += files('nix_api_expr_internal.h') + +subdir('build-utils-meson/export-all-symbols') + +this_library = library( + 'nixexprc', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + link_args: linker_export_flags, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libexpr-c/nix-expr-c.pc.in b/src/libexpr-c/nix-expr-c.pc.in new file mode 100644 index 000000000..06897064d --- /dev/null +++ b/src/libexpr-c/nix-expr-c.pc.in @@ -0,0 +1,10 @@ +prefix=@prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: Nix +Description: Nix Language Evaluator - C API +Version: @PACKAGE_VERSION@ +Requires: nix-store-c +Libs: -L${libdir} -lnixexprc +Cflags: -I${includedir}/nix diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc new file mode 100644 index 000000000..8f21d7022 --- /dev/null +++ b/src/libexpr-c/nix_api_expr.cc @@ -0,0 +1,213 @@ +#include +#include +#include + +#include "eval.hh" +#include "eval-gc.hh" +#include "globals.hh" +#include "eval-settings.hh" + +#include "nix_api_expr.h" +#include "nix_api_expr_internal.h" +#include "nix_api_store.h" +#include "nix_api_store_internal.h" +#include "nix_api_util.h" +#include "nix_api_util_internal.h" + +#if HAVE_BOEHMGC +# include +# define GC_INCLUDE_NEW 1 +# include "gc_cpp.h" +#endif + +nix_err nix_libexpr_init(nix_c_context * context) +{ + if (context) + context->last_err_code = NIX_OK; + { + auto ret = nix_libutil_init(context); + if (ret != NIX_OK) + return ret; + } + { + auto ret = nix_libstore_init(context); + if (ret != NIX_OK) + return ret; + } + try { + nix::initGC(); + } + NIXC_CATCH_ERRS +} + +nix_err nix_expr_eval_from_string( + nix_c_context * context, EvalState * state, const char * expr, const char * path, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + nix::Expr * parsedExpr = state->state.parseExprFromString(expr, state->state.rootPath(nix::CanonPath(path))); + state->state.eval(parsedExpr, value->value); + state->state.forceValue(value->value, nix::noPos); + } + NIXC_CATCH_ERRS +} + +nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, nix_value * arg, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + state->state.callFunction(fn->value, arg->value, value->value, nix::noPos); + state->state.forceValue(value->value, nix::noPos); + } + NIXC_CATCH_ERRS +} + +nix_err nix_value_call_multi(nix_c_context * context, EvalState * state, nix_value * fn, size_t nargs, nix_value ** args, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + state->state.callFunction(fn->value, nargs, (nix::Value * *)args, value->value, nix::noPos); + state->state.forceValue(value->value, nix::noPos); + } + NIXC_CATCH_ERRS +} + +nix_err nix_value_force(nix_c_context * context, EvalState * state, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + state->state.forceValue(value->value, nix::noPos); + } + NIXC_CATCH_ERRS +} + +nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + state->state.forceValueDeep(value->value); + } + NIXC_CATCH_ERRS +} + +EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c, Store * store) +{ + if (context) + context->last_err_code = NIX_OK; + try { + nix::Strings lookupPath; + if (lookupPath_c != nullptr) + for (size_t i = 0; lookupPath_c[i] != nullptr; i++) + lookupPath.push_back(lookupPath_c[i]); + + void * p = ::operator new( + sizeof(EvalState), + static_cast(alignof(EvalState))); + auto * p2 = static_cast(p); + new (p) EvalState { + .fetchSettings = nix::fetchers::Settings{}, + .settings = nix::EvalSettings{ + nix::settings.readOnlyMode, + }, + .state = nix::EvalState( + nix::LookupPath::parse(lookupPath), + store->ptr, + p2->fetchSettings, + p2->settings), + }; + loadConfFile(p2->settings); + return p2; + } + NIXC_CATCH_ERRS_NULL +} + +void nix_state_free(EvalState * state) +{ + delete state; +} + +#if HAVE_BOEHMGC +std::unordered_map< + const void *, + unsigned int, + std::hash, + std::equal_to, + traceable_allocator
; } - > ``` - > - > For human-readable output, `nix eval` (experimental) is more informative: - > - > ```console - > $ nix-instantiate --eval --expr 'a: a' - > - > $ nix eval --expr 'a: a' - > «lambda @ «string»:1:1» - > ``` - > - > For machine-readable output, the `--xml` option produces unambiguous - > output: - > - > ```console - > $ nix-instantiate --eval --xml --expr '{ foo = ; }' - > - > - > - > - > - > - > - > - > ``` + Just parse the input files, and print their abstract syntax trees on + standard output as a Nix expression. - - `--find-file`\ - Look up the given files in Nix’s search path (as specified by the - `NIX_PATH` environment variable). If found, print the corresponding - absolute paths on standard output. For instance, if `NIX_PATH` is - `nixpkgs=/home/alice/nixpkgs`, then `nix-instantiate --find-file - nixpkgs/default.nix` will print `/home/alice/nixpkgs/default.nix`. +- `--eval` - - `--strict`\ - When used with `--eval`, recursively evaluate list elements and - attributes. Normally, such sub-expressions are left unevaluated - (since the Nix language is lazy). + Just parse and evaluate the input files, and print the resulting + values on standard output. No instantiation of store derivations + takes place. - > **Warning** - > - > This option can cause non-termination, because lazy data - > structures can be infinitely large. + > **Warning** + > + > This option produces output which can be parsed as a Nix expression which + > will produce a different result than the input expression when evaluated. + > For example, these two Nix expressions print the same result despite + > having different meaning: + > + > ```console + > $ nix-instantiate --eval --expr '{ a = {}; }' + > { a = ; } + > $ nix-instantiate --eval --expr '{ a = ; }' + > { a = ; } + > ``` + > + > For human-readable output, `nix eval` (experimental) is more informative: + > + > ```console + > $ nix-instantiate --eval --expr 'a: a' + > + > $ nix eval --expr 'a: a' + > «lambda @ «string»:1:1» + > ``` + > + > For machine-readable output, the `--xml` option produces unambiguous + > output: + > + > ```console + > $ nix-instantiate --eval --xml --expr '{ foo = ; }' + > + > + > + > + > + > + > + > + > ``` - - `--json`\ - When used with `--eval`, print the resulting value as an JSON - representation of the abstract syntax tree rather than as a Nix expression. +- `--find-file` - - `--xml`\ - When used with `--eval`, print the resulting value as an XML - representation of the abstract syntax tree rather than as a Nix expression. - The schema is the same as that used by the [`toXML` - built-in](../language/builtins.md). + Look up the given files in Nix’s search path (as specified by the + `NIX_PATH` environment variable). If found, print the corresponding + absolute paths on standard output. For instance, if `NIX_PATH` is + `nixpkgs=/home/alice/nixpkgs`, then `nix-instantiate --find-file + nixpkgs/default.nix` will print `/home/alice/nixpkgs/default.nix`. - - `--read-write-mode`\ - When used with `--eval`, perform evaluation in read/write mode so - nix language features that require it will still work (at the cost - of needing to do instantiation of every evaluated derivation). If - this option is not enabled, there may be uninstantiated store paths - in the final output. +- `--strict` + + When used with `--eval`, recursively evaluate list elements and + attributes. Normally, such sub-expressions are left unevaluated + (since the Nix language is lazy). + + > **Warning** + > + > This option can cause non-termination, because lazy data + > structures can be infinitely large. + +- `--json` + + When used with `--eval`, print the resulting value as an JSON + representation of the abstract syntax tree rather than as a Nix expression. + +- `--xml` + + When used with `--eval`, print the resulting value as an XML + representation of the abstract syntax tree rather than as a Nix expression. + The schema is the same as that used by the [`toXML` + built-in](../language/builtins.md). + +- `--read-write-mode` + + When used with `--eval`, perform evaluation in read/write mode so + nix language features that require it will still work (at the cost + of needing to do instantiation of every evaluated derivation). If + this option is not enabled, there may be uninstantiated store paths + in the final output. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-prefetch-url.md b/doc/manual/src/command-ref/nix-prefetch-url.md index 45ef01e02..ffab94b8a 100644 --- a/doc/manual/src/command-ref/nix-prefetch-url.md +++ b/doc/manual/src/command-ref/nix-prefetch-url.md @@ -39,27 +39,32 @@ the path of the downloaded file in the Nix store is also printed. # Options - - `--type` *hashAlgo*\ - Use the specified cryptographic hash algorithm, - which can be one of `md5`, `sha1`, `sha256`, and `sha512`. - The default is `sha256`. +- `--type` *hashAlgo* - - `--print-path`\ - Print the store path of the downloaded file on standard output. + Use the specified cryptographic hash algorithm, + which can be one of `md5`, `sha1`, `sha256`, and `sha512`. + The default is `sha256`. - - `--unpack`\ - Unpack the archive (which must be a tarball or zip file) and add the - result to the Nix store. The resulting hash can be used with - functions such as Nixpkgs’s `fetchzip` or `fetchFromGitHub`. +- `--print-path` - - `--executable`\ - Set the executable bit on the downloaded file. + Print the store path of the downloaded file on standard output. - - `--name` *name*\ - Override the name of the file in the Nix store. By default, this is - `hash-basename`, where *basename* is the last component of *url*. - Overriding the name is necessary when *basename* contains characters - that are not allowed in Nix store paths. +- `--unpack` + + Unpack the archive (which must be a tarball or zip file) and add the + result to the Nix store. The resulting hash can be used with + functions such as Nixpkgs’s `fetchzip` or `fetchFromGitHub`. + +- `--executable` + + Set the executable bit on the downloaded file. + +- `--name` *name* + + Override the name of the file in the Nix store. By default, this is + `hash-basename`, where *basename* is the last component of *url*. + Overriding the name is necessary when *basename* contains characters + that are not allowed in Nix store paths. # Examples diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index 1eaf3c36a..69a711bd5 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -60,55 +60,63 @@ All options not listed here are passed to `nix-store --realise`, except for `--arg` and `--attr` / `-A` which are passed to `nix-instantiate`. - - `--command` *cmd*\ - In the environment of the derivation, run the shell command *cmd*. - This command is executed in an interactive shell. (Use `--run` to - use a non-interactive shell instead.) However, a call to `exit` is - implicitly added to the command, so the shell will exit after - running the command. To prevent this, add `return` at the end; - e.g. `--command "echo Hello; return"` will print `Hello` and then - drop you into the interactive shell. This can be useful for doing - any additional initialisation. +- `--command` *cmd* - - `--run` *cmd*\ - Like `--command`, but executes the command in a non-interactive - shell. This means (among other things) that if you hit Ctrl-C while - the command is running, the shell exits. + In the environment of the derivation, run the shell command *cmd*. + This command is executed in an interactive shell. (Use `--run` to + use a non-interactive shell instead.) However, a call to `exit` is + implicitly added to the command, so the shell will exit after + running the command. To prevent this, add `return` at the end; + e.g. `--command "echo Hello; return"` will print `Hello` and then + drop you into the interactive shell. This can be useful for doing + any additional initialisation. - - `--exclude` *regexp*\ - Do not build any dependencies whose store path matches the regular - expression *regexp*. This option may be specified multiple times. +- `--run` *cmd* - - `--pure`\ - If this flag is specified, the environment is almost entirely - cleared before the interactive shell is started, so you get an - environment that more closely corresponds to the “real†Nix build. A - few variables, in particular `HOME`, `USER` and `DISPLAY`, are - retained. + Like `--command`, but executes the command in a non-interactive + shell. This means (among other things) that if you hit Ctrl-C while + the command is running, the shell exits. - - `--packages` / `-p` *packages*…\ - Set up an environment in which the specified packages are present. - The command line arguments are interpreted as attribute names inside - the Nix Packages collection. Thus, `nix-shell --packages libjpeg openjdk` - will start a shell in which the packages denoted by the attribute - names `libjpeg` and `openjdk` are present. +- `--exclude` *regexp* - - `-i` *interpreter*\ - The chained script interpreter to be invoked by `nix-shell`. Only - applicable in `#!`-scripts (described below). + Do not build any dependencies whose store path matches the regular + expression *regexp*. This option may be specified multiple times. - - `--keep` *name*\ - When a `--pure` shell is started, keep the listed environment - variables. +- `--pure` + + If this flag is specified, the environment is almost entirely + cleared before the interactive shell is started, so you get an + environment that more closely corresponds to the “real†Nix build. A + few variables, in particular `HOME`, `USER` and `DISPLAY`, are + retained. + +- `--packages` / `-p` *packages*… + + Set up an environment in which the specified packages are present. + The command line arguments are interpreted as attribute names inside + the Nix Packages collection. Thus, `nix-shell --packages libjpeg openjdk` + will start a shell in which the packages denoted by the attribute + names `libjpeg` and `openjdk` are present. + +- `-i` *interpreter* + + The chained script interpreter to be invoked by `nix-shell`. Only + applicable in `#!`-scripts (described below). + +- `--keep` *name* + + When a `--pure` shell is started, keep the listed environment + variables. {{#include ./opt-common.md}} # Environment variables - - `NIX_BUILD_SHELL`\ - Shell used to start the interactive environment. Defaults to the - `bash` found in ``, falling back to the `bash` found in - `PATH` if not found. +- `NIX_BUILD_SHELL` + + Shell used to start the interactive environment. Defaults to the + `bash` found in ``, falling back to the `bash` found in + `PATH` if not found. {{#include ./env-common.md}} @@ -202,14 +210,14 @@ For example, here is a Python script that depends on Python and the ```python #! /usr/bin/env nix-shell -#! nix-shell -i python --packages python pythonPackages.prettytable +#! nix-shell -i python3 --packages python3 python3Packages.prettytable import prettytable # Print a simple table. t = prettytable.PrettyTable(["N", "N^2"]) for n in range(1, 10): t.add_row([n, n * n]) -print t +print(t) ``` Similarly, the following is a Perl script that specifies that it @@ -289,3 +297,8 @@ with import {}; runCommand "dummy" { buildInputs = [ python pythonPackages.prettytable ]; } "" ``` + +The script's file name is passed as the first argument to the interpreter specified by the `-i` flag. + +Aside from the very first line, which is a directive to the operating system, the additional `#! nix-shell` lines do not need to be at the beginning of the file. +This allows wrapping them in block comments for languages where `#` does not start a comment, such as ECMAScript, Erlang, PHP, or Ruby. diff --git a/doc/manual/src/command-ref/nix-store/add-fixed.md b/doc/manual/src/command-ref/nix-store/add-fixed.md index d25db091c..bebf15026 100644 --- a/doc/manual/src/command-ref/nix-store/add-fixed.md +++ b/doc/manual/src/command-ref/nix-store/add-fixed.md @@ -16,9 +16,10 @@ public url or broke since the download expression was written. This operation has the following options: - - `--recursive`\ - Use recursive instead of flat hashing mode, used when adding - directories to the store. +- `--recursive` + + Use recursive instead of flat hashing mode, used when adding + directories to the store. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/dump.md b/doc/manual/src/command-ref/nix-store/dump.md index c2f3c42ef..3de0e27b0 100644 --- a/doc/manual/src/command-ref/nix-store/dump.md +++ b/doc/manual/src/command-ref/nix-store/dump.md @@ -1,6 +1,6 @@ # Name -`nix-store --dump` - write a single path to a Nix Archive +`nix-store --dump` - write a single path to a [Nix Archive] ## Synopsis @@ -8,7 +8,7 @@ ## Description -The operation `--dump` produces a NAR (Nix ARchive) file containing the +The operation `--dump` produces a [Nix archive](@docroot@/glossary.md#gloss-nar) (NAR) file containing the contents of the file system tree rooted at *path*. The archive is written to standard output. @@ -30,8 +30,9 @@ NAR archives support filenames of unlimited length and 64-bit file sizes. They can contain regular files, directories, and symbolic links, but not other types of files (such as device nodes). -A Nix archive can be unpacked using `nix-store ---restore`. +A Nix archive can be unpacked using [`nix-store --restore`](@docroot@/command-ref/nix-store/restore.md). + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/export.md b/doc/manual/src/command-ref/nix-store/export.md index 1bc46f53b..ba772eb43 100644 --- a/doc/manual/src/command-ref/nix-store/export.md +++ b/doc/manual/src/command-ref/nix-store/export.md @@ -1,6 +1,6 @@ # Name -`nix-store --export` - export store paths to a Nix Archive +`nix-store --export` - export store paths to a [Nix Archive] ## Synopsis @@ -8,16 +8,22 @@ ## Description -The operation `--export` writes a serialisation of the specified store -paths to standard output in a format that can be imported into another -Nix store with `nix-store --import`. This is like `nix-store ---dump`, except that the NAR archive produced by that command doesn’t -contain the necessary meta-information to allow it to be imported into -another Nix store (namely, the set of references of the path). +The operation `--export` writes a serialisation of the given [store objects](@docroot@/glossary.md#gloss-store-object) to standard output in a format that can be imported into another [Nix store](@docroot@/store/index.md) with [`nix-store --import`](./import.md). -This command does not produce a *closure* of the specified paths, so if -a store path references other store paths that are missing in the target -Nix store, the import will fail. +> **Warning** +> +> This command *does not* produce a [closure](@docroot@/glossary.md#gloss-closure) of the specified store paths. +> Trying to import a store object that refers to store paths not available in the target Nix store will fail. +> +> Use [`nix-store --query`](@docroot@/command-ref/nix-store/query.md) to obtain the closure of a store path. + +This command is different from [`nix-store --dump`](./dump.md), which produces a [Nix archive](@docroot@/glossary.md#gloss-nar) that *does not* contain the set of [references](@docroot@/glossary.md#gloss-reference) of a given store path. + +> **Note** +> +> For efficient transfer of closures to remote machines over SSH, use [`nix-copy-closure`](@docroot@/command-ref/nix-copy-closure.md). + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive {{#include ./opt-common.md}} @@ -27,15 +33,21 @@ Nix store, the import will fail. # Examples -To copy a whole closure, do something -like: - -```console -$ nix-store --export $(nix-store --query --requisites paths) > out -``` - -To import the whole closure again, run: - -```console -$ nix-store --import < out -``` +> **Example** +> +> Deploy GNU Hello to an airgapped machine via USB stick. +> +> Write the closure to the block device on a machine with internet connection: +> +> ```shell-session +> [alice@itchy]$ storePath=$(nix-build '' -I nixpkgs=channel:nixpkgs-unstable -A hello --no-out-link) +> [alice@itchy]$ nix-store --export $(nix-store --query --requisites $storePath) | sudo dd of=/dev/usb +> ``` +> +> Read the closure from the block device on the machine without internet connection: +> +> ```shell-session +> [bob@scratchy]$ hello=$(sudo dd if=/dev/usb | nix-store --import | tail -1) +> [bob@scratchy]$ $hello/bin/hello +> Hello, world! +> ``` diff --git a/doc/manual/src/command-ref/nix-store/gc.md b/doc/manual/src/command-ref/nix-store/gc.md index 7be0d559a..f432e00eb 100644 --- a/doc/manual/src/command-ref/nix-store/gc.md +++ b/doc/manual/src/command-ref/nix-store/gc.md @@ -14,30 +14,34 @@ reachable via file system references from a set of “rootsâ€, are deleted. The following suboperations may be specified: - - `--print-roots`\ - This operation prints on standard output the set of roots used by - the garbage collector. +- `--print-roots` - - `--print-live`\ - This operation prints on standard output the set of “live†store - paths, which are all the store paths reachable from the roots. Live - paths should never be deleted, since that would break consistency — - it would become possible that applications are installed that - reference things that are no longer present in the store. + This operation prints on standard output the set of roots used by + the garbage collector. - - `--print-dead`\ - This operation prints out on standard output the set of “dead†store - paths, which is just the opposite of the set of live paths: any path - in the store that is not live (with respect to the roots) is dead. +- `--print-live` + + This operation prints on standard output the set of “live†store + paths, which are all the store paths reachable from the roots. Live + paths should never be deleted, since that would break consistency — + it would become possible that applications are installed that + reference things that are no longer present in the store. + +- `--print-dead` + + This operation prints out on standard output the set of “dead†store + paths, which is just the opposite of the set of live paths: any path + in the store that is not live (with respect to the roots) is dead. By default, all unreachable paths are deleted. The following options control what gets deleted and in what order: - - `--max-freed` *bytes*\ - Keep deleting paths until at least *bytes* bytes have been deleted, - then stop. The argument *bytes* can be followed by the - multiplicative suffix `K`, `M`, `G` or `T`, denoting KiB, MiB, GiB - or TiB units. +- `--max-freed` *bytes* + + Keep deleting paths until at least *bytes* bytes have been deleted, + then stop. The argument *bytes* can be followed by the + multiplicative suffix `K`, `M`, `G` or `T`, denoting KiB, MiB, GiB + or TiB units. The behaviour of the collector is also influenced by the `keep-outputs` and `keep-derivations` settings in the Nix diff --git a/doc/manual/src/command-ref/nix-store/import.md b/doc/manual/src/command-ref/nix-store/import.md index 2711316a7..3f6b3d076 100644 --- a/doc/manual/src/command-ref/nix-store/import.md +++ b/doc/manual/src/command-ref/nix-store/import.md @@ -1,6 +1,8 @@ # Name -`nix-store --import` - import Nix Archive into the store +`nix-store --import` - import [Nix Archive] into the store + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive # Synopsis @@ -8,14 +10,34 @@ # Description -The operation `--import` reads a serialisation of a set of store paths -produced by `nix-store --export` from standard input and adds those -store paths to the Nix store. Paths that already exist in the Nix store -are ignored. If a path refers to another path that doesn’t exist in the -Nix store, the import fails. +The operation `--import` reads a serialisation of a set of [store objects](@docroot@/glossary.md#gloss-store-object) produced by [`nix-store --export`](./export.md) from standard input, and adds those store objects to the specified [Nix store](@docroot@/store/index.md). +Paths that already exist in the target Nix store are ignored. +If a path [refers](@docroot@/glossary.md#gloss-reference) to another path that doesn’t exist in the target Nix store, the import fails. + +> **Note** +> +> For efficient transfer of closures to remote machines over SSH, use [`nix-copy-closure`](@docroot@/command-ref/nix-copy-closure.md). {{#include ./opt-common.md}} {{#include ../opt-common.md}} {{#include ../env-common.md}} + +# Examples + +> **Example** +> +> Given a closure of GNU Hello as a file: +> +> ```shell-session +> $ storePath="$(nix-build '' -I nixpkgs=channel:nixpkgs-unstable -A hello --no-out-link)" +> $ nix-store --export $(nix-store --query --requisites $storePath) > hello.closure +> ``` +> +> Import the closure into a [remote SSH store](@docroot@/store/types/ssh-store.md) using the [`--store`](@docroot@/command-ref/conf-file.md#conf-store) option: +> +> ```console +> $ nix-store --import --store ssh://alice@itchy.example.org < hello.closure +> ``` + diff --git a/doc/manual/src/command-ref/nix-store/optimise.md b/doc/manual/src/command-ref/nix-store/optimise.md index dc392aeb8..b257466b2 100644 --- a/doc/manual/src/command-ref/nix-store/optimise.md +++ b/doc/manual/src/command-ref/nix-store/optimise.md @@ -12,7 +12,7 @@ The operation `--optimise` reduces Nix store disk space usage by finding identical files in the store and hard-linking them to each other. It typically reduces the size of the store by something like 25-35%. Only regular files and symlinks are hard-linked in this manner. Files are -considered identical when they have the same NAR archive serialisation: +considered identical when they have the same [Nix Archive (NAR)][Nix Archive] serialisation: that is, regular files must have the same contents and permission (executable or non-executable), and symlinks must have the same contents. @@ -38,3 +38,4 @@ hashing files in `/nix/store/qhqx7l2f1kmwihc9bnxs7rc159hsxnf3-gcc-4.1.1' there are 114486 files with equal contents out of 215894 files in total ``` +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive diff --git a/doc/manual/src/command-ref/nix-store/query.md b/doc/manual/src/command-ref/nix-store/query.md index a158c76aa..b4efa734e 100644 --- a/doc/manual/src/command-ref/nix-store/query.md +++ b/doc/manual/src/command-ref/nix-store/query.md @@ -24,122 +24,138 @@ symlink. # Common query options - - `--use-output`; `-u`\ - For each argument to the query that is a [store derivation], apply the - query to the output path of the derivation instead. +- `--use-output` / `-u` - - `--force-realise`; `-f`\ - Realise each argument to the query first (see [`nix-store --realise`](./realise.md)). + For each argument to the query that is a [store derivation], apply the + query to the output path of the derivation instead. + +- `--force-realise` / `-f` + + Realise each argument to the query first (see [`nix-store --realise`](./realise.md)). [store derivation]: @docroot@/glossary.md#gloss-store-derivation # Queries - - `--outputs`\ - Prints out the [output paths] of the store - derivations *paths*. These are the paths that will be produced when - the derivation is built. +- `--outputs` - [output paths]: ../../glossary.md#gloss-output-path + Prints out the [output paths] of the store + derivations *paths*. These are the paths that will be produced when + the derivation is built. - - `--requisites`; `-R`\ - Prints out the [closure] of the store path *paths*. + [output paths]: @docroot@/glossary.md#gloss-output-path - [closure]: ../../glossary.md#gloss-closure +- `--requisites` / `-R` - This query has one option: + Prints out the [closure] of the store path *paths*. - - `--include-outputs` - Also include the existing output paths of [store derivation]s, - and their closures. + [closure]: @docroot@/glossary.md#gloss-closure - This query can be used to implement various kinds of deployment. A - *source deployment* is obtained by distributing the closure of a - store derivation. A *binary deployment* is obtained by distributing - the closure of an output path. A *cache deployment* (combined - source/binary deployment, including binaries of build-time-only - dependencies) is obtained by distributing the closure of a store - derivation and specifying the option `--include-outputs`. + This query has one option: - - `--references`\ - Prints the set of [references] of the store paths - *paths*, that is, their immediate dependencies. (For *all* - dependencies, use `--requisites`.) + - `--include-outputs` + Also include the existing output paths of [store derivation]s, + and their closures. - [references]: ../../glossary.md#gloss-reference + This query can be used to implement various kinds of deployment. A + *source deployment* is obtained by distributing the closure of a + store derivation. A *binary deployment* is obtained by distributing + the closure of an output path. A *cache deployment* (combined + source/binary deployment, including binaries of build-time-only + dependencies) is obtained by distributing the closure of a store + derivation and specifying the option `--include-outputs`. - - `--referrers`\ - Prints the set of *referrers* of the store paths *paths*, that is, - the store paths currently existing in the Nix store that refer to - one of *paths*. Note that contrary to the references, the set of - referrers is not constant; it can change as store paths are added or - removed. +- `--references` - - `--referrers-closure`\ - Prints the closure of the set of store paths *paths* under the - referrers relation; that is, all store paths that directly or - indirectly refer to one of *paths*. These are all the path currently - in the Nix store that are dependent on *paths*. + Prints the set of [references] of the store paths + *paths*, that is, their immediate dependencies. (For *all* + dependencies, use `--requisites`.) - - `--deriver`; `-d`\ - Prints the [deriver] that was used to build the store paths *paths*. If - the path has no deriver (e.g., if it is a source file), or if the - deriver is not known (e.g., in the case of a binary-only - deployment), the string `unknown-deriver` is printed. - The returned deriver is not guaranteed to exist in the local store, for - example when *paths* were substituted from a binary cache. - Use `--valid-derivers` instead to obtain valid paths only. + [references]: @docroot@/glossary.md#gloss-reference - [deriver]: ../../glossary.md#gloss-deriver +- `--referrers` - - `--valid-derivers`\ - Prints a set of derivation files (`.drv`) which are supposed produce - said paths when realized. Might print nothing, for example for source paths - or paths subsituted from a binary cache. + Prints the set of *referrers* of the store paths *paths*, that is, + the store paths currently existing in the Nix store that refer to + one of *paths*. Note that contrary to the references, the set of + referrers is not constant; it can change as store paths are added or + removed. - - `--graph`\ - Prints the references graph of the store paths *paths* in the format - of the `dot` tool of AT\&T's [Graphviz - package](http://www.graphviz.org/). This can be used to visualise - dependency graphs. To obtain a build-time dependency graph, apply - this to a store derivation. To obtain a runtime dependency graph, - apply it to an output path. +- `--referrers-closure` - - `--tree`\ - Prints the references graph of the store paths *paths* as a nested - ASCII tree. References are ordered by descending closure size; this - tends to flatten the tree, making it more readable. The query only - recurses into a store path when it is first encountered; this - prevents a blowup of the tree representation of the graph. + Prints the closure of the set of store paths *paths* under the + referrers relation; that is, all store paths that directly or + indirectly refer to one of *paths*. These are all the path currently + in the Nix store that are dependent on *paths*. - - `--graphml`\ - Prints the references graph of the store paths *paths* in the - [GraphML](http://graphml.graphdrawing.org/) file format. This can be - used to visualise dependency graphs. To obtain a build-time - dependency graph, apply this to a [store derivation]. To obtain a - runtime dependency graph, apply it to an output path. +- `--deriver` / `-d` - - `--binding` *name*; `-b` *name*\ - Prints the value of the attribute *name* (i.e., environment - variable) of the [store derivation]s *paths*. It is an error for a - derivation to not have the specified attribute. + Prints the [deriver] that was used to build the store paths *paths*. If + the path has no deriver (e.g., if it is a source file), or if the + deriver is not known (e.g., in the case of a binary-only + deployment), the string `unknown-deriver` is printed. + The returned deriver is not guaranteed to exist in the local store, for + example when *paths* were substituted from a binary cache. + Use `--valid-derivers` instead to obtain valid paths only. - - `--hash`\ - Prints the SHA-256 hash of the contents of the store paths *paths* - (that is, the hash of the output of `nix-store --dump` on the given - paths). Since the hash is stored in the Nix database, this is a fast - operation. + [deriver]: @docroot@/glossary.md#gloss-deriver - - `--size`\ - Prints the size in bytes of the contents of the store paths *paths* - — to be precise, the size of the output of `nix-store --dump` on - the given paths. Note that the actual disk space required by the - store paths may be higher, especially on filesystems with large - cluster sizes. +- `--valid-derivers` - - `--roots`\ - Prints the garbage collector roots that point, directly or - indirectly, at the store paths *paths*. + Prints a set of derivation files (`.drv`) which are supposed produce + said paths when realized. Might print nothing, for example for source paths + or paths subsituted from a binary cache. + +- `--graph` + + Prints the references graph of the store paths *paths* in the format + of the `dot` tool of AT\&T's [Graphviz + package](http://www.graphviz.org/). This can be used to visualise + dependency graphs. To obtain a build-time dependency graph, apply + this to a store derivation. To obtain a runtime dependency graph, + apply it to an output path. + +- `--tree` + + Prints the references graph of the store paths *paths* as a nested + ASCII tree. References are ordered by descending closure size; this + tends to flatten the tree, making it more readable. The query only + recurses into a store path when it is first encountered; this + prevents a blowup of the tree representation of the graph. + +- `--graphml` + + Prints the references graph of the store paths *paths* in the + [GraphML](http://graphml.graphdrawing.org/) file format. This can be + used to visualise dependency graphs. To obtain a build-time + dependency graph, apply this to a [store derivation]. To obtain a + runtime dependency graph, apply it to an output path. + +- `--binding` *name* / `-b` *name* + + Prints the value of the attribute *name* (i.e., environment + variable) of the [store derivation]s *paths*. It is an error for a + derivation to not have the specified attribute. + +- `--hash` + + Prints the SHA-256 hash of the contents of the store paths *paths* + (that is, the hash of the output of `nix-store --dump` on the given + paths). Since the hash is stored in the Nix database, this is a fast + operation. + +- `--size` + + Prints the size in bytes of the contents of the store paths *paths* + — to be precise, the size of the output of `nix-store --dump` on + the given paths. Note that the actual disk space required by the + store paths may be higher, especially on filesystems with large + cluster sizes. + +- `--roots` + + Prints the garbage collector roots that point, directly or + indirectly, at the store paths *paths*. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index 5428d57fa..a899758df 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -25,14 +25,14 @@ Each of *paths* is processed as follows: If no substitutes are available and no store derivation is given, realisation fails. -[store paths]: @docroot@/glossary.md#gloss-store-path +[store paths]: @docroot@/store/store-path.md [valid]: @docroot@/glossary.md#gloss-validity [store derivation]: @docroot@/glossary.md#gloss-store-derivation [output paths]: @docroot@/glossary.md#gloss-output-path -[store objects]: @docroot@/glossary.md#gloss-store-object +[store objects]: @docroot@/store/store-object.md [closure]: @docroot@/glossary.md#gloss-closure [substituters]: @docroot@/command-ref/conf-file.md#conf-substituters -[content-addressed derivations]: @docroot@/contributing/experimental-features.md#xp-feature-ca-derivations +[content-addressed derivations]: @docroot@/development/experimental-features.md#xp-feature-ca-derivations [Nix database]: @docroot@/glossary.md#gloss-nix-database The resulting paths are printed on standard output. @@ -42,23 +42,26 @@ For non-derivation arguments, the argument itself is printed. # Options - - `--dry-run`\ - Print on standard error a description of what packages would be - built or downloaded, without actually performing the operation. +- `--dry-run` - - `--ignore-unknown`\ - If a non-derivation path does not have a substitute, then silently - ignore it. + Print on standard error a description of what packages would be + built or downloaded, without actually performing the operation. - - `--check`\ - This option allows you to check whether a derivation is - deterministic. It rebuilds the specified derivation and checks - whether the result is bitwise-identical with the existing outputs, - printing an error if that’s not the case. The outputs of the - specified derivation must already exist. When used with `-K`, if an - output path is not identical to the corresponding output from the - previous build, the new output path is left in - `/nix/store/name.check.` +- `--ignore-unknown` + + If a non-derivation path does not have a substitute, then silently + ignore it. + +- `--check` + + This option allows you to check whether a derivation is + deterministic. It rebuilds the specified derivation and checks + whether the result is bitwise-identical with the existing outputs, + printing an error if that’s not the case. The outputs of the + specified derivation must already exist. When used with `-K`, if an + output path is not identical to the corresponding output from the + previous build, the new output path is left in + `/nix/store/name.check.` {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/restore.md b/doc/manual/src/command-ref/nix-store/restore.md index fcba43df4..2d0aa3127 100644 --- a/doc/manual/src/command-ref/nix-store/restore.md +++ b/doc/manual/src/command-ref/nix-store/restore.md @@ -8,9 +8,11 @@ ## Description -The operation `--restore` unpacks a NAR archive to *path*, which must +The operation `--restore` unpacks a [Nix Archive (NAR)][Nix Archive] to *path*, which must not already exist. The archive is read from standard input. +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive + {{#include ./opt-common.md}} {{#include ../opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/serve.md b/doc/manual/src/command-ref/nix-store/serve.md index 0f90f65ae..9a4cf5216 100644 --- a/doc/manual/src/command-ref/nix-store/serve.md +++ b/doc/manual/src/command-ref/nix-store/serve.md @@ -14,10 +14,11 @@ access to a restricted ssh user. The following flags are available: - - `--write`\ - Allow the connected client to request the realization of - derivations. In effect, this can be used to make the host act as a - remote builder. +- `--write` + + Allow the connected client to request the realization of + derivations. In effect, this can be used to make the host act as a + remote builder. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/verify.md b/doc/manual/src/command-ref/nix-store/verify.md index 2695b3361..40c9180db 100644 --- a/doc/manual/src/command-ref/nix-store/verify.md +++ b/doc/manual/src/command-ref/nix-store/verify.md @@ -16,18 +16,20 @@ being modified by non-Nix tools, or of bugs in Nix itself. This operation has the following options: - - `--check-contents`\ - Checks that the contents of every valid store path has not been - altered by computing a SHA-256 hash of the contents and comparing it - with the hash stored in the Nix database at build time. Paths that - have been modified are printed out. For large stores, - `--check-contents` is obviously quite slow. +- `--check-contents` - - `--repair`\ - If any valid path is missing from the store, or (if - `--check-contents` is given) the contents of a valid path has been - modified, then try to repair the path by redownloading it. See - `nix-store --repair-path` for details. + Checks that the contents of every valid store path has not been + altered by computing a SHA-256 hash of the contents and comparing it + with the hash stored in the Nix database at build time. Paths that + have been modified are printed out. For large stores, + `--check-contents` is obviously quite slow. + +- `--repair` + + If any valid path is missing from the store, or (if + `--check-contents` is given) the contents of a valid path has been + modified, then try to repair the path by redownloading it. See + `nix-store --repair-path` for details. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index 114b292f9..69a700207 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -37,7 +37,7 @@ Most Nix commands accept the following command-line options: Print even more informational messages. - `4` “Debug†- + Print debug information. - `5` “Vomit†@@ -143,7 +143,7 @@ Most Nix commands accept the following command-line options: This option is accepted by `nix-env`, `nix-instantiate`, `nix-shell` and `nix-build`. When evaluating Nix expressions, the expression evaluator will automatically try to call functions that it encounters. - It can automatically call functions for which every argument has a [default value](@docroot@/language/constructs.md#functions) (e.g., `{ argName ? defaultValue }: ...`). + It can automatically call functions for which every argument has a [default value](@docroot@/language/syntax.md#functions) (e.g., `{ argName ? defaultValue }: ...`). With `--arg`, you can also call functions that have arguments without a default value (or override a default value). That is, if the evaluator encounters a function with an argument named *name*, it will call it with value *value*. @@ -187,11 +187,12 @@ Most Nix commands accept the following command-line options: For `nix-shell`, this option is commonly used to give you a shell in which you can build the packages returned by the expression. If you want to get a shell which contain the *built* packages ready for use, give your expression to the `nix-shell --packages ` convenience flag instead. -- [`-I`](#opt-I) *path* +- [`-I` / `--include`](#opt-I) *path* - Add an entry to the [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path). + Add an entry to the list of search paths used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). This option may be given multiple times. - Paths added through `-I` take precedence over [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH). + + Paths added through `-I` take precedence over the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path) and the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH). - [`--option`](#opt-option) *name* *value* diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/development/building.md similarity index 67% rename from doc/manual/src/contributing/hacking.md rename to doc/manual/src/development/building.md index 916ec3077..5a5fb3368 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/development/building.md @@ -1,24 +1,67 @@ -# Hacking +# Building Nix -This section provides some notes on how to hack on Nix. To get the -latest version of Nix from GitHub: +This section provides some notes on how to start hacking on Nix. +To get the latest version of Nix from GitHub: ```console $ git clone https://github.com/NixOS/nix.git $ cd nix ``` -The following instructions assume you already have some version of Nix installed locally, so that you can use it to set up the development environment. If you don't have it installed, follow the [installation instructions]. +> **Note** +> +> The following instructions assume you already have some version of Nix installed locally, so that you can use it to set up the development environment. +> If you don't have it installed, follow the [installation instructions](../installation/index.md). -[installation instructions]: ../installation/index.md + +To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: + +```console +$ nix-shell +``` + +To get a shell with one of the other [supported compilation environments](#compilation-environments): + +```console +$ nix-shell --attr devShells.x86_64-linux.native-clangStdenvPackages +``` + +> **Note** +> +> You can use `native-ccacheStdenvPackages` to drastically improve rebuild time. +> By default, [ccache](https://ccache.dev) keeps artifacts in `~/.cache/ccache/`. + +To build Nix itself in this shell: + +```console +[nix-shell]$ autoreconfPhase +[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out +[nix-shell]$ make -j $NIX_BUILD_CORES +``` + +To install it in `$(pwd)/outputs` and test it: + +```console +[nix-shell]$ make install +[nix-shell]$ make installcheck -j $NIX_BUILD_CORES +[nix-shell]$ ./outputs/out/bin/nix --version +nix (Nix) 2.12 +``` + +To build a release version of Nix for the current operating system and CPU architecture: + +```console +$ nix-build +``` + +You can also build Nix for one of the [supported platforms](#platforms). ## Building Nix with flakes This section assumes you are using Nix with the [`flakes`] and [`nix-command`] experimental features enabled. -See the [Building Nix](#building-nix) section for equivalent instructions using stable Nix interfaces. -[`flakes`]: @docroot@/contributing/experimental-features.md#xp-feature-flakes -[`nix-command`]: @docroot@/contributing/experimental-features.md#xp-nix-command +[`flakes`]: @docroot@/development/experimental-features.md#xp-feature-flakes +[`nix-command`]: @docroot@/development/experimental-features.md#xp-nix-command To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: @@ -67,50 +110,6 @@ $ nix build You can also build Nix for one of the [supported platforms](#platforms). -## Building Nix - -To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: - -```console -$ nix-shell -``` - -To get a shell with one of the other [supported compilation environments](#compilation-environments): - -```console -$ nix-shell --attr devShells.x86_64-linux.native-clangStdenvPackages -``` - -> **Note** -> -> You can use `native-ccacheStdenvPackages` to drastically improve rebuild time. -> By default, [ccache](https://ccache.dev) keeps artifacts in `~/.cache/ccache/`. - -To build Nix itself in this shell: - -```console -[nix-shell]$ autoreconfPhase -[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out -[nix-shell]$ make -j $NIX_BUILD_CORES -``` - -To install it in `$(pwd)/outputs` and test it: - -```console -[nix-shell]$ make install -[nix-shell]$ make installcheck -j $NIX_BUILD_CORES -[nix-shell]$ ./outputs/out/bin/nix --version -nix (Nix) 2.12 -``` - -To build a release version of Nix for the current operating system and CPU architecture: - -```console -$ nix-build -``` - -You can also build Nix for one of the [supported platforms](#platforms). - ## Makefile variables You may need `profiledir=$out/etc/profile.d` and `sysconfdir=$out/etc` to run `make install`. @@ -122,7 +121,6 @@ Run `make` with [`-e` / `--environment-overrides`](https://www.gnu.org/software/ The docs can take a while to build, so you may want to disable this for local development. - `ENABLE_FUNCTIONAL_TESTS=yes` to enable building the functional tests. -- `ENABLE_UNIT_TESTS=yes` to enable building the unit tests. - `OPTIMIZE=1` to enable optimizations. - `libraries=libutil programs=` to only build a specific library. @@ -144,6 +142,7 @@ Nix can be built for various platforms, as specified in [`flake.nix`]: - `aarch64-darwin` - `armv6l-linux` - `armv7l-linux` +- `riscv64-linux` In order to build Nix for a different platform than the one you're currently on, you need a way for your current Nix installation to build code for that @@ -166,7 +165,10 @@ or for Nix with the [`flakes`] and [`nix-command`] experimental features enabled $ nix build .#packages.aarch64-linux.default ``` -Cross-compiled builds are available for ARMv6 (`armv6l-linux`) and ARMv7 (`armv7l-linux`). +Cross-compiled builds are available for: +- `armv6l-linux` +- `armv7l-linux` +- `riscv64-linux` Add more [system types](#system-type) to `crossSystems` in `flake.nix` to bootstrap Nix on unsupported platforms. ### Building for multiple platforms at once @@ -196,7 +198,7 @@ In order to facilitate this, Nix has some support for being built out of tree ## System type -Nix uses a string with he following format to identify the *system type* or *platform* it runs on: +Nix uses a string with the following format to identify the *system type* or *platform* it runs on: ``` -[-] @@ -258,10 +260,10 @@ See [supported compilation environments](#compilation-environments) and instruct To use the LSP with your editor, you first need to [set up `clangd`](https://clangd.llvm.org/installation#project-setup) by running: ```console -make clean && bear -- make -j$NIX_BUILD_CORES default check install +make compile_commands.json ``` -Configure your editor to use the `clangd` from the shell, either by running it inside the development shell, or by using [nix-direnv](https://github.com/nix-community/nix-direnv) and [the appropriate editor plugin](https://github.com/direnv/direnv/wiki#editor-integration). +Configure your editor to use the `clangd` from the `.#native-clangStdenvPackages` shell. You can do that either by running it inside the development shell, or by using [nix-direnv](https://github.com/nix-community/nix-direnv) and [the appropriate editor plugin](https://github.com/direnv/direnv/wiki#editor-integration). > **Note** > @@ -269,80 +271,25 @@ Configure your editor to use the `clangd` from the shell, either by running it i > Some other editors (e.g. Emacs, Vim) need a plugin to support LSP servers in general (e.g. [lsp-mode](https://github.com/emacs-lsp/lsp-mode) for Emacs and [vim-lsp](https://github.com/prabirshrestha/vim-lsp) for vim). > Editor-specific setup is typically opinionated, so we will not cover it here in more detail. -## Add a release note +## Formatting and pre-commit hooks -`doc/manual/rl-next` contains release notes entries for all unreleased changes. +You may run the formatters as a one-off using: -User-visible changes should come with a release note. - -### Add an entry - -Here's what a complete entry looks like. The file name is not incorporated in the document. - -``` ---- -synopsis: Basically a title -issues: 1234 -prs: 1238 ---- - -Here's one or more paragraphs that describe the change. - -- It's markdown -- Add references to the manual using @docroot@ +```console +make format ``` -Significant changes should add the following header, which moves them to the top. +If you'd like to run the formatters before every commit, install the hooks: ``` -significance: significant +pre-commit-hooks-install ``` - -See also the [format documentation](https://github.com/haskell/cabal/blob/master/CONTRIBUTING.md#changelog). +This installs [pre-commit](https://pre-commit.com) using [cachix/git-hooks.nix](https://github.com/cachix/git-hooks.nix). -### Build process +When making a commit, pay attention to the console output. +If it fails, run `git add --patch` to approve the suggestions _and commit again_. -Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`. - -## Branches - -- [`master`](https://github.com/NixOS/nix/commits/master) - - The main development branch. All changes are approved and merged here. - When developing a change, create a branch based on the latest `master`. - - Maintainers try to [keep it in a release-worthy state](#reverting). - -- [`maintenance-*.*`](https://github.com/NixOS/nix/branches/all?query=maintenance) - - These branches are the subject of backports only, and are - also [kept](#reverting) in a release-worthy state. - - See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) - -- [`latest-release`](https://github.com/NixOS/nix/tree/latest-release) - - The latest patch release of the latest minor version. - - See [`maintainers/release-process.md`](https://github.com/NixOS/nix/blob/master/maintainers/release-process.md) - -- [`backport-*-to-*`](https://github.com/NixOS/nix/branches/all?query=backport) - - Generally branches created by the backport action. - - See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) - -- [_other_](https://github.com/NixOS/nix/branches/all) - - Branches that do not conform to the above patterns should be feature branches. - -## Reverting - -If a change turns out to be merged by mistake, or contain a regression, it may be reverted. -A revert is not a rejection of the contribution, but merely part of an effective development process. -It makes sure that development keeps running smoothly, with minimal uncertainty, and less overhead. -If maintainers have to worry too much about avoiding reverts, they would not be able to merge as much. -By embracing reverts as a good part of the development process, everyone wins. - -However, taking a step back may be frustrating, so maintainers will be extra supportive on the next try. +To refresh pre-commit hook's config file, do the following: +1. Exit the development shell and start it again by running `nix develop`. +2. If you also use the pre-commit hook, also run `pre-commit-hooks-install` again. diff --git a/doc/manual/src/contributing/cli-guideline.md b/doc/manual/src/development/cli-guideline.md similarity index 88% rename from doc/manual/src/contributing/cli-guideline.md rename to doc/manual/src/development/cli-guideline.md index e90d6de8d..23df844ec 100644 --- a/doc/manual/src/contributing/cli-guideline.md +++ b/doc/manual/src/development/cli-guideline.md @@ -389,88 +389,6 @@ colors, no emojis and using ASCII instead of Unicode symbols). The same should happen when TTY is not detected on STDERR. We should not display progress / status section, but only print warnings and errors. -## Returning future proof JSON - -The schema of JSON output should allow for backwards compatible extension. This section explains how to achieve this. - -Two definitions are helpful here, because while JSON only defines one "key-value" -object type, we use it to cover two use cases: - - - **dictionary**: a map from names to value that all have the same type. In - C++ this would be a `std::map` with string keys. - - **record**: a fixed set of attributes each with their own type. In C++, this - would be represented by a `struct`. - -It is best not to mix these use cases, as that may lead to incompatibilities when the schema changes. For example, adding a record field to a dictionary breaks consumers that assume all JSON object fields to have the same meaning and type. - -This leads to the following guidelines: - - - The top-level (root) value must be a record. - - Otherwise, one can not change the structure of a command's output. - - - The value of a dictionary item must be a record. - - Otherwise, the item type can not be extended. - - - List items should be records. - - Otherwise, one can not change the structure of the list items. - - If the order of the items does not matter, and each item has a unique key that is a string, consider representing the list as a dictionary instead. If the order of the items needs to be preserved, return a list of records. - - - Streaming JSON should return records. - - An example of a streaming JSON format is [JSON lines](https://jsonlines.org/), where each line represents a JSON value. These JSON values can be considered top-level values or list items, and they must be records. - -### Examples - - -This is bad, because all keys must be assumed to be store types: - -```json -{ - "local": { ... }, - "remote": { ... }, - "http": { ... } -} -``` - -This is good, because the it is extensible at the root, and is somewhat self-documenting: - -```json -{ - "storeTypes": { "local": { ... }, ... }, - "pluginSupport": true -} -``` - -While the dictionary of store types seems like a very complete response at first, a use case may arise that warrants returning additional information. -For example, the presence of plugin support may be crucial information for a client to proceed when their desired store type is missing. - - - -The following representation is bad because it is not extensible: - -```json -{ "outputs": [ "out" "bin" ] } -``` - -However, simply converting everything to records is not enough, because the order of outputs must be preserved: - -```json -{ "outputs": { "bin": {}, "out": {} } } -``` - -The first item is the default output. Deriving this information from the outputs ordering is not great, but this is how Nix currently happens to work. -While it is possible for a JSON parser to preserve the order of fields, we can not rely on this capability to be present in all JSON libraries. - -This representation is extensible and preserves the ordering: - -```json -{ "outputs": [ { "outputName": "out" }, { "outputName": "bin" } ] } -``` - ## Dialog with the user CLIs don't always make it clear when an action has taken place. For every diff --git a/doc/manual/src/development/contributing.md b/doc/manual/src/development/contributing.md new file mode 100644 index 000000000..7de7489dc --- /dev/null +++ b/doc/manual/src/development/contributing.md @@ -0,0 +1,79 @@ +# Contributing + +## Add a release note + +`doc/manual/rl-next` contains release notes entries for all unreleased changes. + +User-visible changes should come with a release note. + +### Add an entry + +Here's what a complete entry looks like. The file name is not incorporated in the document. + +``` +--- +synopsis: Basically a title +issues: 1234 +prs: 1238 +--- + +Here's one or more paragraphs that describe the change. + +- It's markdown +- Add references to the manual using @docroot@ +``` + +Significant changes should add the following header, which moves them to the top. + +``` +significance: significant +``` + + +See also the [format documentation](https://github.com/haskell/cabal/blob/master/CONTRIBUTING.md#changelog). + +### Build process + +Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`. + +## Branches + +- [`master`](https://github.com/NixOS/nix/commits/master) + + The main development branch. All changes are approved and merged here. + When developing a change, create a branch based on the latest `master`. + + Maintainers try to [keep it in a release-worthy state](#reverting). + +- [`maintenance-*.*`](https://github.com/NixOS/nix/branches/all?query=maintenance) + + These branches are the subject of backports only, and are + also [kept](#reverting) in a release-worthy state. + + See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) + +- [`latest-release`](https://github.com/NixOS/nix/tree/latest-release) + + The latest patch release of the latest minor version. + + See [`maintainers/release-process.md`](https://github.com/NixOS/nix/blob/master/maintainers/release-process.md) + +- [`backport-*-to-*`](https://github.com/NixOS/nix/branches/all?query=backport) + + Generally branches created by the backport action. + + See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) + +- [_other_](https://github.com/NixOS/nix/branches/all) + + Branches that do not conform to the above patterns should be feature branches. + +## Reverting + +If a change turns out to be merged by mistake, or contain a regression, it may be reverted. +A revert is not a rejection of the contribution, but merely part of an effective development process. +It makes sure that development keeps running smoothly, with minimal uncertainty, and less overhead. +If maintainers have to worry too much about avoiding reverts, they would not be able to merge as much. +By embracing reverts as a good part of the development process, everyone wins. + +However, taking a step back may be frustrating, so maintainers will be extra supportive on the next try. diff --git a/doc/manual/src/contributing/cxx.md b/doc/manual/src/development/cxx.md similarity index 100% rename from doc/manual/src/contributing/cxx.md rename to doc/manual/src/development/cxx.md diff --git a/doc/manual/src/contributing/documentation.md b/doc/manual/src/development/documentation.md similarity index 89% rename from doc/manual/src/contributing/documentation.md rename to doc/manual/src/development/documentation.md index 1dddb207c..63f574ab7 100644 --- a/doc/manual/src/contributing/documentation.md +++ b/doc/manual/src/development/documentation.md @@ -24,14 +24,12 @@ nix build .#^doc and open `./result-doc/share/doc/nix/manual/index.html`. -To build the manual incrementally, [enter the development shell](./hacking.md) and run: +To build the manual incrementally, [enter the development shell](./building.md) and run: ```console -make manual-html -j $NIX_BUILD_CORES +make manual-html-open -j $NIX_BUILD_CORES ``` -and open `./outputs/out/share/doc/nix/manual/language/index.html`. - In order to reflect changes to the [Makefile for the manual], clear all generated files before re-building: [Makefile for the manual]: https://github.com/NixOS/nix/blob/master/doc/manual/local.mk @@ -149,7 +147,7 @@ Please observe these guidelines to ease reviews: ``` A [store object] contains a [file system object] and [references] to other store objects. - [store object]: @docroot@/glossary.md#gloss-store-object + [store object]: @docroot@/store/store-object.md [file system object]: @docroot@/architecture/file-system-object.md [references]: @docroot@/glossary.md#gloss-reference ``` @@ -198,13 +196,35 @@ You can also build and view it yourself: [Doxygen API documentation]: https://hydra.nixos.org/job/nix/master/internal-api-docs/latest/download-by-type/doc/internal-api-docs ```console -# nix build .#hydraJobs.internal-api-docs -# xdg-open ./result/share/doc/nix/internal-api/html/index.html +$ nix build .#hydraJobs.internal-api-docs +$ xdg-open ./result/share/doc/nix/internal-api/html/index.html +``` + +or inside `nix-shell` or `nix develop`: + +```console +$ mesonConfigurePhase +$ ninja src/internal-api-docs/html +$ xdg-open src/internal-api-docs/html/index.html +``` + +## C API documentation + +Note that the C API is not yet stable. +[C API documentation] is available online. +You can also build and view it yourself: + +[C API documentation]: https://hydra.nixos.org/job/nix/master/external-api-docs/latest/download-by-type/doc/external-api-docs + +```console +$ nix build .#hydraJobs.external-api-docs +$ xdg-open ./result/share/doc/nix/external-api/html/index.html ``` or inside `nix-shell` or `nix develop`: ``` -# make internal-api-html -# xdg-open ./outputs/doc/share/doc/nix/internal-api/html/index.html +$ mesonConfigurePhase +$ ninja src/external-api-docs/html +$ xdg-open src/external-api-docs/html/index.html ``` diff --git a/doc/manual/src/contributing/experimental-features.md b/doc/manual/src/development/experimental-features.md similarity index 100% rename from doc/manual/src/contributing/experimental-features.md rename to doc/manual/src/development/experimental-features.md diff --git a/doc/manual/src/contributing/index.md b/doc/manual/src/development/index.md similarity index 77% rename from doc/manual/src/contributing/index.md rename to doc/manual/src/development/index.md index 4d55c17a4..6403c3e66 100644 --- a/doc/manual/src/contributing/index.md +++ b/doc/manual/src/development/index.md @@ -5,4 +5,4 @@ Check the [contributing guide](https://github.com/NixOS/nix/blob/master/CONTRIBU This chapter is a collection of guides for making changes to the code and documentation. -If you're not sure where to start, try to [compile Nix from source](./hacking.md) and consider [making improvements to documentation](./documentation.md). +If you're not sure where to start, try to [compile Nix from source](./building.md) and consider [making improvements to documentation](./documentation.md). diff --git a/doc/manual/src/development/json-guideline.md b/doc/manual/src/development/json-guideline.md new file mode 100644 index 000000000..b4bc92af9 --- /dev/null +++ b/doc/manual/src/development/json-guideline.md @@ -0,0 +1,128 @@ +# JSON guideline + +Nix consumes and produces JSON in a variety of contexts. +These guidelines ensure consistent practices for all our JSON interfaces, for ease of use, and so that experience in one part carries over to another. + +## Extensibility + +The schema of JSON input and output should allow for backwards compatible extension. +This section explains how to achieve this. + +Two definitions are helpful here, because while JSON only defines one "key-value" object type, we use it to cover two use cases: + + - **dictionary**: a map from names to value that all have the same type. + In C++ this would be a `std::map` with string keys. + + - **record**: a fixed set of attributes each with their own type. + In C++, this would be represented by a `struct`. + +It is best not to mix these use cases, as that may lead to incompatibilities when the schema changes. +For example, adding a record field to a dictionary breaks consumers that assume all JSON object fields to have the same meaning and type, and dictionary items with a colliding name can not be represented anymore. + +This leads to the following guidelines: + + - The top-level (root) value must be a record. + + Otherwise, one can not change the structure of a command's output. + + - The value of a dictionary item must be a record. + + Otherwise, the item type can not be extended. + + - List items should be records. + + Otherwise, one can not change the structure of the list items. + + If the order of the items does not matter, and each item has a unique key that is a string, consider representing the list as a dictionary instead. + If the order of the items needs to be preserved, return a list of records. + + - Streaming JSON should return records. + + An example of a streaming JSON format is [JSON lines](https://jsonlines.org/), where each line represents a JSON value. + These JSON values can be considered top-level values or list items, and they must be records. + +### Examples + +This is bad, because all keys must be assumed to be store types: + +```json +{ + "local": { ... }, + "remote": { ... }, + "http": { ... } +} +``` + +This is good, because the it is extensible at the root, and is somewhat self-documenting: + +```json +{ + "storeTypes": { "local": { ... }, ... }, + "pluginSupport": true +} +``` + +While the dictionary of store types seems like a very complete response at first, a use case may arise that warrants returning additional information. +For example, the presence of plugin support may be crucial information for a client to proceed when their desired store type is missing. + + + +The following representation is bad because it is not extensible: + +```json +{ "outputs": [ "out" "bin" ] } +``` + +However, simply converting everything to records is not enough, because the order of outputs must be preserved: + +```json +{ "outputs": { "bin": {}, "out": {} } } +``` + +The first item is the default output. Deriving this information from the outputs ordering is not great, but this is how Nix currently happens to work. +While it is possible for a JSON parser to preserve the order of fields, we can not rely on this capability to be present in all JSON libraries. + +This representation is extensible and preserves the ordering: + +```json +{ "outputs": [ { "outputName": "out" }, { "outputName": "bin" } ] } +``` + +## Self-describing values + +As described in the previous section, it's crucial that schemas can be extended with with new fields without breaking compatibility. +However, that should *not* mean we use the presence/absence of fields to indicate optional information *within* a version of the schema. +Instead, always include the field, and use `null` to indicate the "nothing" case. + +### Examples + +Here are two JSON objects: + +```json +{ + "foo": {} +} +``` +```json +{ + "foo": {}, + "bar": {} +} +``` + +Since they differ in which fields they contain, they should *not* both be valid values of the same schema. +At most, they can match two different schemas where the second (with `foo` and `bar`) is considered a newer version of the first (with just `foo`). +Within each version, all fields are mandatory (always `foo`, and always `foo` and `bar`). +Only *between* each version, `bar` gets added as a new mandatory field. + +Here are another two JSON objects: + +```json +{ "foo": null } +``` +```json +{ "foo": { "bar": 1 } } +``` + +Since they both contain a `foo` field, they could be valid values of the same schema. +The schema would have `foo` has an optional field, which is either `null` or an object where `bar` is an integer. diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/development/testing.md similarity index 81% rename from doc/manual/src/contributing/testing.md rename to doc/manual/src/development/testing.md index 31c39c16c..3949164d5 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/development/testing.md @@ -59,15 +59,15 @@ The unit tests are defined using the [googletest] and [rapidcheck] frameworks. > … > ``` -The tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `tests/unit/${library_name_without-nix}`. -Given a interface (header) and implementation pair in the original library, say, `src/libexpr/value/context.{hh,cc}`, we write tests for it in `tests/unit/libexpr/tests/value/context.cc`, and (possibly) declare/define additional interfaces for testing purposes in `tests/unit/libexpr-support/tests/value/context.{hh,cc}`. +The tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `src/${library_name_without-nix}-test`. +Given an interface (header) and implementation pair in the original library, say, `src/libexpr/value/context.{hh,cc}`, we write tests for it in `src/nix-expr-tests/value/context.cc`, and (possibly) declare/define additional interfaces for testing purposes in `src/nix-expr-test-support/tests/value/context.{hh,cc}`. Data for unit tests is stored in a `data` subdir of the directory for each unit test executable. -For example, `libnixstore` code is in `src/libstore`, and its test data is in `tests/unit/libstore/data`. -The path to the `tests/unit/data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`. +For example, `libnixstore` code is in `src/libstore`, and its test data is in `src/nix-store-tests/data`. +The path to the `src/${library_name_without-nix}-test/data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`. Note that each executable only gets the data for its tests. -The unit test libraries are in `tests/unit/${library_name_without-nix}-lib`. +The unit test libraries are in `src/${library_name_without-nix}-test-support`. All headers are in a `tests` subdirectory so they are included with `#include "tests/"`. The use of all these separate directories for the unit tests might seem inconvenient, as for example the tests are not "right next to" the part of the code they are testing. @@ -76,8 +76,25 @@ there is no risk of any build-system wildcards for the library accidentally pick ### Running tests -You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. -Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option, or the `GTEST_FILTER` environment variable, e.g. `GTEST_FILTER='ErrorTraceTest.*' make check`. +You can run the whole testsuite with `meson test` from the Meson build directory, or the tests for a specific component with `meson test nix-store-tests`. +A environment variables that Google Test accepts are also worth knowing: + +1. [`GTEST_FILTER`](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) + + This is used for finer-grained filtering of which tests to run. + + +2. [`GTEST_BRIEF`](https://google.github.io/googletest/advanced.html#suppressing-test-passes) + + This is used to avoid logging passing tests. + +Putting the two together, one might run + +```bash +GTEST_BRIEF=1 GTEST_FILTER='ErrorTraceTest.*' meson test nix-expr-tests -v +``` + +for short but comprensive output. ### Characterisation testing { #characaterisation-testing-unit } @@ -86,7 +103,7 @@ See [functional characterisation testing](#characterisation-testing-functional) Like with the functional characterisation, `_NIX_TEST_ACCEPT=1` is also used. For example: ```shell-session -$ _NIX_TEST_ACCEPT=1 make libstore-tests_RUN +$ _NIX_TEST_ACCEPT=1 meson test nix-store-tests -v ... [ SKIPPED ] WorkerProtoTest.string_read [ SKIPPED ] WorkerProtoTest.string_write @@ -114,6 +131,8 @@ On other platforms they wouldn't be run at all. The functional tests reside under the `tests/functional` directory and are listed in `tests/functional/local.mk`. Each test is a bash script. +Functional tests are run during `installCheck` in the `nix` package build, as well as separately from the build, in VM tests. + ### Running the whole test suite The whole test suite can be run with: @@ -162,14 +181,14 @@ ran test tests/functional/${testName}.sh... [PASS] or without `make`: ```shell-session -$ ./mk/run-test.sh tests/functional/${testName}.sh tests/functional/init.sh +$ ./mk/run-test.sh tests/functional/${testName}.sh ran test tests/functional/${testName}.sh... [PASS] ``` To see the complete output, one can also run: ```shell-session -$ ./mk/debug-test.sh tests/functional/${testName}.sh tests/functional/init.sh +$ ./mk/debug-test.sh tests/functional/${testName}.sh +(${testName}.sh:1) foo output from foo +(${testName}.sh:2) bar @@ -204,7 +223,7 @@ edit it like so: Then, running the test with `./mk/debug-test.sh` will drop you into GDB once the script reaches that point: ```shell-session -$ ./mk/debug-test.sh tests/functional/${testName}.sh tests/functional/init.sh +$ ./mk/debug-test.sh tests/functional/${testName}.sh ... + gdb blash blub GNU gdb (GDB) 12.1 @@ -252,13 +271,30 @@ Regressions are caught, and improvements always show up in code review. To ensure that characterisation testing doesn't make it harder to intentionally change these interfaces, there always must be an easy way to regenerate the expected output, as we do with `_NIX_TEST_ACCEPT=1`. +### Running functional tests on NixOS + +We run the functional tests not just in the build, but also in VM tests. +This helps us ensure that Nix works correctly on NixOS, and environments that have similar characteristics that are hard to reproduce in a build environment. + +The recommended way to run these tests during development is: + +```shell +nix build .#hydraJobs.tests.functional_user.quickBuild +``` + +The `quickBuild` attribute configures the test to use a `nix` package that's built without integration tests, so that you can iterate on the tests without performing recompilations due to the changed sources for `installCheck`. + +Generally, this build is sufficient, but in nightly or CI we also test the attributes `functional_root` and `functional_trusted`, in which the test suite is run with different levels of authorization. + ## Integration tests The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute. These tests include everything that needs to interact with external services or run Nix in a non-trivial distributed setup. Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on ). -You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}` +You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}`. + +If you are testing a build of `nix` that you haven't compiled yet, you may iterate faster by appending the `quickBuild` attribute: `nix build .#hydraJobs.tests.{testName}.quickBuild`. ## Installer tests diff --git a/doc/manual/src/favicon.png b/doc/manual/src/favicon.png new file mode 100644 index 000000000..1ed2b5fe0 Binary files /dev/null and b/doc/manual/src/favicon.png differ diff --git a/doc/manual/src/favicon.svg b/doc/manual/src/favicon.svg new file mode 100644 index 000000000..1d2a6e835 --- /dev/null +++ b/doc/manual/src/favicon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index a71b2e2b3..877c4668b 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -1,5 +1,24 @@ # Glossary +- [content address]{#gloss-content-address} + + A + [*content address*](https://en.wikipedia.org/wiki/Content-addressable_storage) + is a secure way to reference immutable data. + The reference is calculated directly from the content of the data being referenced, which means the reference is + [*tamper proof*](https://en.wikipedia.org/wiki/Tamperproofing) + --- variations of the data should always calculate to distinct content addresses. + + For how Nix uses content addresses, see: + + - [Content-Addressing File System Objects](@docroot@/store/file-system-object/content-address.md) + - [Content-Addressing Store Objects](@docroot@/store/store-object/content-address.md) + - [content-addressed derivation](#gloss-content-addressed-derivation) + + Software Heritage's writing on [*Intrinsic and Extrinsic identifiers*](https://www.softwareheritage.org/2020/07/09/intrinsic-vs-extrinsic-identifiers) is also a good introduction to the value of content-addressing over other referencing schemes. + + Besides content addressing, the Nix store also uses [input addressing](#gloss-input-addressed-store-object). + - [derivation]{#gloss-derivation} A description of a build task. The result of a derivation is a @@ -52,10 +71,9 @@ [`__contentAddressed`](./language/advanced-attributes.md#adv-attr-__contentAddressed) attribute set to `true`. -- [fixed-output derivation]{#gloss-fixed-output-derivation} +- [fixed-output derivation]{#gloss-fixed-output-derivation} (FOD) - A derivation which includes the - [`outputHash`](./language/advanced-attributes.md#adv-attr-outputHash) attribute. + A [derivation] where a cryptographic hash of the [output] is determined in advance using the [`outputHash`](./language/advanced-attributes.md#adv-attr-outputHash) attribute, and where the [`builder`](@docroot@/language/derivations.md#attr-builder) executable has access to the network. - [store]{#gloss-store} @@ -86,7 +104,7 @@ [store path]: #gloss-store-path -- [file system object]{#gloss-store-object} +- [file system object]{#gloss-file-system-object} The Nix data model for representing simplified file system data. @@ -118,9 +136,12 @@ - [content-addressed store object]{#gloss-content-addressed-store-object} - A [store object] whose [store path] is determined by its contents. + A [store object] which is [content-addressed](#gloss-content-address), + i.e. whose [store path] is determined by its contents. This includes derivations, the outputs of [content-addressed derivations](#gloss-content-addressed-derivation), and the outputs of [fixed-output derivations](#gloss-fixed-output-derivation). + See [Content-Addressing Store Objects](@docroot@/store/store-object/content-address.md) for details. + - [substitute]{#gloss-substitute} A substitute is a command invocation stored in the [Nix database] that @@ -147,7 +168,7 @@ - [impure derivation]{#gloss-impure-derivation} - [An experimental feature](#@docroot@/contributing/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, + [An experimental feature](#@docroot@/development/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, so that they are always rebuilt, and their outputs not reused by subsequent calls to realise them. - [Nix database]{#gloss-nix-database} @@ -215,6 +236,20 @@ [output path]: #gloss-output-path +- [output closure]{#gloss-output-closure}\ + The [closure] of an [output path]. It only contains what is [reachable] from the output. + +- [deriving path]{#gloss-deriving-path} + + Deriving paths are a way to refer to [store objects][store object] that ar not yet [realised][realise]. + This is necessary because, in general and particularly for [content-addressed derivations][content-addressed derivation], the [output path] of an [output] is not known in advance. + There are two forms: + + - *constant*: just a [store path] + It can be made [valid][validity] by copying it into the store: from the evaluator, command line interface or another store. + + - *output*: a pair of a [store path] to a [derivation] and an [output] name. + - [deriver]{#gloss-deriver} The [store derivation] that produced an [output path]. @@ -252,13 +287,15 @@ See [installables](./command-ref/new-cli/nix.md#installables) for [`nix` commands](./command-ref/new-cli/nix.md) (experimental) for details. -- [NAR]{#gloss-nar} +- [Nix Archive (NAR)]{#gloss-nar} A *N*ix *AR*chive. This is a serialisation of a path in the Nix store. It can contain regular files, directories and symbolic links. NARs are generated and unpacked using `nix-store --dump` and `nix-store --restore`. + See [Nix Archive](store/file-system-object/content-address.html#serial-nix-archive) for details. + - [`∅`]{#gloss-emtpy-set} The empty set symbol. In the context of profile history, this denotes a package is not present in a particular version of the profile. @@ -275,7 +312,7 @@ - [package attribute set]{#package-attribute-set} - An [attribute set](@docroot@/language/values.md#attribute-set) containing the attribute `type = "derivation";` (derivation for historical reasons), as well as other attributes, such as + An [attribute set](@docroot@/language/types.md#attribute-set) containing the attribute `type = "derivation";` (derivation for historical reasons), as well as other attributes, such as - attributes that refer to the files of a [package], typically in the form of [derivation outputs](#output), - attributes that declare something about how the package is supposed to be installed or used, - other metadata or arbitrary attributes. @@ -288,16 +325,35 @@ See [String interpolation](./language/string-interpolation.md) for details. - [string]: ./language/values.md#type-string - [path]: ./language/values.md#type-path - [attribute name]: ./language/values.md#attribute-set + [string]: ./language/types.md#type-string + [path]: ./language/types.md#type-path + [attribute name]: ./language/types.md#attribute-set + +- [base directory]{#gloss-base-directory} + + The location from which relative paths are resolved. + + - For expressions in a file, the base directory is the directory containing that file. + This is analogous to the directory of a [base URL](https://datatracker.ietf.org/doc/html/rfc1808#section-3.3). + + + + - For expressions written in command line arguments with [`--expr`](@docroot@/command-ref/opt-common.html#opt-expr), the base directory is the current working directory. + + [base directory]: #gloss-base-directory - [experimental feature]{#gloss-experimental-feature} Not yet stabilized functionality guarded by named experimental feature flags. These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting. - See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/contributing/experimental-features.md). + See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/development/experimental-features.md). [Nix language]: ./language/index.md diff --git a/doc/manual/src/installation/env-variables.md b/doc/manual/src/installation/env-variables.md index db98f52ff..035090421 100644 --- a/doc/manual/src/installation/env-variables.md +++ b/doc/manual/src/installation/env-variables.md @@ -53,7 +53,8 @@ ssl-cert-file = /etc/ssl/my-certificate-bundle.crt The Nix installer has special handling for these proxy-related environment variables: `http_proxy`, `https_proxy`, `ftp_proxy`, -`no_proxy`, `HTTP_PROXY`, `HTTPS_PROXY`, `FTP_PROXY`, `NO_PROXY`. +`all_proxy`, `no_proxy`, `HTTP_PROXY`, `HTTPS_PROXY`, `FTP_PROXY`, +`ALL_PROXY`, `NO_PROXY`. If any of these variables are set when running the Nix installer, then the installer will create an override file at diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index 0dc989159..6a168ff3d 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -50,7 +50,7 @@ Supported systems: To explicitly instruct the installer to perform a multi-user installation on your system: ```console -$ curl -L https://nixos.org/nix/install | sh -s -- --daemon +$ bash <(curl -L https://nixos.org/nix/install) --daemon ``` You can run this under your usual user account or `root`. @@ -61,7 +61,7 @@ The script will invoke `sudo` as needed. To explicitly select a single-user installation on your system: ```console -$ curl -L https://nixos.org/nix/install | sh -s -- --no-daemon +$ bash <(curl -L https://nixos.org/nix/install) --no-daemon ``` In a single-user installation, `/nix` is owned by the invoking user. @@ -77,7 +77,7 @@ $ su root # Installing from a binary tarball You can also download a binary tarball that contains Nix and all its dependencies: -- Choose a [version](https://releases.nixos.org/?prefix=nix/) and [system type](../contributing/hacking.md#platforms) +- Choose a [version](https://releases.nixos.org/?prefix=nix/) and [system type](../development/building.md#platforms) - Download and unpack the tarball - Run the installer diff --git a/doc/manual/src/installation/uninstall.md b/doc/manual/src/installation/uninstall.md index 9ead5e53c..590327fea 100644 --- a/doc/manual/src/installation/uninstall.md +++ b/doc/manual/src/installation/uninstall.md @@ -1,16 +1,8 @@ # Uninstalling Nix -## Single User - -If you have a [single-user installation](./installing-binary.md#single-user-installation) of Nix, uninstall it by running: - -```console -$ rm -rf /nix -``` - ## Multi User -Removing a [multi-user installation](./installing-binary.md#multi-user-installation) of Nix is more involved, and depends on the operating system. +Removing a [multi-user installation](./installing-binary.md#multi-user-installation) depends on the operating system. ### Linux @@ -51,7 +43,15 @@ which you may remove. ### macOS -1. Edit `/etc/zshrc`, `/etc/bashrc`, and `/etc/bash.bashrc` to remove the lines sourcing `nix-daemon.sh`, which should look like this: +1. If system-wide shell initialisation files haven't been altered since installing Nix, use the backups made by the installer: + + ```console + sudo mv /etc/zshrc.backup-before-nix /etc/zshrc + sudo mv /etc/bashrc.backup-before-nix /etc/bashrc + sudo mv /etc/bash.bashrc.backup-before-nix /etc/bash.bashrc + ``` + + Otherwise, edit `/etc/zshrc`, `/etc/bashrc`, and `/etc/bash.bashrc` to remove the lines sourcing `nix-daemon.sh`, which should look like this: ```bash # Nix @@ -61,18 +61,6 @@ which you may remove. # End Nix ``` - If these files haven't been altered since installing Nix you can simply put - the backups back in place: - - ```console - sudo mv /etc/zshrc.backup-before-nix /etc/zshrc - sudo mv /etc/bashrc.backup-before-nix /etc/bashrc - sudo mv /etc/bash.bashrc.backup-before-nix /etc/bash.bashrc - ``` - - This will stop shells from sourcing the file and bringing everything you - installed using Nix in scope. - 2. Stop and remove the Nix daemon services: ```console @@ -82,8 +70,7 @@ which you may remove. sudo rm /Library/LaunchDaemons/org.nixos.darwin-store.plist ``` - This stops the Nix daemon and prevents it from being started next time you - boot the system. + This stops the Nix daemon and prevents it from being started next time you boot the system. 3. Remove the `nixbld` group and the `_nixbuildN` users: @@ -94,25 +81,42 @@ which you may remove. This will remove all the build users that no longer serve a purpose. -4. Edit fstab using `sudo vifs` to remove the line mounting the Nix Store - volume on `/nix`, which looks like - `UUID= /nix apfs rw,noauto,nobrowse,suid,owners` or - `LABEL=Nix\040Store /nix apfs rw,nobrowse`. This will prevent automatic - mounting of the Nix Store volume. +4. Edit fstab using `sudo vifs` to remove the line mounting the Nix Store volume on `/nix`, which looks like -5. Edit `/etc/synthetic.conf` to remove the `nix` line. If this is the only - line in the file you can remove it entirely, `sudo rm /etc/synthetic.conf`. - This will prevent the creation of the empty `/nix` directory to provide a - mountpoint for the Nix Store volume. + ``` + UUID= /nix apfs rw,noauto,nobrowse,suid,owners + ``` + or -6. Remove the files Nix added to your system: + ``` + LABEL=Nix\040Store /nix apfs rw,nobrowse + ``` + + by setting the cursor on the respective line using the error keys, and pressing `dd`, and then `:wq` to save the file. + + This will prevent automatic mounting of the Nix Store volume. + +5. Edit `/etc/synthetic.conf` to remove the `nix` line. + If this is the only line in the file you can remove it entirely: + + ```bash + if [ -f /etc/synthetic.conf ]; then + if [ "$(cat /etc/synthetic.conf)" = "nix" ]; then + sudo rm /etc/synthetic.conf + else + sudo vi /etc/synthetic.conf + fi + fi + ``` + + This will prevent the creation of the empty `/nix` directory. + +6. Remove the files Nix added to your system, except for the store: ```console sudo rm -rf /etc/nix /var/root/.nix-profile /var/root/.nix-defexpr /var/root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels ``` - This gets rid of any data Nix may have created except for the store which is - removed next. 7. Remove the Nix Store volume: @@ -120,29 +124,32 @@ which you may remove. sudo diskutil apfs deleteVolume /nix ``` - This will remove the Nix Store volume and everything that was added to the - store. + This will remove the Nix Store volume and everything that was added to the store. - If the output indicates that the command couldn't remove the volume, you should - make sure you don't have an _unmounted_ Nix Store volume. Look for a - "Nix Store" volume in the output of the following command: + If the output indicates that the command couldn't remove the volume, you should make sure you don't have an _unmounted_ Nix Store volume. + Look for a "Nix Store" volume in the output of the following command: ```console diskutil list ``` - If you _do_ see a "Nix Store" volume, delete it by re-running the diskutil - deleteVolume command, but replace `/nix` with the store volume's `diskXsY` - identifier. + If you _do_ find a "Nix Store" volume, delete it by running `diskutil deleteVolume` with the store volume's `diskXsY` identifier. > **Note** > -> After you complete the steps here, you will still have an empty `/nix` -> directory. This is an expected sign of a successful uninstall. The empty -> `/nix` directory will disappear the next time you reboot. +> After you complete the steps here, you will still have an empty `/nix` directory. +> This is an expected sign of a successful uninstall. +> The empty `/nix` directory will disappear the next time you reboot. > -> You do not have to reboot to finish uninstalling Nix. The uninstall is -> complete. macOS (Catalina+) directly controls root directories and its -> read-only root will prevent you from manually deleting the empty `/nix` -> mountpoint. +> You do not have to reboot to finish uninstalling Nix. +> The uninstall is complete. +> macOS (Catalina+) directly controls root directories, and its read-only root will prevent you from manually deleting the empty `/nix` mountpoint. +## Single User + +To remove a [single-user installation](./installing-binary.md#single-user-installation) of Nix, run: + +```console +$ rm -rf /nix ~/.nix-channels ~/.nix-defexpr ~/.nix-profile +``` +You might also want to manually remove references to Nix from your `~/.profile`. diff --git a/doc/manual/src/installation/upgrading.md b/doc/manual/src/installation/upgrading.md index 38edcdbc5..a433f1d30 100644 --- a/doc/manual/src/installation/upgrading.md +++ b/doc/manual/src/installation/upgrading.md @@ -28,7 +28,7 @@ $ sudo su ## macOS multi-user ```console -$ sudo nix-env --install --file '' --attr nix -I nixpkgs=channel:nixpkgs-unstable +$ sudo nix-env --install --file '' --attr nix cacert -I nixpkgs=channel:nixpkgs-unstable $ sudo launchctl remove org.nixos.nix-daemon $ sudo launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist ``` diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 7306fc182..51b83fc8a 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -113,19 +113,18 @@ Derivations can declare some infrequently used optional attributes. > `nix-build`. If the [`configurable-impure-env` experimental - feature](@docroot@/contributing/experimental-features.md#xp-feature-configurable-impure-env) + feature](@docroot@/development/experimental-features.md#xp-feature-configurable-impure-env) is enabled, these environment variables can also be controlled through the [`impure-env`](@docroot@/command-ref/conf-file.md#conf-impure-env) configuration setting. - [`outputHash`]{#adv-attr-outputHash}; [`outputHashAlgo`]{#adv-attr-outputHashAlgo}; [`outputHashMode`]{#adv-attr-outputHashMode}\ - These attributes declare that the derivation is a so-called - *fixed-output derivation*, which means that a cryptographic hash of - the output is already known in advance. When the build of a - fixed-output derivation finishes, Nix computes the cryptographic - hash of the output and compares it to the hash declared with these - attributes. If there is a mismatch, the build fails. + These attributes declare that the derivation is a so-called *fixed-output derivation* (FOD), which means that a cryptographic hash of the output is already known in advance. + + As opposed to regular derivations, the [`builder`] executable of a fixed-output derivation has access to the network. + Nix computes a cryptographic hash of its output and compares that to the hash declared with these attributes. + If there is a mismatch, the derivation fails. The rationale for fixed-output derivations is derivations such as those produced by the `fetchurl` function. This function downloads a @@ -188,38 +187,49 @@ Derivations can declare some infrequently used optional attributes. } ``` - The `outputHashAlgo` attribute specifies the hash algorithm used to - compute the hash. It can currently be `"sha1"`, `"sha256"` or - `"sha512"`. + The `outputHash` attribute must be a string containing the hash in either hexadecimal or "nix32" encoding, or following the format for integrity metadata as defined by [SRI](https://www.w3.org/TR/SRI/). + The "nix32" encoding is an adaptation of base-32 encoding. + The [`convertHash`](@docroot@/language/builtins.md#builtins-convertHash) function shows how to convert between different encodings, and the [`nix-hash` command](../command-ref/nix-hash.md) has information about obtaining the hash for some contents, as well as converting to and from encodings. + + The `outputHashAlgo` attribute specifies the hash algorithm used to compute the hash. + It can currently be `"sha1"`, `"sha256"`, `"sha512"`, or `null`. + `outputHashAlgo` can only be `null` when `outputHash` follows the SRI format. The `outputHashMode` attribute determines how the hash is computed. - It must be one of the following two values: + It must be one of the following values: - - `"flat"`\ - The output must be a non-executable regular file. If it isn’t, - the build fails. The hash is simply computed over the contents - of that file (so it’s equal to what Unix commands like - `sha256sum` or `sha1sum` produce). + - [`"flat"`](@docroot@/store/store-object/content-address.md#method-flat) This is the default. - - `"recursive"`\ - The hash is computed over the NAR archive dump of the output - (i.e., the result of [`nix-store --dump`](@docroot@/command-ref/nix-store/dump.md)). In - this case, the output can be anything, including a directory - tree. + - [`"recursive"` or `"nar"`](@docroot@/store/store-object/content-address.md#method-nix-archive) - The `outputHash` attribute, finally, must be a string containing - the hash in either hexadecimal or base-32 notation. (See the - [`nix-hash` command](../command-ref/nix-hash.md) for information - about converting to and from base-32 notation.) + > **Compatibility** + > + > `"recursive"` is the traditional way of indicating this, + > and is supported since 2005 (virtually the entire history of Nix). + > `"nar"` is more clear, and consistent with other parts of Nix (such as the CLI), + > however support for it is only added in Nix version 2.21. + + - [`"text"`](@docroot@/store/store-object/content-address.md#method-text) + + > **Warning** + > + > The use of this method for derivation outputs is part of the [`dynamic-derivations`][xp-feature-dynamic-derivations] experimental feature. + + - [`"git"`](@docroot@/store/store-object/content-address.md#method-git) + + > **Warning** + > + > This method is part of the [`git-hashing`][xp-feature-git-hashing] experimental feature. - [`__contentAddressed`]{#adv-attr-__contentAddressed} + > **Warning** - > This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md). + > This attribute is part of an [experimental feature](@docroot@/development/experimental-features.md). > > To use this attribute, you must enable the - > [`ca-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-ca-derivations) experimental feature. + > [`ca-derivations`][xp-feature-ca-derivations] experimental feature. > For example, in [nix.conf](../command-ref/conf-file.md) you could add: > > ``` @@ -268,7 +278,9 @@ Derivations can declare some infrequently used optional attributes. > **Note** > - > If set to `false`, the [`builder`](./derivations.md#attr-builder) should be able to run on the system type specified in the [`system` attribute](./derivations.md#attr-system), since the derivation cannot be substituted. + > If set to `false`, the [`builder`] should be able to run on the system type specified in the [`system` attribute](./derivations.md#attr-system), since the derivation cannot be substituted. + + [`builder`]: ./derivations.md#attr-builder - [`__structuredAttrs`]{#adv-attr-structuredAttrs}\ If the special attribute `__structuredAttrs` is set to `true`, the other derivation @@ -290,6 +302,12 @@ Derivations can declare some infrequently used optional attributes. (associative) arrays. For example, the attribute `hardening.format = true` ends up as the Bash associative array element `${hardening[format]}`. + > **Warning** + > + > If set to `true`, other advanced attributes such as [`allowedReferences`](#adv-attr-allowedReferences), [`allowedReferences`](#adv-attr-allowedReferences), [`allowedRequisites`](#adv-attr-allowedRequisites), + [`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites), maxSize, and maxClosureSize. + will have no effect. + - [`outputChecks`]{#adv-attr-outputChecks}\ When using [structured attributes](#adv-attr-structuredAttrs), the `outputChecks` attribute allows defining checks per-output. @@ -299,7 +317,7 @@ Derivations can declare some infrequently used optional attributes. [`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites), the following attributes are available: - - `maxSize` defines the maximum size of the resulting [store object](../glossary.md#gloss-store-object). + - `maxSize` defines the maximum size of the resulting [store object](@docroot@/store/store-object.md). - `maxClosureSize` defines the maximum size of the output's closure. - `ignoreSelfRefs` controls whether self-references should be considered when checking for allowed references/requisites. @@ -351,3 +369,7 @@ Derivations can declare some infrequently used optional attributes. ``` ensures that the derivation can only be built on a machine with the `kvm` feature. + +[xp-feature-ca-derivations]: @docroot@/development/experimental-features.md#xp-feature-ca-derivations +[xp-feature-dynamic-derivations]: @docroot@/development/experimental-features.md#xp-feature-dynamic-derivations +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/doc/manual/src/language/builtin-constants-prefix.md b/doc/manual/src/language/builtin-constants-prefix.md deleted file mode 100644 index 50f43006d..000000000 --- a/doc/manual/src/language/builtin-constants-prefix.md +++ /dev/null @@ -1,5 +0,0 @@ -# Built-in Constants - -These constants are built into the Nix language evaluator: - - diff --git a/doc/manual/src/language/builtin-constants-suffix.md b/doc/manual/src/language/builtin-constants-suffix.md deleted file mode 100644 index a74db2857..000000000 --- a/doc/manual/src/language/builtin-constants-suffix.md +++ /dev/null @@ -1 +0,0 @@ - diff --git a/doc/manual/src/language/builtins-prefix.md b/doc/manual/src/language/builtins-prefix.md index 7b2321466..fb983bb7f 100644 --- a/doc/manual/src/language/builtins-prefix.md +++ b/doc/manual/src/language/builtins-prefix.md @@ -1,9 +1,11 @@ -# Built-in Functions +# Built-ins -This section lists the functions built into the Nix language evaluator. -All built-in functions are available through the global [`builtins`](./builtin-constants.md#builtins-builtins) constant. +This section lists the values and functions built into the Nix language evaluator. +All built-ins are available through the global [`builtins`](#builtins-builtins) constant. -For convenience, some built-ins can be accessed directly: +Some built-ins are also exposed directly in the global scope: + + - [`derivation`](#builtins-derivation) - [`import`](#builtins-import) diff --git a/doc/manual/src/language/constructs.md b/doc/manual/src/language/constructs.md deleted file mode 100644 index a82ec5960..000000000 --- a/doc/manual/src/language/constructs.md +++ /dev/null @@ -1,408 +0,0 @@ -# Language Constructs - -## Recursive sets - -Recursive sets are like normal [attribute sets](./values.md#attribute-set), but the attributes can refer to each other. - -> *rec-attrset* = `rec {` [ *name* `=` *expr* `;` `]`... `}` - -Example: - -```nix -rec { - x = y; - y = 123; -}.x -``` - -This evaluates to `123`. - -Note that without `rec` the binding `x = y;` would -refer to the variable `y` in the surrounding scope, if one exists, and -would be invalid if no such variable exists. That is, in a normal -(non-recursive) set, attributes are not added to the lexical scope; in a -recursive set, they are. - -Recursive sets of course introduce the danger of infinite recursion. For -example, the expression - -```nix -rec { - x = y; - y = x; -}.x -``` - -will crash with an `infinite recursion encountered` error message. - -## Let-expressions - -A let-expression allows you to define local variables for an expression. - -> *let-in* = `let` [ *identifier* = *expr* ]... `in` *expr* - -Example: - -```nix -let - x = "foo"; - y = "bar"; -in x + y -``` - -This evaluates to `"foobar"`. - -## Inheriting attributes - -When defining an [attribute set](./values.md#attribute-set) or in a [let-expression](#let-expressions) it is often convenient to copy variables from the surrounding lexical scope (e.g., when you want to propagate attributes). -This can be shortened using the `inherit` keyword. - -Example: - -```nix -let x = 123; in -{ - inherit x; - y = 456; -} -``` - -is equivalent to - -```nix -let x = 123; in -{ - x = x; - y = 456; -} -``` - -and both evaluate to `{ x = 123; y = 456; }`. - -> **Note** -> -> This works because `x` is added to the lexical scope by the `let` construct. - -It is also possible to inherit attributes from another attribute set. - -Example: - -In this fragment from `all-packages.nix`, - -```nix -graphviz = (import ../tools/graphics/graphviz) { - inherit fetchurl stdenv libpng libjpeg expat x11 yacc; - inherit (xorg) libXaw; -}; - -xorg = { - libX11 = ...; - libXaw = ...; - ... -} - -libpng = ...; -libjpg = ...; -... -``` - -the set used in the function call to the function defined in -`../tools/graphics/graphviz` inherits a number of variables from the -surrounding scope (`fetchurl` ... `yacc`), but also inherits `libXaw` -(the X Athena Widgets) from the `xorg` set. - -Summarizing the fragment - -```nix -... -inherit x y z; -inherit (src-set) a b c; -... -``` - -is equivalent to - -```nix -... -x = x; y = y; z = z; -a = src-set.a; b = src-set.b; c = src-set.c; -... -``` - -when used while defining local variables in a let-expression or while -defining a set. - -In a `let` expression, `inherit` can be used to selectively bring specific attributes of a set into scope. For example - - -```nix -let - x = { a = 1; b = 2; }; - inherit (builtins) attrNames; -in -{ - names = attrNames x; -} -``` - -is equivalent to - -```nix -let - x = { a = 1; b = 2; }; -in -{ - names = builtins.attrNames x; -} -``` - -both evaluate to `{ names = [ "a" "b" ]; }`. - -## Functions - -Functions have the following form: - -```nix -pattern: body -``` - -The pattern specifies what the argument of the function must look like, -and binds variables in the body to (parts of) the argument. There are -three kinds of patterns: - - - If a pattern is a single identifier, then the function matches any - argument. Example: - - ```nix - let negate = x: !x; - concat = x: y: x + y; - in if negate true then concat "foo" "bar" else "" - ``` - - Note that `concat` is a function that takes one argument and returns - a function that takes another argument. This allows partial - parameterisation (i.e., only filling some of the arguments of a - function); e.g., - - ```nix - map (concat "foo") [ "bar" "bla" "abc" ] - ``` - - evaluates to `[ "foobar" "foobla" "fooabc" ]`. - - - A *set pattern* of the form `{ name1, name2, …, nameN }` matches a - set containing the listed attributes, and binds the values of those - attributes to variables in the function body. For example, the - function - - ```nix - { x, y, z }: z + y + x - ``` - - can only be called with a set containing exactly the attributes `x`, - `y` and `z`. No other attributes are allowed. If you want to allow - additional arguments, you can use an ellipsis (`...`): - - ```nix - { x, y, z, ... }: z + y + x - ``` - - This works on any set that contains at least the three named - attributes. - - It is possible to provide *default values* for attributes, in - which case they are allowed to be missing. A default value is - specified by writing `name ? e`, where *e* is an arbitrary - expression. For example, - - ```nix - { x, y ? "foo", z ? "bar" }: z + y + x - ``` - - specifies a function that only requires an attribute named `x`, but - optionally accepts `y` and `z`. - - - An `@`-pattern provides a means of referring to the whole value - being matched: - - ```nix - args@{ x, y, z, ... }: z + y + x + args.a - ``` - - but can also be written as: - - ```nix - { x, y, z, ... } @ args: z + y + x + args.a - ``` - - Here `args` is bound to the argument *as passed*, which is further - matched against the pattern `{ x, y, z, ... }`. - The `@`-pattern makes mainly sense with an ellipsis(`...`) as - you can access attribute names as `a`, using `args.a`, which was - given as an additional attribute to the function. - - > **Warning** - > - > `args@` binds the name `args` to the attribute set that is passed to the function. - > In particular, `args` does *not* include any default values specified with `?` in the function's set pattern. - > - > For instance - > - > ```nix - > let - > f = args@{ a ? 23, ... }: [ a args ]; - > in - > f {} - > ``` - > - > is equivalent to - > - > ```nix - > let - > f = args @ { ... }: [ (args.a or 23) args ]; - > in - > f {} - > ``` - > - > and both expressions will evaluate to: - > - > ```nix - > [ 23 {} ] - > ``` - -Note that functions do not have names. If you want to give them a name, -you can bind them to an attribute, e.g., - -```nix -let concat = { x, y }: x + y; -in concat { x = "foo"; y = "bar"; } -``` - -## Conditionals - -Conditionals look like this: - -```nix -if e1 then e2 else e3 -``` - -where *e1* is an expression that should evaluate to a Boolean value -(`true` or `false`). - -## Assertions - -Assertions are generally used to check that certain requirements on or -between features and dependencies hold. They look like this: - -```nix -assert e1; e2 -``` - -where *e1* is an expression that should evaluate to a Boolean value. If -it evaluates to `true`, *e2* is returned; otherwise expression -evaluation is aborted and a backtrace is printed. - -Here is a Nix expression for the Subversion package that shows how -assertions can be used:. - -```nix -{ localServer ? false -, httpServer ? false -, sslSupport ? false -, pythonBindings ? false -, javaSwigBindings ? false -, javahlBindings ? false -, stdenv, fetchurl -, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null -}: - -assert localServer -> db4 != null; â‘ -assert httpServer -> httpd != null && httpd.expat == expat; â‘¡ -assert sslSupport -> openssl != null && (httpServer -> httpd.openssl == openssl); â‘¢ -assert pythonBindings -> swig != null && swig.pythonSupport; -assert javaSwigBindings -> swig != null && swig.javaSupport; -assert javahlBindings -> j2sdk != null; - -stdenv.mkDerivation { - name = "subversion-1.1.1"; - ... - openssl = if sslSupport then openssl else null; â‘£ - ... -} -``` - -The points of interest are: - -1. This assertion states that if Subversion is to have support for - local repositories, then Berkeley DB is needed. So if the Subversion - function is called with the `localServer` argument set to `true` but - the `db4` argument set to `null`, then the evaluation fails. - - Note that `->` is the [logical - implication](https://en.wikipedia.org/wiki/Truth_table#Logical_implication) - Boolean operation. - -2. This is a more subtle condition: if Subversion is built with Apache - (`httpServer`) support, then the Expat library (an XML library) used - by Subversion should be same as the one used by Apache. This is - because in this configuration Subversion code ends up being linked - with Apache code, and if the Expat libraries do not match, a build- - or runtime link error or incompatibility might occur. - -3. This assertion says that in order for Subversion to have SSL support - (so that it can access `https` URLs), an OpenSSL library must be - passed. Additionally, it says that *if* Apache support is enabled, - then Apache's OpenSSL should match Subversion's. (Note that if - Apache support is not enabled, we don't care about Apache's - OpenSSL.) - -4. The conditional here is not really related to assertions, but is - worth pointing out: it ensures that if SSL support is disabled, then - the Subversion derivation is not dependent on OpenSSL, even if a - non-`null` value was passed. This prevents an unnecessary rebuild of - Subversion if OpenSSL changes. - -## With-expressions - -A *with-expression*, - -```nix -with e1; e2 -``` - -introduces the set *e1* into the lexical scope of the expression *e2*. -For instance, - -```nix -let as = { x = "foo"; y = "bar"; }; -in with as; x + y -``` - -evaluates to `"foobar"` since the `with` adds the `x` and `y` attributes -of `as` to the lexical scope in the expression `x + y`. The most common -use of `with` is in conjunction with the `import` function. E.g., - -```nix -with (import ./definitions.nix); ... -``` - -makes all attributes defined in the file `definitions.nix` available as -if they were defined locally in a `let`-expression. - -The bindings introduced by `with` do not shadow bindings introduced by -other means, e.g. - -```nix -let a = 3; in with { a = 1; }; let a = 4; in with { a = 2; }; ... -``` - -establishes the same scope as - -```nix -let a = 1; in let a = 2; in let a = 3; in let a = 4; in ... -``` - -## Comments - -Comments can be single-line, started with a `#` character, or -inline/multi-line, enclosed within `/* ... */`. diff --git a/doc/manual/src/language/constructs/lookup-path.md b/doc/manual/src/language/constructs/lookup-path.md index e87d2922b..11b9fe88c 100644 --- a/doc/manual/src/language/constructs/lookup-path.md +++ b/doc/manual/src/language/constructs/lookup-path.md @@ -4,9 +4,9 @@ > > *lookup-path* = `<` *identifier* [ `/` *identifier* ]... `>` -A lookup path is an identifier with an optional path suffix that resolves to a [path value](@docroot@/language/values.md#type-path) if the identifier matches a search path entry. +A lookup path is an identifier with an optional path suffix that resolves to a [path value](@docroot@/language/types.md#type-path) if the identifier matches a search path entry. -The value of a lookup path is determined by [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath). +The value of a lookup path is determined by [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath). See [`builtins.findFile`](@docroot@/language/builtins.md#builtins-findFile) for details on lookup path resolution. diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index 75f824a34..8e3f0f791 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -12,12 +12,12 @@ It outputs an attribute set, and produces a [store derivation] as a side effect ### Required -- [`name`]{#attr-name} ([String](@docroot@/language/values.md#type-string)) +- [`name`]{#attr-name} ([String](@docroot@/language/types.md#type-string)) A symbolic name for the derivation. It is added to the [store path] of the corresponding [store derivation] as well as to its [output paths](@docroot@/glossary.md#gloss-output-path). - [store path]: @docroot@/glossary.md#gloss-store-path + [store path]: @docroot@/store/store-path.md > **Example** > @@ -31,7 +31,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect > The store derivation's path will be `/nix/store/-hello.drv`. > The [output](#attr-outputs) paths will be of the form `/nix/store/-hello[-]` -- [`system`]{#attr-system} ([String](@docroot@/language/values.md#type-string)) +- [`system`]{#attr-system} ([String](@docroot@/language/types.md#type-string)) The system type on which the [`builder`](#attr-builder) executable is meant to be run. @@ -64,9 +64,9 @@ It outputs an attribute set, and produces a [store derivation] as a side effect > } > ``` > - > [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) has the value of the [`system` configuration option], and defaults to the system type of the current Nix installation. + > [`builtins.currentSystem`](@docroot@/language/builtins.md#builtins-currentSystem) has the value of the [`system` configuration option], and defaults to the system type of the current Nix installation. -- [`builder`]{#attr-builder} ([Path](@docroot@/language/values.md#type-path) | [String](@docroot@/language/values.md#type-string)) +- [`builder`]{#attr-builder} ([Path](@docroot@/language/types.md#type-path) | [String](@docroot@/language/types.md#type-string)) Path to an executable that will perform the build. @@ -113,7 +113,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect ### Optional -- [`args`]{#attr-args} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) +- [`args`]{#attr-args} ([List](@docroot@/language/types.md#list) of [String](@docroot@/language/types.md#type-string)) Default: `[ ]` @@ -132,7 +132,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect > }; > ``` -- [`outputs`]{#attr-outputs} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) +- [`outputs`]{#attr-outputs} ([List](@docroot@/language/types.md#list) of [String](@docroot@/language/types.md#type-string)) Default: `[ "out" ]` @@ -141,7 +141,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect By default, a derivation produces a single output called `out`. However, derivations can produce multiple outputs. - This allows the associated [store objects](@docroot@/glossary.md#gloss-store-object) and their [closures](@docroot@/glossary.md#gloss-closure) to be copied or garbage-collected separately. + This allows the associated [store objects](@docroot@/store/store-object.md) and their [closures](@docroot@/glossary.md#gloss-closure) to be copied or garbage-collected separately. > **Example** > diff --git a/doc/manual/src/language/identifiers.md b/doc/manual/src/language/identifiers.md new file mode 100644 index 000000000..861ee3e20 --- /dev/null +++ b/doc/manual/src/language/identifiers.md @@ -0,0 +1,51 @@ +# Identifiers + +An *identifier* is an [ASCII](https://en.wikipedia.org/wiki/ASCII) character sequence that: +- Starts with a letter (`a-z`, `A-Z`) or underscore (`_`) +- Can contain any number of: + - Letters (`a-z`, `A-Z`) + - Digits (`0-9`) + - Underscores (`_`) + - Apostrophes (`'`) + - Hyphens (`-`) +- Is not one of the [keywords](#keywords) + +> **Syntax** +> +> *identifier* ~ `[A-Za-z_][A-Za-z0-9_'-]*` + +# Names + +A *name* can be written as an [identifier](#identifier) or a [string literal](./syntax.md#string-literal). + +> **Syntax** +> +> *name* → *identifier* | *string* + +Names are used in [attribute sets](./syntax.md#attrs-literal), [`let` bindings](./syntax.md#let-expressions), and [`inherit`](./syntax.md#inheriting-attributes). +Two names are the same if they represent the same sequence of characters, regardless of whether they are written as identifiers or strings. + +# Keywords + +These keywords are reserved and cannot be used as [identifiers](#identifiers): + +- [`assert`](./syntax.md#assertions) +- [`else`][if] +- [`if`][if] +- [`in`][let] +- [`inherit`](./syntax.md#inheriting-attributes) +- [`let`][let] +- [`or`](./operators.md#attribute-selection) (see note) +- [`rec`](./syntax.md#recursive-sets) +- [`then`][if] +- [`with`](./syntax.md#with-expressions) + +[if]: ./syntax.md#conditionals +[let]: ./syntax.md#let-expressions + +> **Note** +> +> The Nix language evaluator currently allows `or` to be used as a name in some contexts, for backwards compatibility reasons. +> Users are advised not to rely on this. +> +> There are long-standing issues with how `or` is parsed as a name, which can't be resolved without making a breaking change to the language. diff --git a/doc/manual/src/language/import-from-derivation.md b/doc/manual/src/language/import-from-derivation.md index fb12ba51a..e901f5bcf 100644 --- a/doc/manual/src/language/import-from-derivation.md +++ b/doc/manual/src/language/import-from-derivation.md @@ -2,9 +2,9 @@ The value of a Nix expression can depend on the contents of a [store object]. -[store object]: @docroot@/glossary.md#gloss-store-object +[store object]: @docroot@/store/store-object.md -Passing an expression `expr` that evaluates to a [store path](@docroot@/glossary.md#gloss-store-path) to any built-in function which reads from the filesystem constitutes Import From Derivation (IFD): +Passing an expression `expr` that evaluates to a [store path](@docroot@/store/store-path.md) to any built-in function which reads from the filesystem constitutes Import From Derivation (IFD): - [`import`](./builtins.md#builtins-import)` expr` - [`builtins.readFile`](./builtins.md#builtins-readFile)` expr` diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md index a26e43a05..2bfdbb8a0 100644 --- a/doc/manual/src/language/index.md +++ b/doc/manual/src/language/index.md @@ -1,7 +1,13 @@ # Nix Language The Nix language is designed for conveniently creating and composing *derivations* – precise descriptions of how contents of existing files are used to derive new files. -It is: + +> **Tip** +> +> These pages are written as a reference. +> If you are learning Nix, nix.dev has a good [introduction to the Nix language](https://nix.dev/tutorials/nix-language). + +The language is: - *domain-specific* @@ -47,7 +53,7 @@ This is an incomplete overview of language features, by example. - *Basic values* + *Basic values ([primitives](@docroot@/language/types.md#primitives))* @@ -65,7 +71,7 @@ This is an incomplete overview of language features, by example. - A string + A [string](@docroot@/language/types.md#type-string) @@ -88,6 +94,18 @@ This is an incomplete overview of language features, by example. + + + + `# Explanation` + + + + + A [comment](@docroot@/language/syntax.md#comments). + + + @@ -100,7 +118,7 @@ This is an incomplete overview of language features, by example. - String interpolation (expands to `"hello world"`, `"1 2 3"`, `"/nix/store/-bash-/bin/sh"`) + [String interpolation](@docroot@/language/string-interpolation.md) (expands to `"hello world"`, `"1 2 3"`, `"/nix/store/-bash-/bin/sh"`) @@ -112,7 +130,7 @@ This is an incomplete overview of language features, by example. - Booleans + [Booleans](@docroot@/language/types.md#type-boolean) @@ -124,7 +142,7 @@ This is an incomplete overview of language features, by example. - Null value + [Null](@docroot@/language/types.md#type-null) value @@ -136,7 +154,7 @@ This is an incomplete overview of language features, by example. - An integer + An [integer](@docroot@/language/types.md#type-int) @@ -148,7 +166,7 @@ This is an incomplete overview of language features, by example. - A floating point number + A [floating point number](@docroot@/language/types.md#type-float) @@ -160,7 +178,7 @@ This is an incomplete overview of language features, by example. - An absolute path + An absolute [path](@docroot@/language/types.md#type-path) @@ -172,7 +190,7 @@ This is an incomplete overview of language features, by example. - A path relative to the file containing this Nix expression + A [path](@docroot@/language/types.md#type-path) relative to the file containing this Nix expression @@ -184,7 +202,7 @@ This is an incomplete overview of language features, by example. - A home path. Evaluates to the `"/.config"`. + A home [path](@docroot@/language/types.md#type-path). Evaluates to the `"/.config"`. @@ -196,7 +214,7 @@ This is an incomplete overview of language features, by example. - Search path for Nix files. Value determined by [`$NIX_PATH` environment variable](../command-ref/env-common.md#env-NIX_PATH). + A [lookup path](@docroot@/language/constructs/lookup-path.md) for Nix files. Value determined by [`$NIX_PATH` environment variable](../command-ref/env-common.md#env-NIX_PATH). @@ -220,7 +238,7 @@ This is an incomplete overview of language features, by example. - A set with attributes named `x` and `y` + An [attribute set](@docroot@/language/types.md#attribute-set) with attributes named `x` and `y` @@ -244,7 +262,7 @@ This is an incomplete overview of language features, by example. - A recursive set, equivalent to `{ x = "foo"; y = "foobar"; }` + A [recursive set](@docroot@/language/syntax.md#recursive-sets), equivalent to `{ x = "foo"; y = "foobar"; }`. @@ -260,7 +278,7 @@ This is an incomplete overview of language features, by example. - Lists with three elements. + [Lists](@docroot@/language/types.md#list) with three elements. @@ -344,7 +362,7 @@ This is an incomplete overview of language features, by example. - Attribute selection (evaluates to `1`) + [Attribute selection](@docroot@/language/types.md#attribute-set) (evaluates to `1`) @@ -356,7 +374,7 @@ This is an incomplete overview of language features, by example. - Attribute selection with default (evaluates to `3`) + [Attribute selection](@docroot@/language/types.md#attribute-set) with default (evaluates to `3`) @@ -392,7 +410,7 @@ This is an incomplete overview of language features, by example. - Conditional expression + [Conditional expression](@docroot@/language/syntax.md#conditionals). @@ -404,7 +422,7 @@ This is an incomplete overview of language features, by example. - Assertion check (evaluates to `"yes!"`). + [Assertion](@docroot@/language/syntax.md#assertions) check (evaluates to `"yes!"`). @@ -416,7 +434,7 @@ This is an incomplete overview of language features, by example. - Variable definition + Variable definition. See [`let`-expressions](@docroot@/language/syntax.md#let-expressions). @@ -428,14 +446,44 @@ This is an incomplete overview of language features, by example. - Add all attributes from the given set to the scope (evaluates to `1`) + Add all attributes from the given set to the scope (evaluates to `1`). + + See [`with`-expressions](@docroot@/language/syntax.md#with-expressions) for details and shadowing caveats. - *Functions (lambdas)* + `inherit pkgs src;` + + + + + Adds the variables to the current scope (attribute set or `let` binding). + Desugars to `pkgs = pkgs; src = src;`. + See [Inheriting attributes](@docroot@/language/syntax.md#inheriting-attributes). + + + + + + + `inherit (pkgs) lib stdenv;` + + + + + Adds the attributes, from the attribute set in parentheses, to the current scope (attribute set or `let` binding). + Desugars to `lib = pkgs.lib; stdenv = pkgs.stdenv;`. + See [Inheriting attributes](@docroot@/language/syntax.md#inheriting-attributes). + + + + + + + *[Functions](@docroot@/language/syntax.md#functions) (lambdas)* @@ -452,7 +500,7 @@ This is an incomplete overview of language features, by example. - A function that expects an integer and returns it increased by 1 + A [function](@docroot@/language/syntax.md#functions) that expects an integer and returns it increased by 1. @@ -464,7 +512,7 @@ This is an incomplete overview of language features, by example. - Curried function, equivalent to `x: (y: x + y)`. Can be used like a function that takes two arguments and returns their sum. + Curried [function](@docroot@/language/syntax.md#functions), equivalent to `x: (y: x + y)`. Can be used like a function that takes two arguments and returns their sum. @@ -476,7 +524,7 @@ This is an incomplete overview of language features, by example. - A function call (evaluates to 101) + A [function](@docroot@/language/syntax.md#functions) call (evaluates to 101) @@ -488,7 +536,7 @@ This is an incomplete overview of language features, by example. - A function bound to a variable and subsequently called by name (evaluates to 103) + A [function](@docroot@/language/syntax.md#functions) bound to a variable and subsequently called by name (evaluates to 103) @@ -500,7 +548,7 @@ This is an incomplete overview of language features, by example. - A function that expects a set with required attributes `x` and `y` and concatenates them + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attributes `x` and `y` and concatenates them @@ -512,7 +560,7 @@ This is an incomplete overview of language features, by example. - A function that expects a set with required attribute `x` and optional `y`, using `"bar"` as default value for `y` + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attribute `x` and optional `y`, using `"bar"` as default value for `y` @@ -524,7 +572,7 @@ This is an incomplete overview of language features, by example. - A function that expects a set with required attributes `x` and `y` and ignores any other attributes + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attributes `x` and `y` and ignores any other attributes @@ -538,7 +586,7 @@ This is an incomplete overview of language features, by example. - A function that expects a set with required attributes `x` and `y`, and binds the whole set to `args` + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attributes `x` and `y`, and binds the whole set to `args` @@ -562,7 +610,8 @@ This is an incomplete overview of language features, by example. - Load and return Nix expression in given file + Load and return Nix expression in given file. + See [import](@docroot@/language/builtins.md#builtins-import). @@ -574,7 +623,8 @@ This is an incomplete overview of language features, by example. - Apply a function to every element of a list (evaluates to `[ 2 4 6 ]`) + Apply a function to every element of a list (evaluates to `[ 2 4 6 ]`). + See [`map`](@docroot@/language/builtins.md#builtins-map). diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index 6fd66864b..e1c020781 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -26,12 +26,16 @@ | Logical conjunction (`AND`) | *bool* `&&` *bool* | left | 12 | | Logical disjunction (`OR`) | *bool* \|\| *bool* | left | 13 | | [Logical implication] | *bool* `->` *bool* | right | 14 | +| [Pipe operator] (experimental) | *expr* `\|>` *func* | left | 15 | +| [Pipe operator] (experimental) | *func* `<\|` *expr* | right | 15 | -[string]: ./values.md#type-string -[path]: ./values.md#type-path -[number]: ./values.md#type-number -[list]: ./values.md#list -[attribute set]: ./values.md#attribute-set +[string]: ./types.md#type-string +[path]: ./types.md#type-path +[number]: ./types.md#type-float +[list]: ./types.md#list +[attribute set]: ./types.md#attribute-set + + ## Attribute selection @@ -42,12 +46,6 @@ Select the attribute denoted by attribute path *attrpath* from [attribute set] *attrset*. If the attribute doesn’t exist, return the *expr* after `or` if provided, otherwise abort evaluation. -An attribute path is a dot-separated list of [attribute names](./values.md#attribute-set). - -> **Syntax** -> -> *attrpath* = *name* [ `.` *name* ]... - [Attribute selection]: #attribute-selection ## Has attribute @@ -61,7 +59,7 @@ The result is a [Boolean] value. See also: [`builtins.hasAttr`](@docroot@/language/builtins.md#builtins-hasAttr) -[Boolean]: ./values.md#type-boolean +[Boolean]: ./types.md#type-boolean [Has attribute]: #has-attribute @@ -128,8 +126,8 @@ The result is a string. > The file or directory at *path* must exist and is copied to the [store]. > The path appears in the result as the corresponding [store path]. -[store path]: ../glossary.md#gloss-store-path -[store]: ../glossary.md#gloss-store +[store path]: @docroot@/store/store-path.md +[store]: @docroot@/glossary.md#gloss-store [String and path concatenation]: #string-and-path-concatenation @@ -141,7 +139,7 @@ The result is a string. Update [attribute set] *attrset1* with names and values from *attrset2*. -The returned attribute set will have of all the attributes in *attrset1* and *attrset2*. +The returned attribute set will have all of the attributes in *attrset1* and *attrset2*. If an attribute name is present in both, the attribute value from the latter is taken. [Update]: #update @@ -172,7 +170,7 @@ All comparison operators are implemented in terms of `<`, and the following equi - Numbers are type-compatible, see [arithmetic] operators. - Floating point numbers only differ up to a limited precision. -[function]: ./constructs.md#functions +[function]: ./syntax.md#functions [Equality]: #equality @@ -182,3 +180,34 @@ Equivalent to `!`*b1* `||` *b2*. [Logical implication]: #logical-implication +## Pipe operators + +- *a* `|>` *b* is equivalent to *b* *a* +- *a* `<|` *b* is equivalent to *a* *b* + +> **Example** +> +> ``` +> nix-repl> 1 |> builtins.add 2 |> builtins.mul 3 +> 9 +> +> nix-repl> builtins.add 1 <| builtins.mul 2 <| 3 +> 7 +> ``` + +> **Warning** +> +> This syntax is part of an +> [experimental feature](@docroot@/development/experimental-features.md) +> and may change in future releases. +> +> To use this syntax, make sure the +> [`pipe-operators` experimental feature](@docroot@/development/experimental-features.md#xp-feature-pipe-operators) +> is enabled. +> For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): +> +> ``` +> extra-experimental-features = pipe-operators +> ``` + +[Pipe operator]: #pipe-operators diff --git a/doc/manual/src/language/scope.md b/doc/manual/src/language/scope.md new file mode 100644 index 000000000..9373324e2 --- /dev/null +++ b/doc/manual/src/language/scope.md @@ -0,0 +1,28 @@ +# Scoping rules + +A *scope* in the Nix language is a dictionary keyed by [name](./identifiers.md#names), mapping each name to an expression and a *definition type*. +The definition type is either *explicit* or *implicit*. +Each entry in this dictionary is a *definition*. + +Explicit definitions are created by the following expressions: +- [let-expressions](syntax.md#let-expressions) +- [recursive attribute set literals](syntax.md#recursive-sets) (`rec`) +- [function literals](syntax.md#functions) + +Implicit definitions are only created by [with-expressions](./syntax.md#with-expressions). + +Every expression is *enclosed* by a scope. +The outermost expression is enclosed by the [built-in, global scope](./builtins.md), which contains only explicit definitions. +The expressions listed above *extend* their enclosing scope by adding new definitions, or replacing existing ones with the same name. +An explicit definition can replace a definition of any type; an implicit definition can only replace another implicit definition. + +Each of the above expressions defines which of its subexpressions are enclosed by the extended scope. +In all other cases, the same scope that encloses an expression is the enclosing scope for its subexpressions. + +The Nix language is [statically scoped](https://en.wikipedia.org/wiki/Scope_(computer_science)#Lexical_scope); +the value of a variable is determined only by the variable's enclosing scope, and not by the dynamic context in which the variable is evaluated. + +> **Note** +> +> Expressions entered into the [Nix REPL](@docroot@/command-ref/new-cli/nix3-repl.md) are enclosed by a scope that can be extended by command line arguments or previous REPL commands. +> These ways of extending scope are not, strictly speaking, part of the Nix language. diff --git a/doc/manual/src/language/string-context.md b/doc/manual/src/language/string-context.md new file mode 100644 index 000000000..6a3482cfd --- /dev/null +++ b/doc/manual/src/language/string-context.md @@ -0,0 +1,134 @@ +# String context + +> **Note** +> +> This is an advanced topic. +> The Nix language is designed to be used without the programmer consciously dealing with string contexts or even knowing what they are. + +A string in the Nix language is not just a sequence of characters like strings in other languages. +It is actually a pair of a sequence of characters and a *string context*. +The string context is an (unordered) set of *string context elements*. + +The purpose of string contexts is to collect non-string values attached to strings via +[string concatenation](./operators.md#string-concatenation), +[string interpolation](./string-interpolation.md), +and similar operations. +The idea is that a user can combine together values to create a build instructions for derivations without manually keeping track of where they come from. +Then the Nix language implicitly does that bookkeeping to efficiently obtain the closure of derivation inputs. + +> **Note** +> +> String contexts are *not* explicitly manipulated in idiomatic Nix language code. + +String context elements come in different forms: + +- [deriving path]{#string-context-element-derived-path} + + A string context element of this type is a [deriving path](@docroot@/glossary.md#gloss-deriving-path). + They can be either of type [constant](#string-context-constant) or [output](#string-context-output), which correspond to the types of deriving paths. + + - [Constant string context elements]{#string-context-constant} + + > **Example** + > + > [`builtins.storePath`] creates a string with a single constant string context element: + > + > ```nix + > builtins.getContext (builtins.storePath "/nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10") + > ``` + > evaluates to + > ```nix + > { + > "/nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10" = { + > path = true; + > }; + > } + > ``` + + [deriving path]: @docroot@/glossary.md#gloss-deriving-path + [store path]: @docroot@/glossary.md#gloss-store-path + [`builtins.storePath`]: ./builtins.md#builtins-storePath + + - [Output string context elements]{#string-context-output} + + > **Example** + > + > The behavior of string contexts are best demonstrated with a built-in function that is still experimental: [`builtins.outputOf`]. + > This example will *not* work with stable Nix! + > + > ```nix + > builtins.getContext + > (builtins.outputOf + > (builtins.storePath "/nix/store/fvchh9cvcr7kdla6n860hshchsba305w-hello-2.12.drv") + > "out") + > ``` + > evaluates to + > ```nix + > { + > "/nix/store/fvchh9cvcr7kdla6n860hshchsba305w-hello-2.12.drv" = { + > outputs = [ "out" ]; + > }; + > } + > ``` + + [`builtins.outputOf`]: ./builtins.md#builtins-outputOf + +- [*derivation deep*]{#string-context-element-derivation-deep} + + *derivation deep* is an advanced feature intended to be used with the + [`exportReferencesGraph` derivation attribute](./advanced-attributes.html#adv-attr-exportReferencesGraph). + A *derivation deep* string context element is a derivation path, and refers to both its outputs and the entire build closure of that derivation: + all its outputs, all the other derivations the given derivation depends on, and all the outputs of those. + + > **Example** + > + > The best way to illustrate *derivation deep* string contexts is with [`builtins.addDrvOutputDependencies`]. + > Take a regular constant string context element pointing to a derivation, and transform it into a "Derivation deep" string context element. + > + > ```nix + > builtins.getContext + > (builtins.addDrvOutputDependencies + > (builtins.storePath "/nix/store/fvchh9cvcr7kdla6n860hshchsba305w-hello-2.12.drv")) + > ``` + > evaluates to + > ```nix + > { + > "/nix/store/fvchh9cvcr7kdla6n860hshchsba305w-hello-2.12.drv" = { + > allOutputs = true; + > }; + > } + > ``` + + [`builtins.addDrvOutputDependencies`]: ./builtins.md#builtins-addDrvOutputDependencies + [`builtins.unsafeDiscardOutputDependency`]: ./builtins.md#builtins-unsafeDiscardOutputDependency + +## Inspecting string contexts + +Most basically, [`builtins.hasContext`] will tell whether a string has a non-empty context. + +When more granular information is needed, [`builtins.getContext`] can be used. +It creates an [attribute set] representing the string context, which can be inspected as usual. + +[`builtins.hasContext`]: ./builtins.md#builtins-hasContext +[`builtins.getContext`]: ./builtins.md#builtins-getContext +[attribute set]: ./types.md#attribute-set + +## Clearing string contexts + +[`buitins.unsafeDiscardStringContext`](./builtins.md#builtins-unsafeDiscardStringContext) will make a copy of a string, but with an empty string context. +The returned string can be used in more ways, e.g. by operators that require the string context to be empty. +The requirement to explicitly discard the string context in such use cases helps ensure that string context elements are not lost by mistake. +The "unsafe" marker is only there to remind that Nix normally guarantees that dependencies are tracked, whereas the returned string has lost them. + +## Constructing string contexts + +[`builtins.appendContext`] will create a copy of a string, but with additional string context elements. +The context is specified explicitly by an [attribute set] in the format that [`builtins.hasContext`] produces. +A string with arbitrary contexts can be made like this: + +1. Create a string with the desired string context elements. + (The contents of the string do not matter.) +2. Dump its context with [`builtins.getContext`]. +3. Combine it with a base string and repeated [`builtins.appendContext`] calls. + +[`builtins.appendContext`]: ./builtins.md#builtins-appendContext diff --git a/doc/manual/src/language/string-interpolation.md b/doc/manual/src/language/string-interpolation.md index 7d81c2020..1778bdfa0 100644 --- a/doc/manual/src/language/string-interpolation.md +++ b/doc/manual/src/language/string-interpolation.md @@ -4,9 +4,9 @@ String interpolation is a language feature where a [string], [path], or [attribu Such a construct is called *interpolated string*, and the expression inside is an [interpolated expression](#interpolated-expression). -[string]: ./values.md#type-string -[path]: ./values.md#type-path -[attribute set]: ./values.md#attribute-set +[string]: ./types.md#type-string +[path]: ./types.md#type-path +[attribute set]: ./types.md#attribute-set ## Examples @@ -20,7 +20,7 @@ Rather than writing (where `freetype` is a [derivation]), you can instead write -[derivation]: ../glossary.md#gloss-derivation +[derivation]: @docroot@/glossary.md#gloss-derivation ```nix "--with-freetype2-library=${freetype}/lib" @@ -43,6 +43,47 @@ configureFlags = " Note that Nix expressions and strings can be arbitrarily nested; in this case the outer string contains various interpolated expressions that themselves contain strings (e.g., `"-thread"`), some of which in turn contain interpolated expressions (e.g., `${mesa}`). +To write a literal `${` in an regular string, escape it with a backslash (`\`). + +> **Example** +> +> ```nix +> "echo \${PATH}" +> ``` +> +> "echo ${PATH}" + +To write a literal `${` in an indented string, escape it with two single quotes (`''`). + +> **Example** +> +> ```nix +> '' +> echo ''${PATH} +> '' +> ``` +> +> "echo ${PATH}\n" + +`$${` can be written literally in any string. + +> **Example** +> +> In Make, `$` in file names or recipes is represented as `$$`, see [GNU `make`: Basics of Variable Reference](https://www.gnu.org/software/make/manual/html_node/Reference.html#Basics-of-Variable-References). +> This can be expressed directly in the Nix language strings: +> +> ```nix +> '' +> MAKEVAR = Hello +> all: +> @export BASHVAR=world; echo $(MAKEVAR) $${BASHVAR} +> '' +> ``` +> +> "MAKEVAR = Hello\nall:\n\t@export BASHVAR=world; echo $(MAKEVAR) $\${BASHVAR}\n" + +See the [documentation on strings][string] for details. + ### Path Rather than writing @@ -107,9 +148,9 @@ An expression that is interpolated must evaluate to one of the following: A string interpolates to itself. -A path in an interpolated expression is first copied into the Nix store, and the resulting string is the [store path] of the newly created [store object](../glossary.md#gloss-store-object). +A path in an interpolated expression is first copied into the Nix store, and the resulting string is the [store path] of the newly created [store object](@docroot@/store/store-object.md). -[store path]: ../glossary.md#gloss-store-path +[store path]: @docroot@/store/store-path.md > **Example** > diff --git a/doc/manual/src/language/syntax.md b/doc/manual/src/language/syntax.md new file mode 100644 index 000000000..6108bacd6 --- /dev/null +++ b/doc/manual/src/language/syntax.md @@ -0,0 +1,866 @@ +# Language Constructs + +This section covers syntax and semantics of the Nix language. + +## Basic Literals + +### String {#string-literal} + + *Strings* can be written in three ways. + + The most common way is to enclose the string between double quotes, e.g., `"foo bar"`. + Strings can span multiple lines. + The results of other expressions can be included into a string by enclosing them in `${ }`, a feature known as [string interpolation]. + + [string interpolation]: ./string-interpolation.md + + The following must be escaped to represent them within a string, by prefixing with a backslash (`\`): + + - Double quote (`"`) + + > **Example** + > + > ```nix + > "\"" + > ``` + > + > "\"" + + - Backslash (`\`) + + > **Example** + > + > ```nix + > "\\" + > ``` + > + > "\\" + + - Dollar sign followed by an opening curly bracket (`${`) – "dollar-curly" + + > **Example** + > + > ```nix + > "\${" + > ``` + > + > "\${" + + The newline, carriage return, and tab characters can be written as `\n`, `\r` and `\t`, respectively. + + A "double-dollar-curly" (`$${`) can be written literally. + + > **Example** + > + > ```nix + > "$${" + > ``` + > + > "$\${" + + String values are output on the terminal with Nix-specific escaping. + Strings written to files will contain the characters encoded by the escaping. + + The second way to write string literals is as an *indented string*, which is enclosed between pairs of *double single-quotes* (`''`), like so: + + ```nix + '' + This is the first line. + This is the second line. + This is the third line. + '' + ``` + + This kind of string literal intelligently strips indentation from + the start of each line. To be precise, it strips from each line a + number of spaces equal to the minimal indentation of the string as a + whole (disregarding the indentation of empty lines). For instance, + the first and second line are indented two spaces, while the third + line is indented four spaces. Thus, two spaces are stripped from + each line, so the resulting string is + + ```nix + "This is the first line.\nThis is the second line.\n This is the third line.\n" + ``` + + > **Note** + > + > Whitespace and newline following the opening `''` is ignored if there is no non-whitespace text on the initial line. + + > **Warning** + > + > Prefixed tab characters are not stripped. + > + > > **Example** + > > + > > The following indented string is prefixed with tabs: + > > + > > '' + > > all: + > > @echo hello + > > '' + > > + > > "\tall:\n\t\t@echo hello\n" + + Indented strings support [string interpolation]. + + The following must be escaped to represent them in an indented string: + + - `$` is escaped by prefixing it with two single quotes (`''`) + + > **Example** + > + > ```nix + > '' + > ''$ + > '' + > ``` + > + > "$\n" + + - `''` is escaped by prefixing it with one single quote (`'`) + + > **Example** + > + > ```nix + > '' + > ''' + > '' + > ``` + > + > "''\n" + + These special characters are escaped as follows: + - Linefeed (`\n`): `''\n` + - Carriage return (`\r`): `''\r` + - Tab (`\t`): `''\t` + + `''\` escapes any other character. + + A "double-dollar-curly" (`$${`) can be written literally. + + > **Example** + > + > ```nix + > '' + > $${ + > '' + > ``` + > + > "$\${\n" + + Indented strings are primarily useful in that they allow multi-line + string literals to follow the indentation of the enclosing Nix + expression, and that less escaping is typically necessary for + strings representing languages such as shell scripts and + configuration files because `''` is much less common than `"`. + Example: + + ```nix + stdenv.mkDerivation { + ... + postInstall = + '' + mkdir $out/bin $out/etc + cp foo $out/bin + echo "Hello World" > $out/etc/foo.conf + ${if enableBar then "cp bar $out/bin" else ""} + ''; + ... + } + ``` + + Finally, as a convenience, *URIs* as defined in appendix B of + [RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as + is*, without quotes. For instance, the string + `"http://example.org/foo.tar.bz2"` can also be written as + `http://example.org/foo.tar.bz2`. + +### Number {#number-literal} + + + + Numbers, which can be *integers* (like `123`) or *floating point* + (like `123.43` or `.27e13`). + + See [arithmetic] and [comparison] operators for semantics. + + [arithmetic]: ./operators.md#arithmetic + [comparison]: ./operators.md#comparison + +### Path {#path-literal} + + *Paths* can be expressed by path literals such as `./builder.sh`. + + A path literal must contain at least one slash to be recognised as such. + For instance, `builder.sh` is not a path: + it's parsed as an expression that selects the attribute `sh` from the variable `builder`. + + Path literals are resolved relative to their [base directory](@docroot@/glossary.md#gloss-base-directory). + Path literals may also refer to absolute paths by starting with a slash. + + > **Note** + > + > Absolute paths make expressions less portable. + > In the case where a function translates a path literal into an absolute path string for a configuration file, it is recommended to write a string literal instead. + > This avoids some confusion about whether files at that location will be used during evaluation. + > It also avoids unintentional situations where some function might try to copy everything at the location into the store. + + If the first component of a path is a `~`, it is interpreted such that the rest of the path were relative to the user's home directory. + For example, `~/foo` would be equivalent to `/home/edolstra/foo` for a user whose home directory is `/home/edolstra`. + Path literals that start with `~` are not allowed in [pure](@docroot@/command-ref/conf-file.md#conf-pure-eval) evaluation. + + Path literals can also include [string interpolation], besides being [interpolated into other expressions]. + + [interpolated into other expressions]: ./string-interpolation.md#interpolated-expressions + + At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path. + + `a.${foo}/b.${bar}` is a syntactically valid number division operation. + `./a.${foo}/b.${bar}` is a path. + + [Lookup path](./constructs/lookup-path.md) literals such as `` also resolve to path values. + +## List {#list-literal} + +Lists are formed by enclosing a whitespace-separated list of values +between square brackets. For example, + +```nix +[ 123 ./foo.nix "abc" (f { x = y; }) ] +``` + +defines a list of four elements, the last being the result of a call to +the function `f`. Note that function calls have to be enclosed in +parentheses. If they had been omitted, e.g., + +```nix +[ 123 ./foo.nix "abc" f { x = y; } ] +``` + +the result would be a list of five elements, the fourth one being a +function and the fifth being a set. + +Note that lists are only lazy in values, and they are strict in length. + +Elements in a list can be accessed using [`builtins.elemAt`](./builtins.md#builtins-elemAt). + +## Attribute Set {#attrs-literal} + +An attribute set is a collection of name-value-pairs called *attributes*. + +Attribute sets are written enclosed in curly brackets (`{ }`). +Attribute names and attribute values are separated by an equal sign (`=`). +Each value can be an arbitrary expression, terminated by a semicolon (`;`) + +An attribute name is a string without context, and is denoted by a [name] (an [identifier](./identifiers.md#identifiers) or [string literal](#string-literal)). + +[name]: ./identifiers.md#names + +> **Syntax** +> +> *attrset* → `{` { *name* `=` *expr* `;` } `}` + +Attributes can appear in any order. +An attribute name may only occur once in each attribute set. + +> **Example** +> +> This defines an attribute set with attributes named: +> - `x` with the value `123`, an integer +> - `text` with the value `"Hello"`, a string +> - `y` where the value is the result of applying the function `f` to the attribute set `{ bla = 456; }` +> +> ```nix +> { +> x = 123; +> text = "Hello"; +> y = f { bla = 456; }; +> } +> ``` + +Attributes in nested attribute sets can be written using *attribute paths*. + +> **Syntax** +> +> *attrset* → `{` { *attrpath* `=` *expr* `;` } `}` + +An attribute path is a dot-separated list of [names][name]. + +> **Syntax** +> +> *attrpath* = *name* { `.` *name* } + + + +> **Example** +> +> ```nix +> { a.b.c = 1; a.b.d = 2; } +> ``` +> +> { +> a = { +> b = { +> c = 1; +> d = 2; +> }; +> }; +> } + +Attribute names can also be set implicitly by using the [`inherit` keyword](#inheriting-attributes). + +> **Example** +> +> ```nix +> { inherit (builtins) true; } +> ``` +> +> { true = true; } + +Attributes can be accessed with the [`.` operator](./operators.md#attribute-selection). + +Example: + +```nix +{ a = "Foo"; b = "Bar"; }.a +``` + +This evaluates to `"Foo"`. + +It is possible to provide a default value in an attribute selection using the `or` keyword. + +Example: + +```nix +{ a = "Foo"; b = "Bar"; }.c or "Xyzzy" +``` + +```nix +{ a = "Foo"; b = "Bar"; }.c.d.e.f.g or "Xyzzy" +``` + +will both evaluate to `"Xyzzy"` because there is no `c` attribute in the set. + +You can use arbitrary double-quoted strings as attribute names: + +```nix +{ "$!@#?" = 123; }."$!@#?" +``` + +```nix +let bar = "bar"; in +{ "foo ${bar}" = 123; }."foo ${bar}" +``` + +Both will evaluate to `123`. + +Attribute names support [string interpolation]: + +```nix +let bar = "foo"; in +{ foo = 123; }.${bar} +``` + +```nix +let bar = "foo"; in +{ ${bar} = 123; }.foo +``` + +Both will evaluate to `123`. + +In the special case where an attribute name inside of a set declaration +evaluates to `null` (which is normally an error, as `null` cannot be coerced to +a string), that attribute is simply not added to the set: + +```nix +{ ${if foo then "bar" else null} = true; } +``` + +This will evaluate to `{}` if `foo` evaluates to `false`. + +A set that has a `__functor` attribute whose value is callable (i.e. is +itself a function or a set with a `__functor` attribute whose value is +callable) can be applied as if it were a function, with the set itself +passed in first , e.g., + +```nix +let add = { __functor = self: x: x + self.x; }; + inc = add // { x = 1; }; +in inc 1 +``` + +evaluates to `2`. This can be used to attach metadata to a function +without the caller needing to treat it specially, or to implement a form +of object-oriented programming, for example. + +## Recursive sets + +Recursive sets are like normal [attribute sets](./types.md#attribute-set), but the attributes can refer to each other. + +> *rec-attrset* = `rec {` [ *name* `=` *expr* `;` `]`... `}` + +Example: + +```nix +rec { + x = y; + y = 123; +}.x +``` + +This evaluates to `123`. + +Note that without `rec` the binding `x = y;` would +refer to the variable `y` in the surrounding scope, if one exists, and +would be invalid if no such variable exists. That is, in a normal +(non-recursive) set, attributes are not added to the lexical scope; in a +recursive set, they are. + +Recursive sets of course introduce the danger of infinite recursion. For +example, the expression + +```nix +rec { + x = y; + y = x; +}.x +``` + +will crash with an `infinite recursion encountered` error message. + +## Let-expressions + +A let-expression allows you to define local variables for an expression. + +> *let-in* = `let` [ *identifier* = *expr* ]... `in` *expr* + +Example: + +```nix +let + x = "foo"; + y = "bar"; +in x + y +``` + +This evaluates to `"foobar"`. + +## Inheriting attributes + +When defining an [attribute set](./types.md#attribute-set) or in a [let-expression](#let-expressions) it is often convenient to copy variables from the surrounding lexical scope (e.g., when you want to propagate attributes). +This can be shortened using the `inherit` keyword. + +Example: + +```nix +let x = 123; in +{ + inherit x; + y = 456; +} +``` + +is equivalent to + +```nix +let x = 123; in +{ + x = x; + y = 456; +} +``` + +and both evaluate to `{ x = 123; y = 456; }`. + +> **Note** +> +> This works because `x` is added to the lexical scope by the `let` construct. + +It is also possible to inherit attributes from another attribute set. + +Example: + +In this fragment from `all-packages.nix`, + +```nix +graphviz = (import ../tools/graphics/graphviz) { + inherit fetchurl stdenv libpng libjpeg expat x11 yacc; + inherit (xorg) libXaw; +}; + +xorg = { + libX11 = ...; + libXaw = ...; + ... +} + +libpng = ...; +libjpg = ...; +... +``` + +the set used in the function call to the function defined in +`../tools/graphics/graphviz` inherits a number of variables from the +surrounding scope (`fetchurl` ... `yacc`), but also inherits `libXaw` +(the X Athena Widgets) from the `xorg` set. + +Summarizing the fragment + +```nix +... +inherit x y z; +inherit (src-set) a b c; +... +``` + +is equivalent to + +```nix +... +x = x; y = y; z = z; +a = src-set.a; b = src-set.b; c = src-set.c; +... +``` + +when used while defining local variables in a let-expression or while +defining a set. + +In a `let` expression, `inherit` can be used to selectively bring specific attributes of a set into scope. For example + + +```nix +let + x = { a = 1; b = 2; }; + inherit (builtins) attrNames; +in +{ + names = attrNames x; +} +``` + +is equivalent to + +```nix +let + x = { a = 1; b = 2; }; +in +{ + names = builtins.attrNames x; +} +``` + +both evaluate to `{ names = [ "a" "b" ]; }`. + +## Functions + +Functions have the following form: + +```nix +pattern: body +``` + +The pattern specifies what the argument of the function must look like, +and binds variables in the body to (parts of) the argument. There are +three kinds of patterns: + + - If a pattern is a single identifier, then the function matches any + argument. Example: + + ```nix + let negate = x: !x; + concat = x: y: x + y; + in if negate true then concat "foo" "bar" else "" + ``` + + Note that `concat` is a function that takes one argument and returns + a function that takes another argument. This allows partial + parameterisation (i.e., only filling some of the arguments of a + function); e.g., + + ```nix + map (concat "foo") [ "bar" "bla" "abc" ] + ``` + + evaluates to `[ "foobar" "foobla" "fooabc" ]`. + + - A *set pattern* of the form `{ name1, name2, …, nameN }` matches a + set containing the listed attributes, and binds the values of those + attributes to variables in the function body. For example, the + function + + ```nix + { x, y, z }: z + y + x + ``` + + can only be called with a set containing exactly the attributes `x`, + `y` and `z`. No other attributes are allowed. If you want to allow + additional arguments, you can use an ellipsis (`...`): + + ```nix + { x, y, z, ... }: z + y + x + ``` + + This works on any set that contains at least the three named + attributes. + + It is possible to provide *default values* for attributes, in + which case they are allowed to be missing. A default value is + specified by writing `name ? e`, where *e* is an arbitrary + expression. For example, + + ```nix + { x, y ? "foo", z ? "bar" }: z + y + x + ``` + + specifies a function that only requires an attribute named `x`, but + optionally accepts `y` and `z`. + + - An `@`-pattern provides a means of referring to the whole value + being matched: + + ```nix + args@{ x, y, z, ... }: z + y + x + args.a + ``` + + but can also be written as: + + ```nix + { x, y, z, ... } @ args: z + y + x + args.a + ``` + + Here `args` is bound to the argument *as passed*, which is further + matched against the pattern `{ x, y, z, ... }`. + The `@`-pattern makes mainly sense with an ellipsis(`...`) as + you can access attribute names as `a`, using `args.a`, which was + given as an additional attribute to the function. + + > **Warning** + > + > `args@` binds the name `args` to the attribute set that is passed to the function. + > In particular, `args` does *not* include any default values specified with `?` in the function's set pattern. + > + > For instance + > + > ```nix + > let + > f = args@{ a ? 23, ... }: [ a args ]; + > in + > f {} + > ``` + > + > is equivalent to + > + > ```nix + > let + > f = args @ { ... }: [ (args.a or 23) args ]; + > in + > f {} + > ``` + > + > and both expressions will evaluate to: + > + > ```nix + > [ 23 {} ] + > ``` + +Note that functions do not have names. If you want to give them a name, +you can bind them to an attribute, e.g., + +```nix +let concat = { x, y }: x + y; +in concat { x = "foo"; y = "bar"; } +``` + +## Conditionals + +Conditionals look like this: + +```nix +if e1 then e2 else e3 +``` + +where *e1* is an expression that should evaluate to a Boolean value +(`true` or `false`). + +## Assertions + +Assertions are generally used to check that certain requirements on or +between features and dependencies hold. They look like this: + +```nix +assert e1; e2 +``` + +where *e1* is an expression that should evaluate to a Boolean value. If +it evaluates to `true`, *e2* is returned; otherwise expression +evaluation is aborted and a backtrace is printed. + +Here is a Nix expression for the Subversion package that shows how +assertions can be used:. + +```nix +{ localServer ? false +, httpServer ? false +, sslSupport ? false +, pythonBindings ? false +, javaSwigBindings ? false +, javahlBindings ? false +, stdenv, fetchurl +, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null +}: + +assert localServer -> db4 != null; â‘ +assert httpServer -> httpd != null && httpd.expat == expat; â‘¡ +assert sslSupport -> openssl != null && (httpServer -> httpd.openssl == openssl); â‘¢ +assert pythonBindings -> swig != null && swig.pythonSupport; +assert javaSwigBindings -> swig != null && swig.javaSupport; +assert javahlBindings -> j2sdk != null; + +stdenv.mkDerivation { + name = "subversion-1.1.1"; + ... + openssl = if sslSupport then openssl else null; â‘£ + ... +} +``` + +The points of interest are: + +1. This assertion states that if Subversion is to have support for + local repositories, then Berkeley DB is needed. So if the Subversion + function is called with the `localServer` argument set to `true` but + the `db4` argument set to `null`, then the evaluation fails. + + Note that `->` is the [logical + implication](https://en.wikipedia.org/wiki/Truth_table#Logical_implication) + Boolean operation. + +2. This is a more subtle condition: if Subversion is built with Apache + (`httpServer`) support, then the Expat library (an XML library) used + by Subversion should be same as the one used by Apache. This is + because in this configuration Subversion code ends up being linked + with Apache code, and if the Expat libraries do not match, a build- + or runtime link error or incompatibility might occur. + +3. This assertion says that in order for Subversion to have SSL support + (so that it can access `https` URLs), an OpenSSL library must be + passed. Additionally, it says that *if* Apache support is enabled, + then Apache's OpenSSL should match Subversion's. (Note that if + Apache support is not enabled, we don't care about Apache's + OpenSSL.) + +4. The conditional here is not really related to assertions, but is + worth pointing out: it ensures that if SSL support is disabled, then + the Subversion derivation is not dependent on OpenSSL, even if a + non-`null` value was passed. This prevents an unnecessary rebuild of + Subversion if OpenSSL changes. + +## With-expressions + +A *with-expression*, + +```nix +with e1; e2 +``` + +introduces the set *e1* into the lexical scope of the expression *e2*. +For instance, + +```nix +let as = { x = "foo"; y = "bar"; }; +in with as; x + y +``` + +evaluates to `"foobar"` since the `with` adds the `x` and `y` attributes +of `as` to the lexical scope in the expression `x + y`. The most common +use of `with` is in conjunction with the `import` function. E.g., + +```nix +with (import ./definitions.nix); ... +``` + +makes all attributes defined in the file `definitions.nix` available as +if they were defined locally in a `let`-expression. + +The bindings introduced by `with` do not shadow bindings introduced by +other means, e.g. + +```nix +let a = 3; in with { a = 1; }; let a = 4; in with { a = 2; }; ... +``` + +establishes the same scope as + +```nix +let a = 1; in let a = 2; in let a = 3; in let a = 4; in ... +``` + +Variables coming from outer `with` expressions *are* shadowed: + +```nix +with { a = "outer"; }; +with { a = "inner"; }; +a +``` + +Does evaluate to `"inner"`. + +## Comments + +- Inline comments start with `#` and run until the end of the line. + + > **Example** + > + > ```nix + > # A number + > 2 # Equals 1 + 1 + > ``` + > + > ```console + > 2 + > ``` + +- Block comments start with `/*` and run until the next occurrence of `*/`. + + > **Example** + > + > ```nix + > /* + > Block comments + > can span multiple lines. + > */ "hello" + > ``` + > + > ```console + > "hello" + > ``` + + This means that block comments cannot be nested. + + > **Example** + > + > ```nix + > /* /* nope */ */ 1 + > ``` + > + > ```console + > error: syntax error, unexpected '*' + > + > at «string»:1:15: + > + > 1| /* /* nope */ * + > | ^ + > ``` + + Consider escaping nested comments and unescaping them in post-processing. + + > **Example** + > + > ```nix + > /* /* nested *\/ */ 1 + > ``` + > + > ```console + > 1 + > ``` diff --git a/doc/manual/src/language/types.md b/doc/manual/src/language/types.md new file mode 100644 index 000000000..229756e6b --- /dev/null +++ b/doc/manual/src/language/types.md @@ -0,0 +1,120 @@ +# Data Types + +Every value in the Nix language has one of the following types: + +* [Integer](#type-int) +* [Float](#type-float) +* [Boolean](#type-bool) +* [String](#type-string) +* [Path](#type-path) +* [Null](#type-null) +* [Attribute set](#type-attrs) +* [List](#type-list) +* [Function](#type-function) +* [External](#type-external) + +## Primitives + +### Integer {#type-int} + +An _integer_ in the Nix language is a signed 64-bit integer. + +Non-negative integers can be expressed as [integer literals](syntax.md#number-literal). +Negative integers are created with the [arithmetic negation operator](./operators.md#arithmetic). +The function [`builtins.isInt`](builtins.md#builtins-isInt) can be used to determine if a value is an integer. + +### Float {#type-float} + +A _float_ in the Nix language is a 64-bit [IEEE 754](https://en.wikipedia.org/wiki/IEEE_754) floating-point number. + +Most non-negative floats can be expressed as [float literals](syntax.md#number-literal). +Negative floats are created with the [arithmetic negation operator](./operators.md#arithmetic). +The function [`builtins.isFloat`](builtins.md#builtins-isFloat) can be used to determine if a value is a float. + +### Boolean {#type-bool} + +A _boolean_ in the Nix language is one of _true_ or _false_. + + + +These values are available as attributes of [`builtins`](builtins.md#builtins-builtins) as [`builtins.true`](builtins.md#builtins-true) and [`builtins.false`](builtins.md#builtins-false). +The function [`builtins.isBool`](builtins.md#builtins-isBool) can be used to determine if a value is a boolean. + +### String {#type-string} + +A _string_ in the Nix language is an immutable, finite-length sequence of bytes, along with a [string context](string-context.md). +Nix does not assume or support working natively with character encodings. + +String values without string context can be expressed as [string literals](syntax.md#string-literal). +The function [`builtins.isString`](builtins.md#builtins-isString) can be used to determine if a value is a string. + +### Path {#type-path} + +A _path_ in the Nix language is an immutable, finite-length sequence of bytes starting with `/`, representing a POSIX-style, canonical file system path. +Path values are distinct from string values, even if they contain the same sequence of bytes. +Operations that produce paths will simplify the result as the standard C function [`realpath`] would, except that there is no symbolic link resolution. + +[`realpath`]: https://pubs.opengroup.org/onlinepubs/9699919799/functions/realpath.html + +Paths are suitable for referring to local files, and are often preferable over strings. +- Path values do not contain trailing or duplicate slashes, `.`, or `..`. +- Relative path literals are automatically resolved relative to their [base directory]. +- Tooling can recognize path literals and provide additional features, such as autocompletion, refactoring automation and jump-to-file. + +[base directory]: @docroot@/glossary.md#gloss-base-directory + +A file is not required to exist at a given path in order for that path value to be valid, but a path that is converted to a string with [string interpolation] or [string-and-path concatenation] must resolve to a readable file or directory which will be copied into the Nix store. +For instance, evaluating `"${./foo.txt}"` will cause `foo.txt` from the same directory to be copied into the Nix store and result in the string `"/nix/store/-foo.txt"`. +Operations such as [`import`] can also expect a path to resolve to a readable file or directory. + +[string interpolation]: string-interpolation.md#interpolated-expression +[string-and-path concatenation]: operators.md#string-and-path-concatenation +[`import`]: builtins.md#builtins-import + +> **Note** +> +> The Nix language assumes that all input files will remain _unchanged_ while evaluating a Nix expression. +> For example, assume you used a file path in an interpolated string during a `nix repl` session. +> Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new [store path], since Nix might not re-read the file contents. +> Use `:r` to reset the repl as needed. + +[store path]: @docroot@/store/store-path.md + +Path values can be expressed as [path literals](syntax.md#path-literal). +The function [`builtins.isPath`](builtins.md#builtins-isPath) can be used to determine if a value is a path. + +### Null {#type-null} + +There is a single value of type _null_ in the Nix language. + + + +This value is available as an attribute on the [`builtins`](builtins.md#builtins-builtins) attribute set as [`builtins.null`](builtins.md#builtins-null). + +## Compound values + +### Attribute set {#type-attrs} + + + +An attribute set can be constructed with an [attribute set literal](syntax.md#attrs-literal). +The function [`builtins.isAttrs`](builtins.md#builtins-isAttrs) can be used to determine if a value is an attribute set. + +### List {#type-list} + + + +A list can be constructed with a [list literal](syntax.md#list-literal). +The function [`builtins.isList`](builtins.md#builtins-isList) can be used to determine if a value is a list. + +## Function {#type-function} + + + +A function can be constructed with a [function expression](syntax.md#functions). +The function [`builtins.isFunction`](builtins.md#builtins-isFunction) can be used to determine if a value is a function. + +## External {#type-external} + +An _external_ value is an opaque value created by a Nix [plugin](../command-ref/conf-file.md#conf-plugin-files). +Such a value can be substituted in Nix expressions but only created and used by plugin code. diff --git a/doc/manual/src/language/values.md b/doc/manual/src/language/values.md deleted file mode 100644 index 74ffc7070..000000000 --- a/doc/manual/src/language/values.md +++ /dev/null @@ -1,269 +0,0 @@ -# Data Types - -## Primitives - -- String - - *Strings* can be written in three ways. - - The most common way is to enclose the string between double quotes, - e.g., `"foo bar"`. Strings can span multiple lines. The special - characters `"` and `\` and the character sequence `${` must be - escaped by prefixing them with a backslash (`\`). Newlines, carriage - returns and tabs can be written as `\n`, `\r` and `\t`, - respectively. - - You can include the results of other expressions into a string by enclosing them in `${ }`, a feature known as [string interpolation]. - - [string interpolation]: ./string-interpolation.md - - The second way to write string literals is as an *indented string*, - which is enclosed between pairs of *double single-quotes*, like so: - - ```nix - '' - This is the first line. - This is the second line. - This is the third line. - '' - ``` - - This kind of string literal intelligently strips indentation from - the start of each line. To be precise, it strips from each line a - number of spaces equal to the minimal indentation of the string as a - whole (disregarding the indentation of empty lines). For instance, - the first and second line are indented two spaces, while the third - line is indented four spaces. Thus, two spaces are stripped from - each line, so the resulting string is - - ```nix - "This is the first line.\nThis is the second line.\n This is the third line.\n" - ``` - - Note that the whitespace and newline following the opening `''` is - ignored if there is no non-whitespace text on the initial line. - - Indented strings support [string interpolation]. - - Since `${` and `''` have special meaning in indented strings, you - need a way to quote them. `$` can be escaped by prefixing it with - `''` (that is, two single quotes), i.e., `''$`. `''` can be escaped - by prefixing it with `'`, i.e., `'''`. `$` removes any special - meaning from the following `$`. Linefeed, carriage-return and tab - characters can be written as `''\n`, `''\r`, `''\t`, and `''\` - escapes any other character. - - Indented strings are primarily useful in that they allow multi-line - string literals to follow the indentation of the enclosing Nix - expression, and that less escaping is typically necessary for - strings representing languages such as shell scripts and - configuration files because `''` is much less common than `"`. - Example: - - ```nix - stdenv.mkDerivation { - ... - postInstall = - '' - mkdir $out/bin $out/etc - cp foo $out/bin - echo "Hello World" > $out/etc/foo.conf - ${if enableBar then "cp bar $out/bin" else ""} - ''; - ... - } - ``` - - Finally, as a convenience, *URIs* as defined in appendix B of - [RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as - is*, without quotes. For instance, the string - `"http://example.org/foo.tar.bz2"` can also be written as - `http://example.org/foo.tar.bz2`. - -- Number - - Numbers, which can be *integers* (like `123`) or *floating point* - (like `123.43` or `.27e13`). - - See [arithmetic] and [comparison] operators for semantics. - - [arithmetic]: ./operators.md#arithmetic - [comparison]: ./operators.md#comparison - -- Path - - *Paths*, e.g., `/bin/sh` or `./builder.sh`. A path must contain at - least one slash to be recognised as such. For instance, `builder.sh` - is not a path: it's parsed as an expression that selects the - attribute `sh` from the variable `builder`. If the file name is - relative, i.e., if it does not begin with a slash, it is made - absolute at parse time relative to the directory of the Nix - expression that contained it. For instance, if a Nix expression in - `/foo/bar/bla.nix` refers to `../xyzzy/fnord.nix`, the absolute path - is `/foo/xyzzy/fnord.nix`. - - If the first component of a path is a `~`, it is interpreted as if - the rest of the path were relative to the user's home directory. - e.g. `~/foo` would be equivalent to `/home/edolstra/foo` for a user - whose home directory is `/home/edolstra`. - - For instance, evaluating `"${./foo.txt}"` will cause `foo.txt` in the current directory to be copied into the Nix store and result in the string `"/nix/store/-foo.txt"`. - - Note that the Nix language assumes that all input files will remain _unchanged_ while evaluating a Nix expression. - For example, assume you used a file path in an interpolated string during a `nix repl` session. - Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new [store path], since Nix might not re-read the file contents. - - [store path]: ../glossary.md#gloss-store-path - - Paths can include [string interpolation] and can themselves be [interpolated in other expressions]. - - [interpolated in other expressions]: ./string-interpolation.md#interpolated-expressions - - At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path. - - `a.${foo}/b.${bar}` is a syntactically valid division operation. - `./a.${foo}/b.${bar}` is a path. - - [Lookup paths](./constructs/lookup-path.md) such as `` resolve to path values. - -- Boolean - - *Booleans* with values `true` and `false`. - -- Null - - The null value, denoted as `null`. - -## List - -Lists are formed by enclosing a whitespace-separated list of values -between square brackets. For example, - -```nix -[ 123 ./foo.nix "abc" (f { x = y; }) ] -``` - -defines a list of four elements, the last being the result of a call to -the function `f`. Note that function calls have to be enclosed in -parentheses. If they had been omitted, e.g., - -```nix -[ 123 ./foo.nix "abc" f { x = y; } ] -``` - -the result would be a list of five elements, the fourth one being a -function and the fifth being a set. - -Note that lists are only lazy in values, and they are strict in length. - -Elements in a list can be accessed using [`builtins.elemAt`](./builtins.md#builtins-elemAt). - -## Attribute Set - -An attribute set is a collection of name-value-pairs (called *attributes*) enclosed in curly brackets (`{ }`). - -An attribute name can be an identifier or a [string](#string). -An identifier must start with a letter (`a-z`, `A-Z`) or underscore (`_`), and can otherwise contain letters (`a-z`, `A-Z`), numbers (`0-9`), underscores (`_`), apostrophes (`'`), or dashes (`-`). - -> **Syntax** -> -> *name* = *identifier* | *string* \ -> *identifier* ~ `[a-zA-Z_][a-zA-Z0-9_'-]*` - -Names and values are separated by an equal sign (`=`). -Each value is an arbitrary expression terminated by a semicolon (`;`). - -> **Syntax** -> -> *attrset* = `{` [ *name* `=` *expr* `;` ]... `}` - -Attributes can appear in any order. -An attribute name may only occur once. - -Example: - -```nix -{ - x = 123; - text = "Hello"; - y = f { bla = 456; }; -} -``` - -This defines a set with attributes named `x`, `text`, `y`. - -Attributes can be accessed with the [`.` operator](./operators.md#attribute-selection). - -Example: - -```nix -{ a = "Foo"; b = "Bar"; }.a -``` - -This evaluates to `"Foo"`. - -It is possible to provide a default value in an attribute selection using the `or` keyword. - -Example: - -```nix -{ a = "Foo"; b = "Bar"; }.c or "Xyzzy" -``` - -```nix -{ a = "Foo"; b = "Bar"; }.c.d.e.f.g or "Xyzzy" -``` - -will both evaluate to `"Xyzzy"` because there is no `c` attribute in the set. - -You can use arbitrary double-quoted strings as attribute names: - -```nix -{ "$!@#?" = 123; }."$!@#?" -``` - -```nix -let bar = "bar"; in -{ "foo ${bar}" = 123; }."foo ${bar}" -``` - -Both will evaluate to `123`. - -Attribute names support [string interpolation]: - -```nix -let bar = "foo"; in -{ foo = 123; }.${bar} -``` - -```nix -let bar = "foo"; in -{ ${bar} = 123; }.foo -``` - -Both will evaluate to `123`. - -In the special case where an attribute name inside of a set declaration -evaluates to `null` (which is normally an error, as `null` cannot be coerced to -a string), that attribute is simply not added to the set: - -```nix -{ ${if foo then "bar" else null} = true; } -``` - -This will evaluate to `{}` if `foo` evaluates to `false`. - -A set that has a `__functor` attribute whose value is callable (i.e. is -itself a function or a set with a `__functor` attribute whose value is -callable) can be applied as if it were a function, with the set itself -passed in first , e.g., - -```nix -let add = { __functor = self: x: x + self.x; }; - inc = add // { x = 1; }; -in inc 1 -``` - -evaluates to `2`. This can be used to attach metadata to a function -without the caller needing to treat it specially, or to implement a form -of object-oriented programming, for example. diff --git a/doc/manual/src/language/variables.md b/doc/manual/src/language/variables.md new file mode 100644 index 000000000..af6aff8a2 --- /dev/null +++ b/doc/manual/src/language/variables.md @@ -0,0 +1,10 @@ +# Variables + +A *variable* is an [identifier](identifiers.md) used as an expression. + +> **Syntax** +> +> *expression* → *identifier* + +A variable must have the same name as a definition in the [scope](./scope.md) that encloses it. +The value of a variable is the value of the corresponding expression in the enclosing scope. diff --git a/doc/manual/src/package-management/copy-closure.md b/doc/manual/src/package-management/copy-closure.md deleted file mode 100644 index 14326298b..000000000 --- a/doc/manual/src/package-management/copy-closure.md +++ /dev/null @@ -1,34 +0,0 @@ -# Copying Closures via SSH - -The command `nix-copy-closure` copies a Nix store path along with all -its dependencies to or from another machine via the SSH protocol. It -doesn’t copy store paths that are already present on the target machine. -For example, the following command copies Firefox with all its -dependencies: - - $ nix-copy-closure --to alice@itchy.example.org $(type -p firefox) - -See the [manpage for `nix-copy-closure`](../command-ref/nix-copy-closure.md) for details. - -With `nix-store ---export` and `nix-store --import` you can write the closure of a store -path (that is, the path and all its dependencies) to a file, and then -unpack that file into another Nix store. For example, - - $ nix-store --export $(nix-store --query --requisites $(type -p firefox)) > firefox.closure - -writes the closure of Firefox to a file. You can then copy this file to -another machine and install the closure: - - $ nix-store --import < firefox.closure - -Any store paths in the closure that are already present in the target -store are ignored. It is also possible to pipe the export into another -command, e.g. to copy and install a closure directly to/on another -machine: - - $ nix-store --export $(nix-store --query --requisites $(type -p firefox)) | bzip2 | \ - ssh alice@itchy.example.org "bunzip2 | nix-store --import" - -However, `nix-copy-closure` is generally more efficient because it only -copies paths that are not already present in the target Nix store. diff --git a/doc/manual/src/protocols/derivation-aterm.md b/doc/manual/src/protocols/derivation-aterm.md index e58b602a3..1ba757ae0 100644 --- a/doc/manual/src/protocols/derivation-aterm.md +++ b/doc/manual/src/protocols/derivation-aterm.md @@ -14,6 +14,6 @@ Derivations are serialised in one of the following formats: DrvWithVersion(, ...) ``` - The only `version-string`s that are in use today are for [experimental features](@docroot@/contributing/experimental-features.md): + The only `version-string`s that are in use today are for [experimental features](@docroot@/development/experimental-features.md): - - `"xp-dyn-drv"` for the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. + - `"xp-dyn-drv"` for the [`dynamic-derivations`](@docroot@/development/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. diff --git a/doc/manual/src/protocols/json/derivation.md b/doc/manual/src/protocols/json/derivation.md index 649d543cc..2f85340d6 100644 --- a/doc/manual/src/protocols/json/derivation.md +++ b/doc/manual/src/protocols/json/derivation.md @@ -3,7 +3,7 @@ > **Warning** > > This JSON format is currently -> [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) +> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) > and subject to change. The JSON serialization of a @@ -18,10 +18,30 @@ is a JSON object with the following fields: Information about the output paths of the derivation. This is a JSON object with one member per output, where the key is the output name and the value is a JSON object with these fields: - * `path`: The output path. + * `path`: + The output path, if it is known in advanced. + Otherwise, `null`. + + + * `method`: + For an output which will be [content addresed], a string representing the [method](@docroot@/store/store-object/content-address.md) of content addressing that is chosen. + Valid method strings are: + + - [`flat`](@docroot@/store/store-object/content-address.md#method-flat) + - [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive) + - [`text`](@docroot@/store/store-object/content-address.md#method-text) + - [`git`](@docroot@/store/store-object/content-address.md#method-git) + + Otherwise, `null`. * `hashAlgo`: - For fixed-output derivations, the hashing algorithm (e.g. `sha256`), optionally prefixed by `r:` if `hash` denotes a NAR hash rather than a flat file hash. + For an output which will be [content addresed], the name of the hash algorithm used. + Valid algorithm strings are: + + - `md5` + - `sha1` + - `sha256` + - `sha512` * `hash`: For fixed-output derivations, the expected content hash in base-16. @@ -32,7 +52,8 @@ is a JSON object with the following fields: > "outputs": { > "out": { > "path": "/nix/store/2543j7c6jn75blc3drf4g5vhb1rhdq29-source", - > "hashAlgo": "r:sha256", + > "method": "nar", + > "hashAlgo": "sha256", > "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62" > } > } diff --git a/doc/manual/src/protocols/json/store-object-info.md b/doc/manual/src/protocols/json/store-object-info.md index ba4ab098f..6b4f48437 100644 --- a/doc/manual/src/protocols/json/store-object-info.md +++ b/doc/manual/src/protocols/json/store-object-info.md @@ -3,7 +3,7 @@ > **Warning** > > This JSON format is currently -> [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) +> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) > and subject to change. Info about a [store object]. @@ -24,41 +24,45 @@ Info about a [store object]. An array of [store paths][store path], possibly including this one. -* `ca` (optional): +* `ca`: - Content address of this store object's file system object, used to compute its store path. + If the store object is [content-addressed], + this is the content address of this store object's file system object, used to compute its store path. + Otherwise (i.e. if it is [input-addressed]), this is `null`. -[store path]: @docroot@/glossary.md#gloss-store-path +[store path]: @docroot@/store/store-path.md [file system object]: @docroot@/store/file-system-object.md -[Nix Archive]: @docroot@/glossary.md#gloss-nar +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive ## Impure fields These are not intrinsic properties of the store object. In other words, the same store object residing in different store could have different values for these properties. -* `deriver` (optional): +* `deriver`: - The path to the [derivation] from which this store object is produced. + If known, the path to the [derivation] from which this store object was produced. + Otherwise `null`. [derivation]: @docroot@/glossary.md#gloss-store-derivation * `registrationTime` (optional): - When this derivation was added to the store. + If known, when this derivation was added to the store. + Otherwise `null`. -* `ultimate` (optional): +* `ultimate`: Whether this store object is trusted because we built it ourselves, rather than substituted a build product from elsewhere. -* `signatures` (optional): +* `signatures`: Signatures claiming that this store object is what it claims to be. Not relevant for [content-addressed] store objects, but useful for [input-addressed] store objects. - [content-addressed]: @docroot@/glossary.md#gloss-content-addressed-store-object - [input-addressed]: @docroot@/glossary.md#gloss-input-addressed-store-object +[content-addressed]: @docroot@/store/store-object/content-address.md +[input-addressed]: @docroot@/glossary.md#gloss-input-addressed-store-object ### `.narinfo` extra fields @@ -83,7 +87,7 @@ This information is not intrinsic to the store object, but about how it is store ## Computed closure fields -These fields are not stored at all, but computed by traverising the other other fields across all the store objects in a [closure]. +These fields are not stored at all, but computed by traversing the other fields across all the store objects in a [closure]. * `closureSize`: diff --git a/doc/manual/src/protocols/nix-archive.md b/doc/manual/src/protocols/nix-archive.md new file mode 100644 index 000000000..640b527f1 --- /dev/null +++ b/doc/manual/src/protocols/nix-archive.md @@ -0,0 +1,43 @@ +# Nix Archive (NAR) format + +This is the complete specification of the [Nix Archive] format. +The Nix Archive format closely follows the abstract specification of a [file system object] tree, +because it is designed to serialize exactly that data structure. + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#nix-archive +[file system object]: @docroot@/store/file-system-object.md + +The format of this specification is close to [Extended Backus–Naur form](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form), with the exception of the `str(..)` function / parameterized rule, which length-prefixes and pads strings. +This makes the resulting binary format easier to parse. + +Regular users do *not* need to know this information. +But for those interested in exactly how Nix works, e.g. if they are reimplementing it, this information can be useful. + +```ebnf +nar = str("nix-archive-1"), nar-obj; + +nar-obj = str("("), nar-obj-inner, str(")"); + +nar-obj-inner + = str("type"), str("regular") regular + | str("type"), str("symlink") symlink + | str("type"), str("directory") directory + ; + +regular = [ str("executable"), str("") ], str("contents"), str(contents); + +symlink = str("target"), str(target); + +(* side condition: directory entries must be ordered by their names *) +directory = { directory-entry }; + +directory-entry = str("entry"), str("("), str("name"), str(name), str("node"), nar-obj, str(")"); +``` + +The `str` function / parameterized rule is defined as follows: + +- `str(s)` = `int(|s|), pad(s);` + +- `int(n)` = the 64-bit little endian representation of the number `n` + +- `pad(s)` = the byte sequence `s`, padded with 0s to a multiple of 8 byte diff --git a/doc/manual/src/protocols/store-path.md b/doc/manual/src/protocols/store-path.md index 565c4fa75..52352d358 100644 --- a/doc/manual/src/protocols/store-path.md +++ b/doc/manual/src/protocols/store-path.md @@ -1,12 +1,14 @@ # Complete Store Path Calculation -This is the complete specification for how store paths are calculated. +This is the complete specification for how [store path]s are calculated. The format of this specification is close to [Extended Backus–Naur form](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form), but must deviate for a few things such as hash functions which we treat as bidirectional for specification purposes. Regular users do *not* need to know this information --- store paths can be treated as black boxes computed from the properties of the store objects they refer to. But for those interested in exactly how Nix works, e.g. if they are reimplementing it, this information can be useful. +[store path](@docroot@/store/store-path.md) + ## Store path proper ```ebnf @@ -34,18 +36,23 @@ where - `type` = one of: - ```ebnf - | "text" ( ":" store-path )* + | "text" { ":" store-path } ``` - for encoded derivations written to the store. + This is for the + ["Text"](@docroot@/store/store-object/content-address.md#method-text) + method of content addressing store objects. The optional trailing store paths are the references of the store object. - ```ebnf - | "source" ( ":" store-path )* + | "source" { ":" store-path } [ ":self" ] ``` - For paths copied to the store and hashed via a [Nix Archive (NAR)] and [SHA-256][sha-256]. - Just like in the text case, we can have the store objects referenced by their paths. + This is for the + ["Nix Archive"](@docroot@/store/store-object/content-address.md#method-nix-archive) + method of content addressing store objects, + if the hash algorithm is [SHA-256]. + Just like in the "Text" case, we can have the store objects referenced by their paths. Additionally, we can have an optional `:self` label to denote self reference. - ```ebnf @@ -53,8 +60,12 @@ where ``` For either the outputs built from derivations, - paths copied to the store hashed that area single file hashed directly, or the via a hash algorithm other than [SHA-256][sha-256]. - (in that case "source" is used; this is only necessary for compatibility). + or content-addressed store objects that are not using one of the two above cases. + To be explicit about the latter, that is currently these methods: + + - ["Flat"](@docroot@/store/store-object/content-address.md#method-flat) + - ["Git"](@docroot@/store/store-object/content-address.md#method-git) + - ["Nix Archive"](@docroot@/store/store-object/content-address.md#method-nix-archive) if the hash algorithm is not [SHA-256]. `id` is the name of the output (usually, "out"). For content-addressed store objects, `id`, is always "out". @@ -113,8 +124,8 @@ where Note that `id` = `"out"`, regardless of the name part of the store path. Also note that NAR + SHA-256 must not use this case, and instead must use the `type` = `"source:" ...` case. -[Nix Archive (NAR)]: @docroot@/glossary.md#gloss-NAR -[sha-256]: https://en.m.wikipedia.org/wiki/SHA-256 +[Nix Archive (NAR)]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive +[SHA-256]: https://en.m.wikipedia.org/wiki/SHA-256 ### Historical Note diff --git a/doc/manual/src/protocols/tarball-fetcher.md b/doc/manual/src/protocols/tarball-fetcher.md index 274fa6d63..5cff05d66 100644 --- a/doc/manual/src/protocols/tarball-fetcher.md +++ b/doc/manual/src/protocols/tarball-fetcher.md @@ -22,7 +22,7 @@ Link: ; rel="immutable" *flakeref* must be a tarball flakeref. It can contain the tarball flake attributes `narHash`, `rev`, `revCount` and `lastModified`. If `narHash` is included, its -value must be the NAR hash of the unpacked tarball (as computed via +value must be the [NAR hash][Nix Archive] of the unpacked tarball (as computed via `nix hash path`). Nix checks the contents of the returned tarball against the `narHash` attribute. The `rev` and `revCount` attributes are useful when the tarball flake is a mirror of a fetcher type that @@ -40,3 +40,31 @@ Link: ///archive/.tar.gz +``` + +> **Example** +> +> +> ```nix +> # flake.nix +> { +> inputs = { +> foo.url = "https://gitea.example.org/some-person/some-flake/archive/main.tar.gz"; +> bar.url = "https://gitea.example.org/some-other-person/other-flake/archive/442793d9ec0584f6a6e82fa253850c8085bb150a.tar.gz"; +> qux = { +> url = "https://forgejo.example.org/another-person/some-non-flake-repo/archive/development.tar.gz"; +> flake = false; +> }; +> }; +> outputs = { foo, bar, qux }: { /* ... */ }; +> } +``` + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive diff --git a/doc/manual/src/quick-start.md b/doc/manual/src/quick-start.md index 75853ced7..9eb7a3265 100644 --- a/doc/manual/src/quick-start.md +++ b/doc/manual/src/quick-start.md @@ -34,7 +34,7 @@ For more in-depth information you are kindly referred to subsequent chapters. lolcat: command not found ``` -1. Search for more packages on to try them out. +1. Search for more packages on [search.nixos.org](https://search.nixos.org/) to try them out. 1. Free up storage space: diff --git a/doc/manual/src/release-notes/index.md b/doc/manual/src/release-notes/index.md index cc805e631..d4e6292a6 100644 --- a/doc/manual/src/release-notes/index.md +++ b/doc/manual/src/release-notes/index.md @@ -1,12 +1,13 @@ # Nix Release Notes +The Nix release cycle is calendar-based as follows: + Nix has a release cycle of roughly 6 weeks. Notable changes and additions are announced in the release notes for each version. -Bugfixes can be backported on request to previous Nix releases. -We typically backport only as far back as the Nix version used in the latest NixOS release, which is announced in the [NixOS release notes](https://nixos.org/manual/nixos/stable/release-notes.html#ch-release-notes). - -Backports never skip releases. -If a feature is backported to version `x.y`, it must also be available in version `x.(y+1)`. -This ensures that upgrading from an older version with backports is still safe and no backported functionality will go missing. +The supported Nix versions are: +- The latest release +- The version used in the stable NixOS release, which is announced in the [NixOS release notes](https://nixos.org/manual/nixos/stable/release-notes.html#ch-release-notes). +Bugfixes and security issues are backported to every supported version. +Patch releases are published as needed. diff --git a/doc/manual/src/release-notes/rl-2.15.md b/doc/manual/src/release-notes/rl-2.15.md index 133121999..e7e52631b 100644 --- a/doc/manual/src/release-notes/rl-2.15.md +++ b/doc/manual/src/release-notes/rl-2.15.md @@ -11,7 +11,7 @@ As the choice of hash formats is no longer binary, the `--base16` flag is also added to explicitly specify the Base16 format, which is still the default. -* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](../glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents. +* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](@docroot@/glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents. The new `^` syntax for store paths introduced in Nix 2.13 allows explicitly referencing output paths of a derivation. Using this is better and more clear than relying on the now-removed `.drv` special handling. diff --git a/doc/manual/src/release-notes/rl-2.18.md b/doc/manual/src/release-notes/rl-2.18.md index 4bbc52b50..eb26fc9e7 100644 --- a/doc/manual/src/release-notes/rl-2.18.md +++ b/doc/manual/src/release-notes/rl-2.18.md @@ -13,7 +13,7 @@ - The `discard-references` feature has been stabilized. This means that the - [unsafeDiscardReferences](@docroot@/contributing/experimental-features.md#xp-feature-discard-references) + [unsafeDiscardReferences](@docroot@/development/experimental-features.md#xp-feature-discard-references) attribute is no longer guarded by an experimental flag and can be used freely. @@ -21,7 +21,7 @@ This only affects `nix-build --json` when "building" non-derivation things like fetched sources, which is a no-op. - A new builtin [`outputOf`](@docroot@/language/builtins.md#builtins-outputOf) has been added. - It is part of the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. + It is part of the [`dynamic-derivations`](@docroot@/development/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. - Flake follow paths at depths greater than 2 are now handled correctly, preventing "follows a non-existent input" errors. diff --git a/doc/manual/src/release-notes/rl-2.19.md b/doc/manual/src/release-notes/rl-2.19.md index ba6eb9c64..e2e2f85cc 100644 --- a/doc/manual/src/release-notes/rl-2.19.md +++ b/doc/manual/src/release-notes/rl-2.19.md @@ -17,8 +17,8 @@ - `nix-shell` shebang lines now support single-quoted arguments. -- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). - This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). +- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/development/experimental-features.md#xp-fetch-tree). + This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/development/experimental-features.md#xp-fetch-tree). - The interface for creating and updating lock files has been overhauled: @@ -33,7 +33,7 @@ - The flake-specific flags `--recreate-lock-file` and `--update-input` have been removed from all commands operating on installables. They are superceded by `nix flake update`. -- Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). +- Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches). - [`nix path-info --json`](@docroot@/command-ref/new-cli/nix3-path-info.md) (experimental) now returns a JSON map rather than JSON list. diff --git a/doc/manual/src/release-notes/rl-2.20.md b/doc/manual/src/release-notes/rl-2.20.md index 8ede168a4..eb724f600 100644 --- a/doc/manual/src/release-notes/rl-2.20.md +++ b/doc/manual/src/release-notes/rl-2.20.md @@ -200,3 +200,9 @@ while performing various operations (including `nix develop`, `nix flake update`, and so on). With several fixes to Nix's signal handlers, Nix commands will now exit quickly after Ctrl-C is pressed. + +- `nix copy` to a `ssh-ng` store now needs `--substitute-on-destination` (a.k.a. `-s`) + in order to substitute paths on the remote store instead of copying them. + The behavior is consistent with `nix copy` to a different kind of remote store. + Previously this behavior was controlled by the + `builders-use-substitutes` setting and `--substitute-on-destination` was ignored. diff --git a/doc/manual/src/release-notes/rl-2.21.md b/doc/manual/src/release-notes/rl-2.21.md new file mode 100644 index 000000000..75114f117 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.21.md @@ -0,0 +1,302 @@ +# Release 2.21.0 (2024-03-11) + +- Fix a fixed-output derivation sandbox escape (CVE-2024-27297) + + Cooperating Nix derivations could send file descriptors to files in the Nix + store to each other via Unix domain sockets in the abstract namespace. This + allowed one derivation to modify the output of the other derivation, after Nix + has registered the path as "valid" and immutable in the Nix database. + In particular, this allowed the output of fixed-output derivations to be + modified from their expected content. + + This isn't the case any more. + +- CLI options `--arg-from-file` and `--arg-from-stdin` [#10122](https://github.com/NixOS/nix/pull/10122) + + The new CLI option `--arg-from-file` *name* *path* passes the contents + of file *path* as a string value via the function argument *name* to a + Nix expression. Similarly, the new option `--arg-from-stdin` *name* + reads the contents of the string from standard input. + +- Concise error printing in `nix repl` [#9928](https://github.com/NixOS/nix/pull/9928) + + Previously, if an element of a list or attribute set threw an error while + evaluating, `nix repl` would print the entire error (including source location + information) inline. This output was clumsy and difficult to parse: + + ``` + nix-repl> { err = builtins.throw "uh oh!"; } + { err = «error: + … while calling the 'throw' builtin + at «string»:1:9: + 1| { err = builtins.throw "uh oh!"; } + | ^ + + error: uh oh!»; } + ``` + + Now, only the error message is displayed, making the output much more readable. + ``` + nix-repl> { err = builtins.throw "uh oh!"; } + { err = «error: uh oh!»; } + ``` + + However, if the whole expression being evaluated throws an error, source + locations and (if applicable) a stack trace are printed, just like you'd expect: + + ``` + nix-repl> builtins.throw "uh oh!" + error: + … while calling the 'throw' builtin + at «string»:1:1: + 1| builtins.throw "uh oh!" + | ^ + + error: uh oh! + ``` + +- `--debugger` can now access bindings from `let` expressions [#8827](https://github.com/NixOS/nix/issues/8827) [#9918](https://github.com/NixOS/nix/pull/9918) + + Breakpoints and errors in the bindings of a `let` expression can now access + those bindings in the debugger. Previously, only the body of `let` expressions + could access those bindings. + +- Enter the `--debugger` when `builtins.trace` is called if `debugger-on-trace` is set [#9914](https://github.com/NixOS/nix/pull/9914) + + If the `debugger-on-trace` option is set and `--debugger` is given, + `builtins.trace` calls will behave similarly to `builtins.break` and will enter + the debug REPL. This is useful for determining where warnings are being emitted + from. + +- Debugger prints source position information [#9913](https://github.com/NixOS/nix/pull/9913) + + The `--debugger` now prints source location information, instead of the + pointers of source location information. Before: + + ``` + nix-repl> :bt + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + 0x600001522598 + ``` + + After: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + /nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27 + + 131| + 132| bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs; + | ^ + 133| in + ``` + +- The `--debugger` will start more reliably in `let` expressions and function calls [#6649](https://github.com/NixOS/nix/issues/6649) [#9917](https://github.com/NixOS/nix/pull/9917) + + Previously, if you attempted to evaluate this file with the debugger: + + ```nix + let + a = builtins.trace "before inner break" ( + builtins.break "hello" + ); + b = builtins.trace "before outer break" ( + builtins.break a + ); + in + b + ``` + + Nix would correctly enter the debugger at `builtins.break a`, but if you asked + it to `:continue`, it would skip over the `builtins.break "hello"` expression + entirely. + + Now, Nix will correctly enter the debugger at both breakpoints. + +- Nested debuggers are no longer supported [#9920](https://github.com/NixOS/nix/pull/9920) + + Previously, evaluating an expression that throws an error in the debugger would + enter a second, nested debugger: + + ``` + nix-repl> builtins.throw "what" + error: what + + + Starting REPL to allow you to inspect the current state of the evaluator. + + Welcome to Nix 2.18.1. Type :? for help. + + nix-repl> + ``` + + Now, it just prints the error message like `nix repl`: + + ``` + nix-repl> builtins.throw "what" + error: + … while calling the 'throw' builtin + at «string»:1:1: + 1| builtins.throw "what" + | ^ + + error: what + ``` + +- Consistent order of function arguments in printed expressions [#9874](https://github.com/NixOS/nix/pull/9874) + + Function arguments are now printed in lexicographic order rather than the internal, creation-time based symbol order. + +- Fix duplicate attribute error positions for `inherit` [#9874](https://github.com/NixOS/nix/pull/9874) + + When an `inherit` caused a duplicate attribute error the position of the error was not reported correctly, placing the error with the inherit itself or at the start of the bindings block instead of the offending attribute name. + +- `inherit (x) ...` evaluates `x` only once [#9847](https://github.com/NixOS/nix/pull/9847) + + `inherit (x) a b ...` now evaluates the expression `x` only once for all inherited attributes rather than once for each inherited attribute. + This does not usually have a measurable impact, but side-effects (such as `builtins.trace`) would be duplicated and expensive expressions (such as derivations) could cause a measurable slowdown. + +- Store paths are allowed to start with `.` [#912](https://github.com/NixOS/nix/issues/912) [#9091](https://github.com/NixOS/nix/pull/9091) [#9095](https://github.com/NixOS/nix/pull/9095) [#9120](https://github.com/NixOS/nix/pull/9120) [#9121](https://github.com/NixOS/nix/pull/9121) [#9122](https://github.com/NixOS/nix/pull/9122) [#9130](https://github.com/NixOS/nix/pull/9130) [#9219](https://github.com/NixOS/nix/pull/9219) [#9224](https://github.com/NixOS/nix/pull/9224) [#9867](https://github.com/NixOS/nix/pull/9867) + + Leading periods were allowed by accident in Nix 2.4. The Nix team has considered this to be a bug, but this behavior has since been relied on by users, leading to unnecessary difficulties. + From now on, leading periods are supported. The names `.` and `..` are disallowed, as well as those starting with `.-` or `..-`. + + Nix versions that denied leading periods are documented [in the issue](https://github.com/NixOS/nix/issues/912#issuecomment-1919583286). + +- `nix repl` pretty-prints values [#9931](https://github.com/NixOS/nix/pull/9931) + + `nix repl` will now pretty-print values: + + ``` + { + attrs = { + a = { + b = { + c = { }; + }; + }; + }; + list = [ 1 ]; + list' = [ + 1 + 2 + 3 + ]; + } + ``` + +- Introduction of `--regex` and `--all` in `nix profile remove` and `nix profile upgrade` [#10166](https://github.com/NixOS/nix/pull/10166) + + Previously the command-line arguments for `nix profile remove` and `nix profile upgrade` matched the package entries using regular expression. + For instance: + + ``` + nix profile remove '.*vim.*' + ``` + + This would remove all packages that contain `vim` in their name. + + In most cases, only singular package names were used to remove and upgrade packages. Mixing this with regular expressions sometimes lead to unintended behavior. For instance, `python3.1` could match `python311`. + + To avoid unintended behavior, the arguments are now only matching exact names. + + Matching using regular expressions is still possible by using the new `--regex` flag: + + ``` + nix profile remove --regex '.*vim.*' + ``` + + One of the most useful cases for using regular expressions was to upgrade all packages. This was previously accomplished by: + + ``` + nix profile upgrade '.*' + ``` + + With the introduction of the `--all` flag, this now becomes more straightforward: + + ``` + nix profile upgrade --all + ``` + +- Visual clutter in `--debugger` is reduced [#9919](https://github.com/NixOS/nix/pull/9919) + + Before: + ``` + info: breakpoint reached + + + Starting REPL to allow you to inspect the current state of the evaluator. + + Welcome to Nix 2.20.0pre20231222_dirty. Type :? for help. + + nix-repl> :continue + error: uh oh + + + Starting REPL to allow you to inspect the current state of the evaluator. + + Welcome to Nix 2.20.0pre20231222_dirty. Type :? for help. + + nix-repl> + ``` + + After: + + ``` + info: breakpoint reached + + Nix 2.20.0pre20231222_dirty debugger + Type :? for help. + nix-repl> :continue + error: uh oh + + nix-repl> + ``` + +- Cycle detection in `nix repl` is simpler and more reliable [#8672](https://github.com/NixOS/nix/issues/8672) [#9926](https://github.com/NixOS/nix/pull/9926) + + The cycle detection in `nix repl`, `nix eval`, `builtins.trace`, and everywhere + else values are printed is now simpler and matches the cycle detection in + `nix-instantiate --eval` output. + + Before: + + ``` + nix eval --expr 'let self = { inherit self; }; in self' + { self = { self = «repeated»; }; } + ``` + + After: + + ``` + { self = «repeated»; } + ``` + +- In the debugger, `while evaluating the attribute` errors now include position information [#9915](https://github.com/NixOS/nix/pull/9915) + + Before: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + 0x600001522598 + ``` + + After: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + /nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27 + + 131| + 132| bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs; + | ^ + 133| in + ``` + +- Stack size is increased on macOS [#9860](https://github.com/NixOS/nix/pull/9860) + + Previously, Nix would set the stack size to 64MiB on Linux, but would leave the + stack size set to the default (approximately 8KiB) on macOS. Now, the stack + size is correctly set to 64MiB on macOS as well, which should reduce stack + overflow segfaults in deeply-recursive Nix expressions. + diff --git a/doc/manual/src/release-notes/rl-2.22.md b/doc/manual/src/release-notes/rl-2.22.md new file mode 100644 index 000000000..c78d3d692 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.22.md @@ -0,0 +1,21 @@ +# Release 2.22.0 (2024-04-23) + +### Significant changes + +- Remove experimental repl-flake [#10103](https://github.com/NixOS/nix/issues/10103) [#10299](https://github.com/NixOS/nix/pull/10299) + + The `repl-flake` experimental feature has been removed. The `nix repl` command now works like the rest of the new CLI in that `nix repl {path}` now tries to load a flake at `{path}` (or fails if the `flakes` experimental feature isn't enabled). + +### Other changes + +- `nix eval` prints derivations as `.drv` paths [#10200](https://github.com/NixOS/nix/pull/10200) + + `nix eval` will now print derivations as their `.drv` paths, rather than as + attribute sets. This makes commands like `nix eval nixpkgs#bash` terminate + instead of infinitely looping into recursive self-referential attributes: + + ```ShellSession + $ nix eval nixpkgs#bash + «derivation /nix/store/m32cbgbd598f4w299g0hwyv7gbw6rqcg-bash-5.2p26.drv» + ``` + diff --git a/doc/manual/src/release-notes/rl-2.23.md b/doc/manual/src/release-notes/rl-2.23.md new file mode 100644 index 000000000..ac842fdc0 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.23.md @@ -0,0 +1,102 @@ +# Release 2.23.0 (2024-06-03) + +- New builtin: `builtins.warn` [#306026](https://github.com/NixOS/nix/issues/306026) [#10592](https://github.com/NixOS/nix/pull/10592) + + `builtins.warn` behaves like `builtins.trace "warning: ${msg}"`, has an accurate log level, and is controlled by the options + [`debugger-on-trace`](@docroot@/command-ref/conf-file.md#conf-debugger-on-trace), + [`debugger-on-warn`](@docroot@/command-ref/conf-file.md#conf-debugger-on-warn) and + [`abort-on-warn`](@docroot@/command-ref/conf-file.md#conf-abort-on-warn). + +- Make `nix build --keep-going` consistent with `nix-build --keep-going` + + This means that if e.g. multiple fixed-output derivations fail to + build, all hash mismatches are displayed. + +- Modify `nix derivation {add,show}` JSON format [#9866](https://github.com/NixOS/nix/issues/9866) [#10722](https://github.com/NixOS/nix/pull/10722) + + The JSON format for derivations has been slightly revised to better conform to our [JSON guidelines](@docroot@/development/cli-guideline.md#returning-future-proof-json). + In particular, the hash algorithm and content addressing method of content-addresed derivation outputs are now separated into two fields `hashAlgo` and `method`, + rather than one field with an arcane `:`-separated format. + + This JSON format is only used by the experimental `nix derivation` family of commands, at this time. + Future revisions are expected as the JSON format is still not entirely in compliance even after these changes. + +- Warn on unknown settings anywhere in the command line [#10701](https://github.com/NixOS/nix/pull/10701) + + All `nix` commands will now properly warn when an unknown option is specified anywhere in the command line. + + Before: + + ```console + $ nix-instantiate --option foobar baz --expr '{}' + warning: unknown setting 'foobar' + $ nix-instantiate '{}' --option foobar baz --expr + $ nix eval --expr '{}' --option foobar baz + { } + ``` + + After: + + ```console + $ nix-instantiate --option foobar baz --expr '{}' + warning: unknown setting 'foobar' + $ nix-instantiate '{}' --option foobar baz --expr + warning: unknown setting 'foobar' + $ nix eval --expr '{}' --option foobar baz + warning: unknown setting 'foobar' + { } + ``` + +- `nix env shell` is the new `nix shell`, and `nix shell` remains an accepted alias [#10504](https://github.com/NixOS/nix/issues/10504) [#10807](https://github.com/NixOS/nix/pull/10807) + + This is part of an effort to bring more structure to the CLI subcommands. + + `nix env` will be about the process environment. + Future commands may include `nix env run` and `nix env print-env`. + + It is also somewhat analogous to the [planned](https://github.com/NixOS/nix/issues/10504) `nix dev shell` (currently `nix develop`), which is less about environment variables, and more about running a development shell, which is a more powerful command, but also requires more setup. + +- Flake operations that expect derivations now print the failing value and its type [#10778](https://github.com/NixOS/nix/pull/10778) + + In errors like `flake output attribute 'nixosConfigurations.yuki.config' is not a derivation or path`, the message now includes the failing value and type. + + Before: + + ``` + error: flake output attribute 'nixosConfigurations.yuki.config' is not a derivation or path + ```` + + After: + + ``` + error: expected flake output attribute 'nixosConfigurations.yuki.config' to be a derivation or path but found a set: { appstream = «thunk»; assertions = «thunk»; boot = { bcache = «thunk»; binfmt = «thunk»; binfmtMiscRegistrations = «thunk»; blacklistedKernelModules = «thunk»; bootMount = «thunk»; bootspec = «thunk»; cleanTmpDir = «thunk»; consoleLogLevel = «thunk»; «43 attributes elided» }; «48 attributes elided» } + ``` + +- `fetchTree` now fetches Git repositories shallowly by default [#10028](https://github.com/NixOS/nix/pull/10028) + + `builtins.fetchTree` now clones Git repositories shallowly by default, which reduces network traffic and disk usage significantly in many cases. + + Previously, the default behavior was to clone the full history of a specific tag or branch (e.g. `ref`) and only afterwards extract the files of one specific revision. + + From now on, the `ref` and `allRefs` arguments will be ignored, except if shallow cloning is disabled by setting `shallow = false`. + + The defaults for `builtins.fetchGit` remain unchanged. Here, shallow cloning has to be enabled manually by passing `shallow = true`. + +- Store object info JSON format now uses `null` rather than omitting fields [#9995](https://github.com/NixOS/nix/pull/9995) + + The [store object info JSON format](@docroot@/protocols/json/store-object-info.md), used for e.g. `nix path-info`, no longer omits fields to indicate absent information, but instead includes the fields with a `null` value. + For example, `"ca": null` is used to to indicate a store object that isn't content-addressed rather than omitting the `ca` field entirely. + This makes records of this sort more self-describing, and easier to consume programmatically. + + We will follow this design principle going forward; + the [JSON guidelines](@docroot@/development/json-guideline.md) in the contributing section have been updated accordingly. + +- Large path warnings [#10661](https://github.com/NixOS/nix/pull/10661) + + Nix can now warn when evaluation of a Nix expression causes a large + path to be copied to the Nix store. The threshold for this warning can + be configured using [the `warn-large-path-threshold` + setting](@docroot@/command-ref/conf-file.md#warn-large-path-threshold), + e.g. `--warn-large-path-threshold 100M` will warn about paths larger + than 100 MiB. + diff --git a/doc/manual/src/release-notes/rl-2.24.md b/doc/manual/src/release-notes/rl-2.24.md new file mode 100644 index 000000000..5bcc1d79c --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.24.md @@ -0,0 +1,324 @@ +# Release 2.24.0 (2024-07-31) + +### Significant changes + +- Harden user sandboxing + + The build directory has been hardened against interference with the outside world by nesting it inside another directory owned by (and only readable by) the daemon user. + + This is a low severity security fix, [CVE-2024-38531](https://www.cve.org/CVERecord?id=CVE-2024-38531). + + Credit: [**@alois31**](https://github.com/alois31), [**Linus Heckemann (@lheckemann)**](https://github.com/lheckemann) + Co-authors: [**@edolstra**](https://github.com/edolstra) + +- `nix-shell ` looks for `shell.nix` [#496](https://github.com/NixOS/nix/issues/496) [#2279](https://github.com/NixOS/nix/issues/2279) [#4529](https://github.com/NixOS/nix/issues/4529) [#5431](https://github.com/NixOS/nix/issues/5431) [#11053](https://github.com/NixOS/nix/issues/11053) [#11057](https://github.com/NixOS/nix/pull/11057) + + `nix-shell $x` now looks for `$x/shell.nix` when `$x` resolves to a directory. + + Although this might be seen as a breaking change, its primarily interactive usage makes it a minor issue. + This adjustment addresses a commonly reported problem. + + This also applies to `nix-shell` shebang scripts. Consider the following example: + + ```shell + #!/usr/bin/env nix-shell + #!nix-shell -i bash + ``` + + This will now load `shell.nix` from the script's directory, if it exists; `default.nix` otherwise. + + The old behavior can be opted into by setting the option [`nix-shell-always-looks-for-shell-nix`](@docroot@/command-ref/conf-file.md#conf-nix-shell-always-looks-for-shell-nix) to `false`. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + +- `nix-repl`'s `:doc` shows documentation comments [#3904](https://github.com/NixOS/nix/issues/3904) [#10771](https://github.com/NixOS/nix/issues/10771) [#1652](https://github.com/NixOS/nix/pull/1652) [#9054](https://github.com/NixOS/nix/pull/9054) [#11072](https://github.com/NixOS/nix/pull/11072) + + `nix repl` has a `:doc` command that previously only rendered documentation for internally defined functions. + This feature has been extended to also render function documentation comments, in accordance with [RFC 145]. + + Example: + + ``` + nix-repl> :doc lib.toFunction + Function toFunction + … defined at /home/user/h/nixpkgs/lib/trivial.nix:1072:5 + + Turns any non-callable values into constant functions. Returns + callable values as is. + + Inputs + + v + + : Any value + + Examples + + :::{.example} + + ## lib.trivial.toFunction usage example + + | nix-repl> lib.toFunction 1 2 + | 1 + | + | nix-repl> lib.toFunction (x: x + 1) 2 + | 3 + + ::: + ``` + + Known limitations: + - It does not render documentation for "formals", such as `{ /** the value to return */ x, ... }: x`. + - Some extensions to markdown are not yet supported, as you can see in the example above. + + We'd like to acknowledge [Yingchi Long (@inclyc)](https://github.com/inclyc) for proposing a proof of concept for this functionality in [#9054](https://github.com/NixOS/nix/pull/9054), as well as [@sternenseemann](https://github.com/sternenseemann) and [Johannes Kirschbauer (@hsjobeki)](https://github.com/hsjobeki) for their contributions, proposals, and their work on [RFC 145]. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + + [RFC 145]: https://github.com/NixOS/rfcs/pull/145 + +### Other changes + +- Solve `cached failure of attribute X` [#9165](https://github.com/NixOS/nix/issues/9165) [#10513](https://github.com/NixOS/nix/issues/10513) [#10564](https://github.com/NixOS/nix/pull/10564) + + This eliminates all "cached failure of attribute X" messages by forcing evaluation of the original value when needed to show the exception to the user. This enhancement improves error reporting by providing the underlying message and stack trace. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Run the flake regressions test suite [#10603](https://github.com/NixOS/nix/pull/10603) + + This update introduces a GitHub action to run a subset of the [flake regressions test suite](https://github.com/NixOS/flake-regressions), which includes 259 flakes with their expected evaluation results. Currently, the action runs the first 25 flakes due to the full test suite's extensive runtime. A manually triggered action may be implemented later to run the entire test suite. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Support unit prefixes in configuration settings [#10668](https://github.com/NixOS/nix/pull/10668) + + Configuration settings in Nix now support unit prefixes, allowing for more intuitive and readable configurations. For example, you can now specify [`--min-free 1G`](@docroot@/command-ref/opt-common.md#opt-min-free) to set the minimum free space to 1 gigabyte. + + This enhancement was extracted from [#7851](https://github.com/NixOS/nix/pull/7851) and is also useful for PR [#10661](https://github.com/NixOS/nix/pull/10661). + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- `nix build`: show all FOD errors with `--keep-going` [#10734](https://github.com/NixOS/nix/pull/10734) + + The [`nix build`](@docroot@/command-ref/new-cli/nix3-build.md) command has been updated to improve the behavior of the [`--keep-going`] flag. Now, when `--keep-going` is used, all hash-mismatch errors of failing fixed-output derivations (FODs) are displayed, similar to the behavior for other build failures. This enhancement ensures that all relevant build errors are shown, making it easier for users to update multiple derivations at once or to diagnose and fix issues. + + Author: [**Jörg Thalheim (@Mic92)**](https://github.com/Mic92), [**Maximilian Bosch (@Ma27)**](https://github.com/Ma27) + + [`--keep-going`](@docroot@/command-ref/opt-common.md#opt-keep-going) + +- Build with Meson [#2503](https://github.com/NixOS/nix/issues/2503) [#10378](https://github.com/NixOS/nix/pull/10378) [#10855](https://github.com/NixOS/nix/pull/10855) [#10904](https://github.com/NixOS/nix/pull/10904) [#10908](https://github.com/NixOS/nix/pull/10908) [#10914](https://github.com/NixOS/nix/pull/10914) [#10933](https://github.com/NixOS/nix/pull/10933) [#10936](https://github.com/NixOS/nix/pull/10936) [#10954](https://github.com/NixOS/nix/pull/10954) [#10955](https://github.com/NixOS/nix/pull/10955) [#10963](https://github.com/NixOS/nix/pull/10963) [#10967](https://github.com/NixOS/nix/pull/10967) [#10973](https://github.com/NixOS/nix/pull/10973) [#11034](https://github.com/NixOS/nix/pull/11034) [#11054](https://github.com/NixOS/nix/pull/11054) [#11055](https://github.com/NixOS/nix/pull/11055) [#11060](https://github.com/NixOS/nix/pull/11060) [#11064](https://github.com/NixOS/nix/pull/11064) [#11155](https://github.com/NixOS/nix/pull/11155) + + These changes aim to replace the use of autotools and `make` with Meson for building various components of Nix. Additionally, each library is built in its own derivation, leveraging Meson's "subprojects" feature to allow a single development shell for building all libraries while also supporting separate builds. This approach aims to improve productivity and build modularity, compared to both make and a monolithic Meson-based derivation. + + Special thanks to everyone who has contributed to the Meson port, particularly [**@p01arst0rm**](https://github.com/p01arst0rm) and [**@Qyriad**](https://github.com/Qyriad). + + Authors: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314), [**Tom Bereknyei**](https://github.com/tomberek), [**Théophane Hufschmitt (@thufschmitt)**](https://github.com/thufschmitt), [**Valentin Gagarin (@fricklerhandwerk)**](https://github.com/fricklerhandwerk), [**Robert Hensing (@roberth)**](https://github.com/roberth) + Co-authors: [**@p01arst0rm**](https://github.com/p01arst0rm), [**@Qyriad**](https://github.com/Qyriad) + +- Evaluation cache: fix cache regressions [#10570](https://github.com/NixOS/nix/issues/10570) [#11086](https://github.com/NixOS/nix/pull/11086) + + This update addresses two bugs in the evaluation cache system: + + 1. Regression in #10570: The evaluation cache was not being persisted in `nix develop`. + 2. Nix could sometimes try to commit the evaluation cache SQLite transaction without there being an active transaction, resulting in non-error errors being printed. + + Author: [**Lexi Mattick (@kognise)**](https://github.com/kognise) + +- Introduce `libnixflake` [#9063](https://github.com/NixOS/nix/pull/9063) + + A new library, `libnixflake`, has been introduced to better separate the Flakes layer within Nix. This change refactors the codebase to encapsulate Flakes-specific functionality within its own library. + + See the commits in the pull request for detailed changes, with the only significant code modifications happening in the initial commit. + + This change was alluded to in [RFC 134](https://github.com/nixos/rfcs/blob/master/rfcs/0134-nix-store-layer.md) and is a step towards a more modular and maintainable codebase. + + Author: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314) + +- CLI options `--arg-from-file` and `--arg-from-stdin` [#9913](https://github.com/NixOS/nix/pull/9913) + +- The `--debugger` now prints source location information, instead of the + pointers of source location information. Before: + + ``` + nix-repl> :bt + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + 0x600001522598 + ``` + + After: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + /nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27 + + 131| + 132| bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs; + | ^ + 133| in + ``` + +- Stop vendoring `toml11` + + We don't apply any patches to it, and vendoring it locks users into + bugs (it hasn't been updated since its introduction in late 2021). + + Author: [**Winter (@winterqt)**](https://github.com/winterqt) + +- Rename hash format `base32` to `nix32` [#8678](https://github.com/NixOS/nix/pull/8678) + + Hash format `base32` was renamed to `nix32` since it used a special nix-specific character set for + [Base32](https://en.wikipedia.org/wiki/Base32). + + **Deprecation**: Use `nix32` instead of `base32` as `toHashFormat` + + For the builtin `convertHash`, the `toHashFormat` parameter now accepts the same hash formats as the `--to`/`--from` + parameters of the `nix hash conert` command: `"base16"`, `"nix32"`, `"base64"`, and `"sri"`. The former `"base32"` value + remains as a deprecated alias for `"nix32"`. Please convert your code from: + + ```nix + builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base32";} + ``` + + to + + ```nix + builtins.convertHash { inherit hash hashAlgo; toHashFormat = "nix32";} + ``` + +- Add `pipe-operators` experimental feature [#11131](https://github.com/NixOS/nix/pull/11131) + + This is a draft implementation of [RFC 0148](https://github.com/NixOS/rfcs/pull/148). + + The `pipe-operators` experimental feature adds [`<|` and `|>` operators][pipe operators] to the Nix language. + *a* `|>` *b* is equivalent to the function application *b* *a*, and + *a* `<|` *b* is equivalent to the function application *a* *b*. + + For example: + + ``` + nix-repl> 1 |> builtins.add 2 |> builtins.mul 3 + 9 + + nix-repl> builtins.add 1 <| builtins.mul 2 <| 3 + 7 + ``` + + `<|` and `|>` are right and left associative, respectively, and have lower precedence than any other operator. + These properties may change in future releases. + + See [the RFC](https://github.com/NixOS/rfcs/pull/148) for more examples and rationale. + + [pipe operators]: @docroot@/language/operators.md#pipe-operators + +- `nix-shell` shebang uses relative path [#4232](https://github.com/NixOS/nix/issues/4232) [#5088](https://github.com/NixOS/nix/pull/5088) [#11058](https://github.com/NixOS/nix/pull/11058) + + + Relative [path](@docroot@/language/types.md#type-path) literals in `nix-shell` shebang scripts' options are now resolved relative to the [script's location](@docroot@/glossary.md?highlight=base%20directory#gloss-base-directory). + Previously they were resolved relative to the current working directory. + + For example, consider the following script in `~/myproject/say-hi`: + + ```shell + #!/usr/bin/env nix-shell + #!nix-shell --expr 'import ./shell.nix' + #!nix-shell --arg toolset './greeting-tools.nix' + #!nix-shell -i bash + hello + ``` + + Older versions of `nix-shell` would resolve `shell.nix` relative to the current working directory, such as the user's home directory in this example: + + ```console + [hostname:~]$ ./myproject/say-hi + error: + … while calling the 'import' builtin + at «string»:1:2: + 1| (import ./shell.nix) + | ^ + + error: path '/home/user/shell.nix' does not exist + ``` + + Since this release, `nix-shell` resolves `shell.nix` relative to the script's location, and `~/myproject/shell.nix` is used. + + ```console + $ ./myproject/say-hi + Hello, world! + ``` + + **Opt-out** + + This is technically a breaking change, so we have added an option so you can adapt independently of your Nix update. + The old behavior can be opted into by setting the option [`nix-shell-shebang-arguments-relative-to-script`](@docroot@/command-ref/conf-file.md#conf-nix-shell-shebang-arguments-relative-to-script) to `false`. + This option will be removed in a future release. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + +- Improve handling of tarballs that don't consist of a single top-level directory [#11195](https://github.com/NixOS/nix/pull/11195) + + In previous Nix releases, the tarball fetcher (used by `builtins.fetchTarball`) erroneously merged top-level directories into a single directory, and silently discarded top-level files that are not directories. This is no longer the case. The new behaviour is that *only* if the tarball consists of a single directory, the top-level path component of the files in the tarball is removed (similar to `tar`'s `--strip-components=1`). + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Improve handling of tarballs that don't consist of a single top-level directory [#11195](https://github.com/NixOS/nix/pull/11195) + + In previous Nix releases, the tarball fetcher (used by `builtins.fetchTarball`) erroneously merged top-level directories into a single directory, and silently discarded top-level files that are not directories. This is no longer the case. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Setting to warn about large paths [#10778](https://github.com/NixOS/nix/pull/10778) + + Nix can now warn when evaluation of a Nix expression causes a large + path to be copied to the Nix store. The threshold for this warning can + be configured using the `warn-large-path-threshold` setting, + e.g. `--warn-large-path-threshold 100M`. + + +# Contributors + +This release was made possible by the following 43 contributors: + +- Andreas Rammhold [**(@andir)**](https://github.com/andir) +- Andrew Marshall [**(@amarshall)**](https://github.com/amarshall) +- Brian McKenna [**(@puffnfresh)**](https://github.com/puffnfresh) +- Cameron [**(@SkamDart)**](https://github.com/SkamDart) +- Cole Helbling [**(@cole-h)**](https://github.com/cole-h) +- Corbin Simpson [**(@MostAwesomeDude)**](https://github.com/MostAwesomeDude) +- Eelco Dolstra [**(@edolstra)**](https://github.com/edolstra) +- Emily [**(@emilazy)**](https://github.com/emilazy) +- Enno Richter [**(@elohmeier)**](https://github.com/elohmeier) +- Farid Zakaria [**(@fzakaria)**](https://github.com/fzakaria) +- HaeNoe [**(@haenoe)**](https://github.com/haenoe) +- Hamir Mahal [**(@hamirmahal)**](https://github.com/hamirmahal) +- Harmen [**(@alicebob)**](https://github.com/alicebob) +- Ivan Trubach [**(@tie)**](https://github.com/tie) +- Jared Baur [**(@jmbaur)**](https://github.com/jmbaur) +- John Ericson [**(@Ericson2314)**](https://github.com/Ericson2314) +- Jonathan De Troye [**(@detroyejr)**](https://github.com/detroyejr) +- Jörg Thalheim [**(@Mic92)**](https://github.com/Mic92) +- Klemens Nanni [**(@klemensn)**](https://github.com/klemensn) +- Las Safin [**(@L-as)**](https://github.com/L-as) +- Lexi Mattick [**(@kognise)**](https://github.com/kognise) +- Matthew Bauer [**(@matthewbauer)**](https://github.com/matthewbauer) +- Max “Goldstein†Siling [**(@GoldsteinE)**](https://github.com/GoldsteinE) +- Mingye Wang [**(@Artoria2e5)**](https://github.com/Artoria2e5) +- Philip Taron [**(@philiptaron)**](https://github.com/philiptaron) +- Pierre Bourdon [**(@delroth)**](https://github.com/delroth) +- Pino Toscano [**(@pinotree)**](https://github.com/pinotree) +- RTUnreal [**(@RTUnreal)**](https://github.com/RTUnreal) +- Robert Hensing [**(@roberth)**](https://github.com/roberth) +- Romain Neil [**(@romain-neil)**](https://github.com/romain-neil) +- Ryan Hendrickson [**(@rhendric)**](https://github.com/rhendric) +- Sergei Trofimovich [**(@trofi)**](https://github.com/trofi) +- Shogo Takata [**(@pineapplehunter)**](https://github.com/pineapplehunter) +- Siddhant Kumar [**(@siddhantk232)**](https://github.com/siddhantk232) +- Silvan Mosberger [**(@infinisil)**](https://github.com/infinisil) +- Théophane Hufschmitt [**(@thufschmitt)**](https://github.com/thufschmitt) +- Valentin Gagarin [**(@fricklerhandwerk)**](https://github.com/fricklerhandwerk) +- Winter [**(@winterqt)**](https://github.com/winterqt) +- jade [**(@lf-)**](https://github.com/lf-) +- kirillrdy [**(@kirillrdy)**](https://github.com/kirillrdy) +- pennae [**(@pennae)**](https://github.com/pennae) +- poweredbypie [**(@poweredbypie)**](https://github.com/poweredbypie) +- tomberek [**(@tomberek)**](https://github.com/tomberek) diff --git a/doc/manual/src/release-notes/rl-2.4.md b/doc/manual/src/release-notes/rl-2.4.md index 8b566fc7b..1201e53b6 100644 --- a/doc/manual/src/release-notes/rl-2.4.md +++ b/doc/manual/src/release-notes/rl-2.4.md @@ -23,7 +23,7 @@ more than 2800 commits from 195 contributors since release 2.3. * The **`nix` command** has seen a lot of work and is now almost at feature parity with the old command-line interface (the `nix-*` commands). It aims to be [more modern, consistent and pleasant to - use](../contributing/cli-guideline.md) than the old CLI. It is still + use](../development/cli-guideline.md) than the old CLI. It is still marked as experimental but its interface should not change much anymore in future releases. diff --git a/doc/manual/src/store/file-system-object/content-address.md b/doc/manual/src/store/file-system-object/content-address.md new file mode 100644 index 000000000..410d7fb7c --- /dev/null +++ b/doc/manual/src/store/file-system-object/content-address.md @@ -0,0 +1,85 @@ +# Content-Addressing File System Objects + +For many operations, Nix needs to calculate [a content addresses](@docroot@/glossary.md#gloss-content-address) of [a file system object][file system object]. +Usually this is needed as part of +[content addressing store objects](../store-object/content-address.md), +since store objects always have a root file system object. +But some command-line utilities also just work on "raw" file system objects, not part of any store object. + +Every content addressing scheme Nix uses ultimately involves feeding data into a [hash function](https://en.wikipedia.org/wiki/Hash_function), and getting back an opaque fixed-size digest which is deemed a content address. +The various *methods* of content addressing thus differ in how abstract data (in this case, a file system object and its descendents) are fed into the hash function. + +## Serialising File System Objects { #serial } + +The simplest method is to serialise the entire file system object tree into a single binary string, and then hash that binary string, yielding the content address. +In this section we describe the currently-supported methods of serialising file system objects. + +### Flat { #serial-flat } + +A single file object can just be hashed by its contents. +This is not enough information to encode the fact that the file system object is a file, +but if we *already* know that the FSO is a single non-executable file by other means, it is sufficient. + +Because the hashed data is just the raw file, as is, this choice is good for compatibility with other systems. +For example, Unix commands like `sha256sum` or `sha1sum` will produce hashes for single files that match this. + +### Nix Archive (NAR) { #serial-nix-archive } + +For the other cases of [file system objects][file system object], especially directories with arbitrary descendents, we need a more complex serialisation format. +Examples of such serialisations are the ZIP and TAR file formats. +However, for our purposes these formats have two problems: + +- They do not have a canonical serialisation, meaning that given an FSO, there can +be many different serialisations. + For instance, TAR files can have variable amounts of padding between archive members; + and some archive formats leave the order of directory entries undefined. + This would be bad because we use serialisation to compute cryptographic hashes over file system objects, and for those hashes to be useful as a content address or for integrity checking, uniqueness is crucial. + Otherwise, correct hashes would report false mismatches, and the store would fail to find the content. + +- They store more information than we have in our notion of FSOs, such as time stamps. + This can cause FSOs that Nix should consider equal to hash to different values on different machines, just because the dates differ. + +- As a practical consideration, the TAR format is the only truly universal format in the Unix environment. + It has many problems, such as an inability to deal with long file names and files larger than 2^33 bytes. + Current implementations such as GNU Tar work around these limitations in various ways. + +For these reasons, Nix has its very own archive format—the Nix Archive (NAR) format, +which is carefully designed to avoid the problems described above. + +The exact specification of the Nix Archive format is in `protocols/nix-archive.md` + +## Content addressing File System Objects beyond a single serialisation pass + +Serialising the entire tree and then hashing that binary string is not the only option for content addressing, however. +Another technique is that of a [Merkle graph](https://en.wikipedia.org/wiki/Merkle_tree), where previously computed hashes are included in subsequent byte strings to be hashed. + +In particular, the Merkle graphs can match the original graph structure of file system objects: +we can first hash (serialised) child file system objects, and then hash parent objects using the hashes of their children in the serialisation (to be hashed) of the parent file system objects. + +Currently, there is one such Merkle DAG content addressing method supported. + +### Git ([experimental][xp-feature-git-hashing]) { #git } + +> **Warning** +> +> This method is part of the [`git-hashing`][xp-feature-git-hashing] experimental feature. + +Git's file system model is very close to Nix's, and so Git's content addressing method is a pretty good fit. +Just as with regular Git, files and symlinks are hashed as git "blobs", and directories are hashed as git "trees". + +However, one difference between Nix's and Git's file system model needs special treatment. +Plain files, executable files, and symlinks are not differentiated as distinctly addressable objects, but by their context: by the directory entry that refers to them. +That means so long as the root object is a directory, there is no problem: +every non-directory object is owned by a parent directory, and the entry that refers to it provides the missing information. +However, if the root object is not a directory, then we have no way of knowing which one of an executable file, non-executable file, or symlink it is supposed to be. + +In response to this, we have decided to treat a bare file as non-executable file. +This is similar to do what we do with [flat serialisation](#serial-flat), which also lacks this information. +To avoid an address collision, attempts to hash a bare executable file or symlink will result in an error (just as would happen for flat serialisation also). +Thus, Git can encode some, but not all of Nix's "File System Objects", and this sort of content-addressing is likewise partial. + +In the future, we may support a Git-like hash for such file system objects, or we may adopt another Merkle DAG format which is capable of representing all Nix file system objects. + +[file system object]: ../file-system-object.md +[store object]: ../store-object.md +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/doc/manual/src/store/store-object/content-address.md b/doc/manual/src/store/store-object/content-address.md new file mode 100644 index 000000000..02dce2836 --- /dev/null +++ b/doc/manual/src/store/store-object/content-address.md @@ -0,0 +1,95 @@ +# Content-Addressing Store Objects + +Just [like][fso-ca] [File System Objects][File System Object], +[Store Objects][Store Object] can also be [content-addressed](@docroot@/glossary.md#gloss-content-addressed), +unless they are [input-addressed](@docroot@/glossary.md#gloss-input-addressed-store-object). + +For store objects, the content address we produce will take the form of a [Store Path] rather than regular hash. +In particular, the content-addressing scheme will ensure that the digest of the store path is solely computed from the + +- file system object graph (the root one and its children, if it has any) +- references +- [store directory](../store-path.md#store-directory) +- name + +of the store object, and not any other information, which would not be an intrinsic property of that store object. + +For the full specification of the algorithms involved, see the [specification of store path digests][sp-spec]. + +[File System Object]: ../file-system-object.md +[Store Object]: ../store-object.md +[Store Path]: ../store-path.md + +## Content addressing each part of a store object + +### File System Objects + +With all currently supported store object content addressing methods, the file system object is always [content-addressed][fso-ca] first, and then that hash is incorporated into content address computation for the store object. + +### References + +With all currently supported store object content addressing methods, +other objects are referred to by their regular (string-encoded-) [store paths][Store Path]. + +Self-references however cannot be referred to by their path, because we are in the midst of describing how to compute that path! + +> The alternative would require finding as hash function fixed point, i.e. the solution to an equation in the form +> ``` +> digest = hash(..... || digest || ....) +> ``` +> which is computationally infeasible. +> As far as we know, this is equivalent to finding a hash collision. + +Instead we just have a "has self reference" boolean, which will end up affecting the digest. + +### Name and Store Directory + +These two items affect the digest in a way that is standard for store path digest computations and not specific to content-addressing. +Consult the [specification of store path digests][sp-spec] for further details. + +## Content addressing Methods + +For historical reasons, we don't support all features in all combinations. +Each currently supported method of content addressing chooses a single method of file system object hashing, and may offer some restrictions on references. +The names and store directories are unrestricted however. + +### Flat { #method-flat } + +This uses the corresponding [Flat](../file-system-object/content-address.md#serial-flat) method of file system object content addressing. + +References are not supported: store objects with flat hashing *and* references can not be created. + +### Text { #method-text } + +This also uses the corresponding [Flat](../file-system-object/content-address.md#serial-flat) method of file system object content addressing. + +References to other store objects are supported, but self references are not. + +This is the only store-object content-addressing method that is not named identically with a corresponding file system object method. +It is somewhat obscure, mainly used for "drv files" +(derivations serialized as store objects in their ["ATerm" file format](@docroot@/protocols/derivation-aterm.md)). +Prefer another method if possible. + +### Nix Archive { #method-nix-archive } + +This uses the corresponding [Nix Archive](../file-system-object/content-address.md#serial-nix-archive) method of file system object content addressing. + +References (to other store objects and self references alike) are supported so long as the hash algorithm is SHA-256, but not (neither kind) otherwise. + +### Git { #method-git } + +> **Warning** +> +> This method is part of the [`git-hashing`][xp-feature-git-hashing] experimental feature. + +This uses the corresponding [Git](../file-system-object/content-address.md#serial-git) method of file system object content addressing. + +References are not supported. + +Only SHA-1 is supported at this time. +If [SHA-256-based Git](https://git-scm.com/docs/hash-function-transition) +becomes more widespread, this restriction will be revisited. + +[fso-ca]: ../file-system-object/content-address.md +[sp-spec]: @docroot@/protocols/store-path.md +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/doc/manual/src/store/store-path.md b/doc/manual/src/store/store-path.md index b5ad0c654..beec2389b 100644 --- a/doc/manual/src/store/store-path.md +++ b/doc/manual/src/store/store-path.md @@ -1,5 +1,11 @@ # Store Path +> **Example** +> +> `/nix/store/a040m110amc4h71lds2jmr8qrkj2jhxd-git-2.38.1` +> +> A rendered store path + Nix implements references to [store objects](./index.md#store-object) as *store paths*. Think of a store path as an [opaque], [unique identifier]: @@ -37,6 +43,10 @@ A store path is rendered to a file system path as the concatenation of > store directory digest name > ``` +Exactly how the digest is calculated depends on the type of store path. +Store path digests are *supposed* to be opaque, and so for most operations, it is not necessary to know the details. +That said, the manual has a full [specification of store path digests](@docroot@/protocols/store-path.md). + ## Store Directory Every [Nix store](./index.md) has a store directory. @@ -46,7 +56,7 @@ But if the store has a file system representation, the store directory contains [file system objects]: ./file-system-object.md -This means a store path is not just derived from the referenced store object itself, but depends on the store the store object is in. +This means a store path is not just derived from the referenced store object itself, but depends on the store that the store object is in. > **Note** > diff --git a/flake.lock b/flake.lock index bb2e400c0..2ac413a69 100644 --- a/flake.lock +++ b/flake.lock @@ -3,11 +3,11 @@ "flake-compat": { "flake": false, "locked": { - "lastModified": 1673956053, - "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", "owner": "edolstra", "repo": "flake-compat", - "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", "type": "github" }, "original": { @@ -16,38 +16,100 @@ "type": "github" } }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1719994518, + "narHash": "sha256-pQMhCCHyQGRzdfAkdJ4cIWiw+JNuWsTX7f0ZYSyz0VY=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "9227223f6d922fee3c7b190b2cc238a99527bbb7", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "git-hooks-nix": { + "inputs": { + "flake-compat": [], + "gitignore": [], + "nixpkgs": [ + "nixpkgs" + ], + "nixpkgs-stable": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1721042469, + "narHash": "sha256-6FPUl7HVtvRHCCBQne7Ylp4p+dpP3P/OYuzjztZ4s70=", + "owner": "cachix", + "repo": "git-hooks.nix", + "rev": "f451c19376071a90d8c58ab1a953c6e9840527fd", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "git-hooks.nix", + "type": "github" + } + }, "libgit2": { "flake": false, "locked": { - "lastModified": 1697646580, - "narHash": "sha256-oX4Z3S9WtJlwvj0uH9HlYcWv+x1hqp8mhXl7HsLu2f0=", + "lastModified": 1715853528, + "narHash": "sha256-J2rCxTecyLbbDdsyBWn9w7r3pbKRMkI9E7RvRgAqBdY=", "owner": "libgit2", "repo": "libgit2", - "rev": "45fd9ed7ae1a9b74b957ef4f337bc3c8b3df01b5", + "rev": "36f7e21ad757a3dacc58cf7944329da6bc1d6e96", "type": "github" }, "original": { "owner": "libgit2", + "ref": "v1.8.1", "repo": "libgit2", "type": "github" } }, "nixpkgs": { "locked": { - "lastModified": 1709083642, - "narHash": "sha256-7kkJQd4rZ+vFrzWu8sTRtta5D1kBG0LSRYAfhtmMlSo=", + "lastModified": 1721548954, + "narHash": "sha256-7cCC8+Tdq1+3OPyc3+gVo9dzUNkNIQfwSDJ2HSi2u3o=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "b550fe4b4776908ac2a861124307045f8e717c8e", + "rev": "63d37ccd2d178d54e7fb691d7ec76000740ea24a", "type": "github" }, "original": { "owner": "NixOS", - "ref": "release-23.11", + "ref": "nixos-24.05", "repo": "nixpkgs", "type": "github" } }, + "nixpkgs-23-11": { + "locked": { + "lastModified": 1717159533, + "narHash": "sha256-oamiKNfr2MS6yH64rUn99mIZjc45nGJlj9eGth/3Xuw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446", + "type": "github" + } + }, "nixpkgs-regression": { "locked": { "lastModified": 1643052045, @@ -67,8 +129,11 @@ "root": { "inputs": { "flake-compat": "flake-compat", + "flake-parts": "flake-parts", + "git-hooks-nix": "git-hooks-nix", "libgit2": "libgit2", "nixpkgs": "nixpkgs", + "nixpkgs-23-11": "nixpkgs-23-11", "nixpkgs-regression": "nixpkgs-regression" } } diff --git a/flake.nix b/flake.nix index 42aaace67..9e8592e3a 100644 --- a/flake.nix +++ b/flake.nix @@ -1,18 +1,28 @@ { description = "The purely functional package manager"; - # TODO switch to nixos-23.11-small - # https://nixpk.gs/pr-tracker.html?pr=291954 - inputs.nixpkgs.url = "github:NixOS/nixpkgs/release-23.11"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; + inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446"; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; - inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; }; + inputs.libgit2 = { url = "github:libgit2/libgit2/v1.8.1"; flake = false; }; + + # dev tooling + inputs.flake-parts.url = "github:hercules-ci/flake-parts"; + inputs.git-hooks-nix.url = "github:cachix/git-hooks.nix"; + # work around https://github.com/NixOS/nix/issues/7730 + inputs.flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; + inputs.git-hooks-nix.inputs.nixpkgs.follows = "nixpkgs"; + inputs.git-hooks-nix.inputs.nixpkgs-stable.follows = "nixpkgs"; + # work around 7730 and https://github.com/NixOS/nix/issues/7807 + inputs.git-hooks-nix.inputs.flake-compat.follows = ""; + inputs.git-hooks-nix.inputs.gitignore.follows = ""; + + outputs = inputs@{ self, nixpkgs, nixpkgs-regression, libgit2, ... }: - outputs = { self, nixpkgs, nixpkgs-regression, libgit2, ... }: let inherit (nixpkgs) lib; - inherit (lib) fileset; officialRelease = false; @@ -31,14 +41,9 @@ crossSystems = [ "armv6l-unknown-linux-gnueabihf" "armv7l-unknown-linux-gnueabihf" - "x86_64-unknown-freebsd13" + "riscv64-unknown-linux-gnu" "x86_64-unknown-netbsd" - ]; - - # Nix doesn't yet build on this platform, so we put it in a - # separate list. We just use this for `devShells` and - # `nixpkgsFor`, which this depends on. - shellCrossSystems = crossSystems ++ [ + "x86_64-unknown-freebsd" "x86_64-w64-mingw32" ]; @@ -50,6 +55,18 @@ "stdenv" ]; + /** + `flatMapAttrs attrs f` applies `f` to each attribute in `attrs` and + merges the results into a single attribute set. + + This can be nested to form a build matrix where all the attributes + generated by the innermost `f` are returned as is. + (Provided that the names are unique.) + + See https://nixos.org/manual/nixpkgs/stable/index.html#function-library-lib.attrsets.concatMapAttrs + */ + flatMapAttrs = attrs: f: lib.concatMapAttrs f attrs; + forAllSystems = lib.genAttrs systems; forAllCrossSystems = lib.genAttrs crossSystems; @@ -63,6 +80,17 @@ }) stdenvs); + + # We don't apply flake-parts to the whole flake so that non-development attributes + # load without fetching any development inputs. + devFlake = inputs.flake-parts.lib.mkFlake { inherit inputs; } { + imports = [ ./maintainers/flake-module.nix ]; + systems = lib.subtractLists crossSystems systems; + perSystem = { system, ... }: { + _module.args.pkgs = nixpkgsFor.${system}.native; + }; + }; + # Memoize nixpkgs for different platforms for efficiency. nixpkgsFor = forAllSystems (system: let @@ -84,31 +112,9 @@ in { inherit stdenvs native; static = native.pkgsStatic; - cross = lib.genAttrs shellCrossSystems (crossSystem: make-pkgs crossSystem "stdenv"); + cross = forAllCrossSystems (crossSystem: make-pkgs crossSystem "stdenv"); }); - installScriptFor = tarballs: - nixpkgsFor.x86_64-linux.native.callPackage ./scripts/installer.nix { - inherit tarballs; - }; - - testNixVersions = pkgs: client: daemon: - pkgs.callPackage ./package.nix { - pname = - "nix-tests" - + lib.optionalString - (lib.versionAtLeast daemon.version "2.4pre20211005" && - lib.versionAtLeast client.version "2.4pre20211005") - "-${client.version}-against-${daemon.version}"; - - inherit fileset; - - test-client = client; - test-daemon = daemon; - - doBuild = false; - }; - binaryTarball = nix: pkgs: pkgs.callPackage ./scripts/binary-tarball.nix { inherit nix; }; @@ -120,244 +126,130 @@ { nixStable = prev.nix; - default-busybox-sandbox-shell = final.busybox.override { - useMusl = true; - enableStatic = true; - enableMinimal = true; - extraConfig = '' - CONFIG_FEATURE_FANCY_ECHO y - CONFIG_FEATURE_SH_MATH y - CONFIG_FEATURE_SH_MATH_64 y + # A new scope, so that we can use `callPackage` to inject our own interdependencies + # without "polluting" the top level "`pkgs`" attrset. + # This also has the benefit of providing us with a distinct set of packages + # we can iterate over. + nixComponents = lib.makeScope final.nixDependencies.newScope (import ./packaging/components.nix); - CONFIG_ASH y - CONFIG_ASH_OPTIMIZE_FOR_SIZE y - - CONFIG_ASH_ALIAS y - CONFIG_ASH_BASH_COMPAT y - CONFIG_ASH_CMDCMD y - CONFIG_ASH_ECHO y - CONFIG_ASH_GETOPTS y - CONFIG_ASH_INTERNAL_GLOB y - CONFIG_ASH_JOB_CONTROL y - CONFIG_ASH_PRINTF y - CONFIG_ASH_TEST y - ''; - }; - - libgit2-nix = final.libgit2.overrideAttrs (attrs: { - src = libgit2; - version = libgit2.lastModifiedDate; - cmakeFlags = attrs.cmakeFlags or [] - ++ [ "-DUSE_SSH=exec" ]; + # The dependencies are in their own scope, so that they don't have to be + # in Nixpkgs top level `pkgs` or `nixComponents`. + nixDependencies = lib.makeScope final.newScope (import ./packaging/dependencies.nix { + inherit inputs stdenv versionSuffix; + pkgs = final; }); - boehmgc-nix = (final.boehmgc.override { - enableLargeConfig = true; - }).overrideAttrs(o: { - patches = (o.patches or []) ++ [ - ./dep-patches/boehmgc-coroutine-sp-fallback.diff + nix = final.nixComponents.nix; - # https://github.com/ivmai/bdwgc/pull/586 - ./dep-patches/boehmgc-traceable_allocator-public.diff - ]; - }); - - changelog-d-nix = final.buildPackages.callPackage ./misc/changelog-d.nix { }; - - nix = - let - officialRelease = false; - versionSuffix = - if officialRelease - then "" - else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}"; - - in final.callPackage ./package.nix { - inherit - fileset - stdenv - versionSuffix - ; - officialRelease = false; - boehmgc = final.boehmgc-nix; - libgit2 = final.libgit2-nix; - busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell; - } // { - # this is a proper separate downstream package, but put - # here also for back compat reasons. - perl-bindings = final.nix-perl-bindings; - }; - - nix-perl-bindings = final.callPackage ./perl { - inherit fileset stdenv; + nix_noTests = final.nix.override { + doInstallCheck = false; + doCheck = false; }; + # See https://github.com/NixOS/nixpkgs/pull/214409 + # Remove when fixed in this flake's nixpkgs + pre-commit = + if prev.stdenv.hostPlatform.system == "i686-linux" + then (prev.pre-commit.override (o: { dotnet-sdk = ""; })).overridePythonAttrs (o: { doCheck = false; }) + else prev.pre-commit; + }; in { # A Nixpkgs overlay that overrides the 'nix' and - # 'nix.perl-bindings' packages. + # 'nix-perl-bindings' packages. overlays.default = overlayFor (p: p.stdenv); - hydraJobs = { - - # Binary package for various platforms. - build = forAllSystems (system: self.packages.${system}.nix); - - shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation); - - buildStatic = lib.genAttrs linux64BitSystems (system: self.packages.${system}.nix-static); - - buildCross = forAllCrossSystems (crossSystem: - lib.genAttrs ["x86_64-linux"] (system: self.packages.${system}."nix-${crossSystem}")); - - buildNoGc = forAllSystems (system: - self.packages.${system}.nix.override { enableGC = false; } - ); - - buildNoTests = forAllSystems (system: - self.packages.${system}.nix.override { - doCheck = false; - doInstallCheck = false; - installUnitTests = false; - } - ); - - # Toggles some settings for better coverage. Windows needs these - # library combinations, and Debian build Nix with GNU readline too. - buildReadlineNoMarkdown = forAllSystems (system: - self.packages.${system}.nix.override { - enableMarkdown = false; - readlineFlavor = "readline"; - } - ); - - # Perl bindings for various platforms. - perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nix.perl-bindings); - - # Binary tarball for various platforms, containing a Nix store - # with the closure of 'nix' package, and the second half of - # the installation script. - binaryTarball = forAllSystems (system: binaryTarball nixpkgsFor.${system}.native.nix nixpkgsFor.${system}.native); - - binaryTarballCross = lib.genAttrs ["x86_64-linux"] (system: - forAllCrossSystems (crossSystem: - binaryTarball - self.packages.${system}."nix-${crossSystem}" - nixpkgsFor.${system}.cross.${crossSystem})); - - # The first half of the installation script. This is uploaded - # to https://nixos.org/nix/install. It downloads the binary - # tarball for the user's system and calls the second half of the - # installation script. - installerScript = installScriptFor [ - # Native - self.hydraJobs.binaryTarball."x86_64-linux" - self.hydraJobs.binaryTarball."i686-linux" - self.hydraJobs.binaryTarball."aarch64-linux" - self.hydraJobs.binaryTarball."x86_64-darwin" - self.hydraJobs.binaryTarball."aarch64-darwin" - # Cross - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" - ]; - installerScriptForGHA = installScriptFor [ - # Native - self.hydraJobs.binaryTarball."x86_64-linux" - self.hydraJobs.binaryTarball."x86_64-darwin" - # Cross - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" - ]; - - # docker image with Nix inside - dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); - - # Line coverage analysis. - coverage = nixpkgsFor.x86_64-linux.native.nix.override { - pname = "nix-coverage"; - withCoverageChecks = true; - }; - - # API docs for Nix's unstable internal C++ interfaces. - internal-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ./package.nix { - inherit fileset; - doBuild = false; - enableInternalAPIDocs = true; - }; - - # System tests. - tests = import ./tests/nixos { inherit lib nixpkgs nixpkgsFor; } // { - - # Make sure that nix-env still produces the exact same result - # on a particular version of Nixpkgs. - evalNixpkgs = - let - inherit (nixpkgsFor.x86_64-linux.native) runCommand nix; - in - runCommand "eval-nixos" { buildInputs = [ nix ]; } - '' - type -p nix-env - # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593. - time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages - [[ $(sha1sum < packages | cut -c1-40) = ff451c521e61e4fe72bdbe2d0ca5d1809affa733 ]] - mkdir $out - ''; - - nixpkgsLibTests = - forAllSystems (system: - import (nixpkgs + "/lib/tests/release.nix") - { pkgs = nixpkgsFor.${system}.native; - nixVersions = [ self.packages.${system}.nix ]; - } - ); - }; - - metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" { - pkgs = nixpkgsFor.x86_64-linux.native; - nixpkgs = nixpkgs-regression; - }; - - installTests = forAllSystems (system: - let pkgs = nixpkgsFor.${system}.native; in - pkgs.runCommand "install-tests" { - againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix; - againstCurrentUnstable = - # FIXME: temporarily disable this on macOS because of #3605. - if system == "x86_64-linux" - then testNixVersions pkgs pkgs.nix pkgs.nixUnstable - else null; - # Disabled because the latest stable version doesn't handle - # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work - # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable; - } "touch $out"); - - installerTests = import ./tests/installer { - binaryTarballs = self.hydraJobs.binaryTarball; - inherit nixpkgsFor; - }; - + hydraJobs = import ./packaging/hydra.nix { + inherit + inputs + binaryTarball + forAllCrossSystems + forAllSystems + lib + linux64BitSystems + nixpkgsFor + self + ; }; checks = forAllSystems (system: { binaryTarball = self.hydraJobs.binaryTarball.${system}; - perlBindings = self.hydraJobs.perlBindings.${system}; installTests = self.hydraJobs.installTests.${system}; nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system}; rl-next = let pkgs = nixpkgsFor.${system}.native; in pkgs.buildPackages.runCommand "test-rl-next-release-notes" { } '' - LANG=C.UTF-8 ${pkgs.changelog-d-nix}/bin/changelog-d ${./doc/manual/rl-next} >$out + LANG=C.UTF-8 ${pkgs.changelog-d}/bin/changelog-d ${./doc/manual/rl-next} >$out ''; + repl-completion = nixpkgsFor.${system}.native.callPackage ./tests/repl-completion.nix { }; } // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) { dockerImage = self.hydraJobs.dockerImage.${system}; - }); + } // (lib.optionalAttrs (!(builtins.elem system linux32BitSystems))) { + # Some perl dependencies are broken on i686-linux. + # Since the support is only best-effort there, disable the perl + # bindings - packages = forAllSystems (system: rec { - inherit (nixpkgsFor.${system}.native) nix changelog-d-nix; - default = nix; - } // (lib.optionalAttrs (builtins.elem system linux64BitSystems) { - nix-static = nixpkgsFor.${system}.static.nix; + # Temporarily disabled because GitHub Actions OOM issues. Once + # the old build system is gone and we are back to one build + # system, we should reenable this. + #perlBindings = self.hydraJobs.perlBindings.${system}; + } + # Add "passthru" tests + // flatMapAttrs ({ + "" = nixpkgsFor.${system}.native; + } // lib.optionalAttrs (! nixpkgsFor.${system}.native.stdenv.hostPlatform.isDarwin) { + # TODO: enable static builds for darwin, blocked on: + # https://github.com/NixOS/nixpkgs/issues/320448 + "static-" = nixpkgsFor.${system}.static; + }) + (nixpkgsPrefix: nixpkgs: + flatMapAttrs nixpkgs.nixComponents + (pkgName: pkg: + flatMapAttrs pkg.tests or {} + (testName: test: { + "${nixpkgsPrefix}${pkgName}-${testName}" = test; + }) + ) + ) + // devFlake.checks.${system} or {} + ); + + packages = forAllSystems (system: + { # Here we put attributes that map 1:1 into packages., ie + # for which we don't apply the full build matrix such as cross or static. + inherit (nixpkgsFor.${system}.native) + changelog-d; + default = self.packages.${system}.nix; + nix-internal-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-internal-api-docs; + nix-external-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-external-api-docs; + } + # We need to flatten recursive attribute sets of derivations to pass `flake check`. + // flatMapAttrs + { # Components we'll iterate over in the upcoming lambda + "nix" = { }; + # Temporarily disabled because GitHub Actions OOM issues. Once + # the old build system is gone and we are back to one build + # system, we should reenable these. + #"nix-util" = { }; + #"nix-store" = { }; + #"nix-fetchers" = { }; + } + (pkgName: {}: { + # These attributes go right into `packages.`. + "${pkgName}" = nixpkgsFor.${system}.native.nixComponents.${pkgName}; + "${pkgName}-static" = nixpkgsFor.${system}.static.nixComponents.${pkgName}; + } + // flatMapAttrs (lib.genAttrs crossSystems (_: { })) (crossSystem: {}: { + # These attributes go right into `packages.`. + "${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName}; + }) + // flatMapAttrs (lib.genAttrs stdenvs (_: { })) (stdenvName: {}: { + # These attributes go right into `packages.`. + "${pkgName}-${stdenvName}" = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nixComponents.${pkgName}; + }) + ) + // lib.optionalAttrs (builtins.elem system linux64BitSystems) { dockerImage = let pkgs = nixpkgsFor.${system}.native; @@ -372,21 +264,27 @@ ln -s ${image} $image echo "file binary-dist $image" >> $out/nix-support/hydra-build-products ''; - } // builtins.listToAttrs (map - (crossSystem: { - name = "nix-${crossSystem}"; - value = nixpkgsFor.${system}.cross.${crossSystem}.nix; - }) - crossSystems) - // builtins.listToAttrs (map - (stdenvName: { - name = "nix-${stdenvName}"; - value = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nix; - }) - stdenvs))); + }); devShells = let - makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; forDevShell = true; }).overrideAttrs (attrs: { + makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; forDevShell = true; }).overrideAttrs (attrs: + let + modular = devFlake.getSystem stdenv.buildPlatform.system; + transformFlag = prefix: flag: + assert builtins.isString flag; + let + rest = builtins.substring 2 (builtins.stringLength flag) flag; + in + "-D${prefix}:${rest}"; + havePerl = stdenv.buildPlatform == stdenv.hostPlatform && stdenv.hostPlatform.isUnix; + ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags; + in { + pname = "shell-for-" + attrs.pname; + + # Remove the version suffix to avoid unnecessary attempts to substitute in nix develop + version = lib.fileContents ./.version; + name = attrs.pname; + installFlags = "sysconfdir=$(out)/etc"; shellHook = '' PATH=$prefix/bin:$PATH @@ -397,11 +295,62 @@ XDG_DATA_DIRS+=:$out/share ''; + # We use this shell with the local checkout, not unpackPhase. + src = null; + + env = { + # Needed for Meson to find Boost. + # https://github.com/NixOS/nixpkgs/issues/86131. + BOOST_INCLUDEDIR = "${lib.getDev pkgs.nixDependencies.boost}/include"; + BOOST_LIBRARYDIR = "${lib.getLib pkgs.nixDependencies.boost}/lib"; + # For `make format`, to work without installing pre-commit + _NIX_PRE_COMMIT_HOOKS_CONFIG = + "${(pkgs.formats.yaml { }).generate "pre-commit-config.yaml" modular.pre-commit.settings.rawConfig}"; + }; + + mesonFlags = + map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents.nix-util.mesonFlags) + ++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents.nix-store.mesonFlags) + ++ map (transformFlag "libfetchers") (ignoreCrossFile pkgs.nixComponents.nix-fetchers.mesonFlags) + ++ lib.optionals havePerl (map (transformFlag "perl") (ignoreCrossFile pkgs.nixComponents.nix-perl-bindings.mesonFlags)) + ++ map (transformFlag "libexpr") (ignoreCrossFile pkgs.nixComponents.nix-expr.mesonFlags) + ++ map (transformFlag "libcmd") (ignoreCrossFile pkgs.nixComponents.nix-cmd.mesonFlags) + ; + nativeBuildInputs = attrs.nativeBuildInputs or [] + ++ pkgs.nixComponents.nix-util.nativeBuildInputs + ++ pkgs.nixComponents.nix-store.nativeBuildInputs + ++ pkgs.nixComponents.nix-fetchers.nativeBuildInputs + ++ lib.optionals havePerl pkgs.nixComponents.nix-perl-bindings.nativeBuildInputs + ++ pkgs.nixComponents.nix-internal-api-docs.nativeBuildInputs + ++ pkgs.nixComponents.nix-external-api-docs.nativeBuildInputs + ++ lib.optional + (!stdenv.buildPlatform.canExecute stdenv.hostPlatform + # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479 + && !(stdenv.hostPlatform.isWindows && stdenv.buildPlatform.isDarwin) + && stdenv.hostPlatform.emulatorAvailable pkgs.buildPackages + && lib.meta.availableOn stdenv.buildPlatform (stdenv.hostPlatform.emulator pkgs.buildPackages)) + pkgs.buildPackages.mesonEmulatorHook + ++ [ + pkgs.buildPackages.cmake + pkgs.buildPackages.shellcheck + pkgs.buildPackages.changelog-d + modular.pre-commit.settings.package + (pkgs.writeScriptBin "pre-commit-hooks-install" + modular.pre-commit.settings.installationScript) + ] # TODO: Remove the darwin check once # https://github.com/NixOS/nixpkgs/pull/291814 is available ++ lib.optional (stdenv.cc.isClang && !stdenv.buildPlatform.isDarwin) pkgs.buildPackages.bear ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) pkgs.buildPackages.clang-tools; + + buildInputs = attrs.buildInputs or [] + ++ [ + pkgs.gtest + pkgs.rapidcheck + ] + ++ lib.optional havePerl pkgs.perl + ; }); in forAllSystems (system: @@ -413,8 +362,8 @@ in (makeShells "native" nixpkgsFor.${system}.native) // (lib.optionalAttrs (!nixpkgsFor.${system}.native.stdenv.isDarwin) - (makeShells "static" nixpkgsFor.${system}.static)) // - (lib.genAttrs shellCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv)) // + (makeShells "static" nixpkgsFor.${system}.static) // + (forAllCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv))) // { default = self.devShells.${system}.native-stdenvPackages; } diff --git a/local.mk b/local.mk index 3f3abb9f0..b27c7031e 100644 --- a/local.mk +++ b/local.mk @@ -2,9 +2,14 @@ GLOBAL_CXXFLAGS += -Wno-deprecated-declarations -Werror=switch # Allow switch-enum to be overridden for files that do not support it, usually because of dependency headers. ERROR_SWITCH_ENUM = -Werror=switch-enum -$(foreach i, config.h $(wildcard src/lib*/*.hh), \ +$(foreach i, config.h $(wildcard src/lib*/*.hh) $(filter-out %_internal.h, $(wildcard src/lib*c/*.h)), \ $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) +ifdef HOST_UNIX + $(foreach i, $(wildcard src/lib*/unix/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) +endif + $(GCH): src/libutil/util.hh config.h -GCH_CXXFLAGS = -I src/libutil +GCH_CXXFLAGS = $(INCLUDE_libutil) diff --git a/m4/gcc_bug_80431.m4 b/m4/gcc_bug_80431.m4 index e42f01956..cdc4ddb40 100644 --- a/m4/gcc_bug_80431.m4 +++ b/m4/gcc_bug_80431.m4 @@ -46,11 +46,13 @@ AC_DEFUN([ENSURE_NO_GCC_BUG_80431], ]])], [status_80431=0], [status_80431=$?], - [ - # Assume we're bug-free when cross-compiling - ]) + [status_80431='']) AC_LANG_POP(C++) AS_CASE([$status_80431], + [''],[ + AC_MSG_RESULT(cannot check because cross compiling) + AC_MSG_NOTICE(assume we are bug free) + ], [0],[ AC_MSG_RESULT(yes) ], diff --git a/maintainers/README.md b/maintainers/README.md index fa321c7c0..b92833497 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -30,17 +30,18 @@ We aim to achieve this by improving the contributor experience and attracting mo ## Members - Eelco Dolstra (@edolstra) – Team lead -- Théophane Hufschmitt (@thufschmitt) - Valentin Gagarin (@fricklerhandwerk) - Thomas Bereknyei (@tomberek) - Robert Hensing (@roberth) - John Ericson (@Ericson2314) +The team is on Github as [@NixOS/nix-team](https://github.com/orgs/NixOS/teams/nix-team). + ## Meeting protocol -The team meets twice a week: +The team meets twice a week (times are denoted in the [Europe/Amsterdam](https://en.m.wikipedia.org/wiki/Time_in_the_Netherlands) time zone): -- Discussion meeting: [Fridays 13:00-14:00 CET](https://calendar.google.com/calendar/event?eid=MHNtOGVuNWtrZXNpZHR2bW1sM3QyN2ZjaGNfMjAyMjExMjVUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) +- Discussion meeting: [Wednesday 21:00-22:00 Europe/Amsterdam](https://www.google.com/calendar/event?eid=ZG5rZzNyajRjajducGV2NGY5aGkzYWIwdnJfMjAyNDA1MDhUMTkwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) 1. Triage issues and pull requests from the [No Status](#no-status) column (30 min) 2. Discuss issues and pull requests from the [To discuss](#to-discuss) column (30 min). @@ -49,15 +50,19 @@ The team meets twice a week: - mark it as draft if it is blocked on the contributor - escalate it back to the team by moving it to To discuss, and leaving a comment as to why the issue needs to be discussed again. -- Work meeting: [Mondays 13:00-15:00 CET](https://calendar.google.com/calendar/event?eid=NTM1MG1wNGJnOGpmOTZhYms3bTB1bnY5cWxfMjAyMjExMjFUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) +- Work meeting: [Mondays 14:00-16:00 Europe/Amsterdam](https://www.google.com/calendar/event?eid=Ym52NDdzYnRic2NzcDcybjZiNDhpNzhpa3NfMjAyNDA1MTNUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) 1. Code review on pull requests from [In review](#in-review). 2. Other chores and tasks. Meeting notes are collected on a [collaborative scratchpad](https://pad.lassul.us/Cv7FpYx-Ri-4VjUykQOLAw). -Notes on issues and pull requests are posted as comments and linked from the meeting notes, so they are easy to find from both places. +Notes on issues and pull requests are posted as comments and linked from the meeting notes, so they are can be found from both places. [All meeting notes](https://discourse.nixos.org/search?expanded=true&q=Nix%20team%20meeting%20minutes%20%23%20%23dev%3Anix%20in%3Atitle%20order%3Alatest_topic) are published on Discourse under the [Nix category](https://discourse.nixos.org/c/dev/nix/50). +Team meetings are generally open to anyone interested. +We can make exceptions to discuss sensitive issues, such as security incidents or people matters. +Contact any team member to get a calendar invite for reminders and updates. + ## Project board protocol The team uses a [GitHub project board](https://github.com/orgs/NixOS/projects/19/views/1) for tracking its work. diff --git a/maintainers/data/release-credits-email-to-handle.json b/maintainers/data/release-credits-email-to-handle.json new file mode 100644 index 000000000..cddc1a6e7 --- /dev/null +++ b/maintainers/data/release-credits-email-to-handle.json @@ -0,0 +1,52 @@ +{ + "bogus": "bogus", + "edolstra@gmail.com": "edolstra", + "roberth@users.noreply.github.com": "roberth", + "toscano.pino@tiscali.it": "pinotree", + "valentin@gagarin.work": "fricklerhandwerk", + "mr.trubach@icloud.com": "tie", + "robert@roberthensing.nl": "roberth", + "lix@jade.fyi": "lf-", + "cole.e.helbling@outlook.com": "cole-h", + "joerg@thalheim.io": "Mic92", + "John.Ericson@Obsidian.Systems": "Ericson2314", + "ryan.hendrickson@alum.mit.edu": "rhendric", + "67135060+poweredbypie@users.noreply.github.com": "poweredbypie", + "detroyejr@outlook.com": "detroyejr", + "silvan.mosberger@tweag.io": "infinisil", + "vcs@emily.moe": "emilazy", + "farid.m.zakaria@gmail.com": "fzakaria", + "22859658+RTUnreal@users.noreply.github.com": "RTUnreal", + "me@las.rs": "L-as", + "philip.taron@gmail.com": "philiptaron", + "root@goldstein.rs": "GoldsteinE", + "tomberek@users.noreply.github.com": "tomberek", + "lexi.mattick@neuralink.com": "kognise", + "andrew@johnandrewmarshall.com": "amarshall", + "contact@romain-neil.fr": "romain-neil", + "Mic92@users.noreply.github.com": "Mic92", + "valentin.gagarin@tweag.io": "fricklerhandwerk", + "siddhantk232@gmail.com": "siddhantk232", + "kn@openbsd.org": "klemensn", + "slyich@gmail.com": "trofi", + "theophane.hufschmitt@tweag.io": "thufschmitt", + "alicebob@lijzij.de": "alicebob", + "winter@winter.cafe": "winterqt", + "brian@brianmckenna.org": "puffnfresh", + "git@haenoe.party": "haenoe", + "peshogo@gmail.com": "pineapplehunter", + "poweredbypie@users.noreply.github.com": "poweredbypie", + "arthur200126@gmail.com": "Artoria2e5", + "tomberek@gmail.com": "tomberek", + "jaredbaur@fastmail.com": "jmbaur", + "andreas@rammhold.de": "andir", + "hamirmahal@gmail.com": "hamirmahal", + "git@JohnEricson.me": "Ericson2314", + "8763518+SkamDart@users.noreply.github.com": "SkamDart", + "kirillrdy@gmail.com": "kirillrdy", + "pennae@lix.systems": "pennae", + "delroth@gmail.com": "delroth", + "enno@nerdworks.de": "elohmeier", + "mjbauer95@gmail.com": "matthewbauer", + "MostAwesomeDude@gmail.com": "MostAwesomeDude" +} \ No newline at end of file diff --git a/maintainers/data/release-credits-handle-to-name.json b/maintainers/data/release-credits-handle-to-name.json new file mode 100644 index 000000000..abf9ed05b --- /dev/null +++ b/maintainers/data/release-credits-handle-to-name.json @@ -0,0 +1,45 @@ +{ + "fzakaria": "Farid Zakaria", + "kognise": "Lexi Mattick", + "L-as": "Las Safin", + "haenoe": "HaeNoe", + "andir": "Andreas Rammhold", + "matthewbauer": "Matthew Bauer", + "emilazy": "Emily", + "pineapplehunter": "Shogo Takata", + "RTUnreal": null, + "jmbaur": "Jared Baur", + "Ericson2314": "John Ericson", + "pinotree": "Pino Toscano", + "tie": "Ivan Trubach", + "poweredbypie": null, + "fricklerhandwerk": "Valentin Gagarin", + "Mic92": "J\u00f6rg Thalheim", + "alicebob": "Harmen", + "elohmeier": "Enno Richter", + "delroth": "Pierre Bourdon", + "kirillrdy": null, + "thufschmitt": "Th\u00e9ophane Hufschmitt", + "detroyejr": "Jonathan De Troye", + "klemensn": "Klemens Nanni", + "tomberek": null, + "rhendric": "Ryan Hendrickson", + "philiptaron": "Philip Taron", + "puffnfresh": "Brian McKenna", + "lf-": "jade", + "romain-neil": "Romain Neil", + "hamirmahal": "Hamir Mahal", + "edolstra": "Eelco Dolstra", + "Artoria2e5": "Mingye Wang", + "SkamDart": "Cameron", + "roberth": "Robert Hensing", + "amarshall": "Andrew Marshall", + "trofi": "Sergei Trofimovich", + "cole-h": "Cole Helbling", + "infinisil": "Silvan Mosberger", + "siddhantk232": "Siddhant Kumar", + "winterqt": "Winter", + "GoldsteinE": "Max \u201cGoldstein\u201d Siling", + "pennae": null, + "MostAwesomeDude": "Corbin Simpson" +} \ No newline at end of file diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix new file mode 100644 index 000000000..be91df536 --- /dev/null +++ b/maintainers/flake-module.nix @@ -0,0 +1,677 @@ +{ lib, getSystem, inputs, ... }: + +{ + imports = [ + inputs.git-hooks-nix.flakeModule + ]; + + perSystem = { config, pkgs, ... }: { + + # https://flake.parts/options/pre-commit-hooks-nix.html#options + pre-commit.settings = { + hooks = { + clang-format = { + enable = true; + excludes = [ + # We don't want to format test data + # ''tests/(?!nixos/).*\.nix'' + ''^tests/unit/[^/]*/data/.*$'' + + # Don't format vendored code + ''^doc/manual/redirects\.js$'' + ''^doc/manual/theme/highlight\.js$'' + + # We haven't applied formatting to these files yet + ''^doc/manual/redirects\.js$'' + ''^doc/manual/theme/highlight\.js$'' + ''^precompiled-headers\.h$'' + ''^src/build-remote/build-remote\.cc$'' + ''^src/libcmd/built-path\.cc$'' + ''^src/libcmd/built-path\.hh$'' + ''^src/libcmd/command\.cc$'' + ''^src/libcmd/command\.hh$'' + ''^src/libcmd/common-eval-args\.cc$'' + ''^src/libcmd/common-eval-args\.hh$'' + ''^src/libcmd/editor-for\.cc$'' + ''^src/libcmd/installable-attr-path\.cc$'' + ''^src/libcmd/installable-attr-path\.hh$'' + ''^src/libcmd/installable-derived-path\.cc$'' + ''^src/libcmd/installable-derived-path\.hh$'' + ''^src/libcmd/installable-flake\.cc$'' + ''^src/libcmd/installable-flake\.hh$'' + ''^src/libcmd/installable-value\.cc$'' + ''^src/libcmd/installable-value\.hh$'' + ''^src/libcmd/installables\.cc$'' + ''^src/libcmd/installables\.hh$'' + ''^src/libcmd/legacy\.hh$'' + ''^src/libcmd/markdown\.cc$'' + ''^src/libcmd/misc-store-flags\.cc$'' + ''^src/libcmd/repl-interacter\.cc$'' + ''^src/libcmd/repl-interacter\.hh$'' + ''^src/libcmd/repl\.cc$'' + ''^src/libcmd/repl\.hh$'' + ''^src/libexpr-c/nix_api_expr\.cc$'' + ''^src/libexpr-c/nix_api_external\.cc$'' + ''^src/libexpr/attr-path\.cc$'' + ''^src/libexpr/attr-path\.hh$'' + ''^src/libexpr/attr-set\.cc$'' + ''^src/libexpr/attr-set\.hh$'' + ''^src/libexpr/eval-cache\.cc$'' + ''^src/libexpr/eval-cache\.hh$'' + ''^src/libexpr/eval-error\.cc$'' + ''^src/libexpr/eval-inline\.hh$'' + ''^src/libexpr/eval-settings\.cc$'' + ''^src/libexpr/eval-settings\.hh$'' + ''^src/libexpr/eval\.cc$'' + ''^src/libexpr/eval\.hh$'' + ''^src/libexpr/function-trace\.cc$'' + ''^src/libexpr/gc-small-vector\.hh$'' + ''^src/libexpr/get-drvs\.cc$'' + ''^src/libexpr/get-drvs\.hh$'' + ''^src/libexpr/json-to-value\.cc$'' + ''^src/libexpr/nixexpr\.cc$'' + ''^src/libexpr/nixexpr\.hh$'' + ''^src/libexpr/parser-state\.hh$'' + ''^src/libexpr/pos-table\.hh$'' + ''^src/libexpr/primops\.cc$'' + ''^src/libexpr/primops\.hh$'' + ''^src/libexpr/primops/context\.cc$'' + ''^src/libexpr/primops/fetchClosure\.cc$'' + ''^src/libexpr/primops/fetchMercurial\.cc$'' + ''^src/libexpr/primops/fetchTree\.cc$'' + ''^src/libexpr/primops/fromTOML\.cc$'' + ''^src/libexpr/print-ambiguous\.cc$'' + ''^src/libexpr/print-ambiguous\.hh$'' + ''^src/libexpr/print-options\.hh$'' + ''^src/libexpr/print\.cc$'' + ''^src/libexpr/print\.hh$'' + ''^src/libexpr/search-path\.cc$'' + ''^src/libexpr/symbol-table\.hh$'' + ''^src/libexpr/value-to-json\.cc$'' + ''^src/libexpr/value-to-json\.hh$'' + ''^src/libexpr/value-to-xml\.cc$'' + ''^src/libexpr/value-to-xml\.hh$'' + ''^src/libexpr/value\.hh$'' + ''^src/libexpr/value/context\.cc$'' + ''^src/libexpr/value/context\.hh$'' + ''^src/libfetchers/attrs\.cc$'' + ''^src/libfetchers/cache\.cc$'' + ''^src/libfetchers/cache\.hh$'' + ''^src/libfetchers/fetch-settings\.cc$'' + ''^src/libfetchers/fetch-settings\.hh$'' + ''^src/libfetchers/fetch-to-store\.cc$'' + ''^src/libfetchers/fetchers\.cc$'' + ''^src/libfetchers/fetchers\.hh$'' + ''^src/libfetchers/filtering-source-accessor\.cc$'' + ''^src/libfetchers/filtering-source-accessor\.hh$'' + ''^src/libfetchers/fs-source-accessor\.cc$'' + ''^src/libfetchers/fs-source-accessor\.hh$'' + ''^src/libfetchers/git-utils\.cc$'' + ''^src/libfetchers/git-utils\.hh$'' + ''^src/libfetchers/github\.cc$'' + ''^src/libfetchers/indirect\.cc$'' + ''^src/libfetchers/memory-source-accessor\.cc$'' + ''^src/libfetchers/path\.cc$'' + ''^src/libfetchers/registry\.cc$'' + ''^src/libfetchers/registry\.hh$'' + ''^src/libfetchers/tarball\.cc$'' + ''^src/libfetchers/tarball\.hh$'' + ''^src/libfetchers/git\.cc$'' + ''^src/libfetchers/mercurial\.cc$'' + ''^src/libflake/flake/config\.cc$'' + ''^src/libflake/flake/flake\.cc$'' + ''^src/libflake/flake/flake\.hh$'' + ''^src/libflake/flake/flakeref\.cc$'' + ''^src/libflake/flake/flakeref\.hh$'' + ''^src/libflake/flake/lockfile\.cc$'' + ''^src/libflake/flake/lockfile\.hh$'' + ''^src/libflake/flake/url-name\.cc$'' + ''^src/libmain/common-args\.cc$'' + ''^src/libmain/common-args\.hh$'' + ''^src/libmain/loggers\.cc$'' + ''^src/libmain/loggers\.hh$'' + ''^src/libmain/progress-bar\.cc$'' + ''^src/libmain/shared\.cc$'' + ''^src/libmain/shared\.hh$'' + ''^src/libmain/unix/stack\.cc$'' + ''^src/libstore/binary-cache-store\.cc$'' + ''^src/libstore/binary-cache-store\.hh$'' + ''^src/libstore/build-result\.hh$'' + ''^src/libstore/builtins\.hh$'' + ''^src/libstore/builtins/buildenv\.cc$'' + ''^src/libstore/builtins/buildenv\.hh$'' + ''^src/libstore/common-protocol-impl\.hh$'' + ''^src/libstore/common-protocol\.cc$'' + ''^src/libstore/common-protocol\.hh$'' + ''^src/libstore/common-ssh-store-config\.hh$'' + ''^src/libstore/content-address\.cc$'' + ''^src/libstore/content-address\.hh$'' + ''^src/libstore/daemon\.cc$'' + ''^src/libstore/daemon\.hh$'' + ''^src/libstore/derivations\.cc$'' + ''^src/libstore/derivations\.hh$'' + ''^src/libstore/derived-path-map\.cc$'' + ''^src/libstore/derived-path-map\.hh$'' + ''^src/libstore/derived-path\.cc$'' + ''^src/libstore/derived-path\.hh$'' + ''^src/libstore/downstream-placeholder\.cc$'' + ''^src/libstore/downstream-placeholder\.hh$'' + ''^src/libstore/dummy-store\.cc$'' + ''^src/libstore/export-import\.cc$'' + ''^src/libstore/filetransfer\.cc$'' + ''^src/libstore/filetransfer\.hh$'' + ''^src/libstore/gc-store\.hh$'' + ''^src/libstore/globals\.cc$'' + ''^src/libstore/globals\.hh$'' + ''^src/libstore/http-binary-cache-store\.cc$'' + ''^src/libstore/legacy-ssh-store\.cc$'' + ''^src/libstore/legacy-ssh-store\.hh$'' + ''^src/libstore/length-prefixed-protocol-helper\.hh$'' + ''^src/libstore/linux/personality\.cc$'' + ''^src/libstore/linux/personality\.hh$'' + ''^src/libstore/local-binary-cache-store\.cc$'' + ''^src/libstore/local-fs-store\.cc$'' + ''^src/libstore/local-fs-store\.hh$'' + ''^src/libstore/log-store\.cc$'' + ''^src/libstore/log-store\.hh$'' + ''^src/libstore/machines\.cc$'' + ''^src/libstore/machines\.hh$'' + ''^src/libstore/make-content-addressed\.cc$'' + ''^src/libstore/make-content-addressed\.hh$'' + ''^src/libstore/misc\.cc$'' + ''^src/libstore/names\.cc$'' + ''^src/libstore/names\.hh$'' + ''^src/libstore/nar-accessor\.cc$'' + ''^src/libstore/nar-accessor\.hh$'' + ''^src/libstore/nar-info-disk-cache\.cc$'' + ''^src/libstore/nar-info-disk-cache\.hh$'' + ''^src/libstore/nar-info\.cc$'' + ''^src/libstore/nar-info\.hh$'' + ''^src/libstore/outputs-spec\.cc$'' + ''^src/libstore/outputs-spec\.hh$'' + ''^src/libstore/parsed-derivations\.cc$'' + ''^src/libstore/path-info\.cc$'' + ''^src/libstore/path-info\.hh$'' + ''^src/libstore/path-references\.cc$'' + ''^src/libstore/path-regex\.hh$'' + ''^src/libstore/path-with-outputs\.cc$'' + ''^src/libstore/path\.cc$'' + ''^src/libstore/path\.hh$'' + ''^src/libstore/pathlocks\.cc$'' + ''^src/libstore/pathlocks\.hh$'' + ''^src/libstore/profiles\.cc$'' + ''^src/libstore/profiles\.hh$'' + ''^src/libstore/realisation\.cc$'' + ''^src/libstore/realisation\.hh$'' + ''^src/libstore/remote-fs-accessor\.cc$'' + ''^src/libstore/remote-fs-accessor\.hh$'' + ''^src/libstore/remote-store-connection\.hh$'' + ''^src/libstore/remote-store\.cc$'' + ''^src/libstore/remote-store\.hh$'' + ''^src/libstore/s3-binary-cache-store\.cc$'' + ''^src/libstore/s3\.hh$'' + ''^src/libstore/serve-protocol-impl\.cc$'' + ''^src/libstore/serve-protocol-impl\.hh$'' + ''^src/libstore/serve-protocol\.cc$'' + ''^src/libstore/serve-protocol\.hh$'' + ''^src/libstore/sqlite\.cc$'' + ''^src/libstore/sqlite\.hh$'' + ''^src/libstore/ssh-store\.cc$'' + ''^src/libstore/ssh\.cc$'' + ''^src/libstore/ssh\.hh$'' + ''^src/libstore/store-api\.cc$'' + ''^src/libstore/store-api\.hh$'' + ''^src/libstore/store-dir-config\.hh$'' + ''^src/libstore/build/derivation-goal\.cc$'' + ''^src/libstore/build/derivation-goal\.hh$'' + ''^src/libstore/build/drv-output-substitution-goal\.cc$'' + ''^src/libstore/build/drv-output-substitution-goal\.hh$'' + ''^src/libstore/build/entry-points\.cc$'' + ''^src/libstore/build/goal\.cc$'' + ''^src/libstore/build/goal\.hh$'' + ''^src/libstore/unix/build/hook-instance\.cc$'' + ''^src/libstore/unix/build/local-derivation-goal\.cc$'' + ''^src/libstore/unix/build/local-derivation-goal\.hh$'' + ''^src/libstore/build/substitution-goal\.cc$'' + ''^src/libstore/build/substitution-goal\.hh$'' + ''^src/libstore/build/worker\.cc$'' + ''^src/libstore/build/worker\.hh$'' + ''^src/libstore/builtins/fetchurl\.cc$'' + ''^src/libstore/builtins/unpack-channel\.cc$'' + ''^src/libstore/gc\.cc$'' + ''^src/libstore/local-overlay-store\.cc$'' + ''^src/libstore/local-overlay-store\.hh$'' + ''^src/libstore/local-store\.cc$'' + ''^src/libstore/local-store\.hh$'' + ''^src/libstore/unix/user-lock\.cc$'' + ''^src/libstore/unix/user-lock\.hh$'' + ''^src/libstore/optimise-store\.cc$'' + ''^src/libstore/unix/pathlocks\.cc$'' + ''^src/libstore/posix-fs-canonicalise\.cc$'' + ''^src/libstore/posix-fs-canonicalise\.hh$'' + ''^src/libstore/uds-remote-store\.cc$'' + ''^src/libstore/uds-remote-store\.hh$'' + ''^src/libstore/windows/build\.cc$'' + ''^src/libstore/worker-protocol-impl\.hh$'' + ''^src/libstore/worker-protocol\.cc$'' + ''^src/libstore/worker-protocol\.hh$'' + ''^src/libutil-c/nix_api_util_internal\.h$'' + ''^src/libutil/archive\.cc$'' + ''^src/libutil/archive\.hh$'' + ''^src/libutil/args\.cc$'' + ''^src/libutil/args\.hh$'' + ''^src/libutil/args/root\.hh$'' + ''^src/libutil/callback\.hh$'' + ''^src/libutil/canon-path\.cc$'' + ''^src/libutil/canon-path\.hh$'' + ''^src/libutil/chunked-vector\.hh$'' + ''^src/libutil/closure\.hh$'' + ''^src/libutil/comparator\.hh$'' + ''^src/libutil/compute-levels\.cc$'' + ''^src/libutil/config-impl\.hh$'' + ''^src/libutil/config\.cc$'' + ''^src/libutil/config\.hh$'' + ''^src/libutil/current-process\.cc$'' + ''^src/libutil/current-process\.hh$'' + ''^src/libutil/english\.cc$'' + ''^src/libutil/english\.hh$'' + ''^src/libutil/environment-variables\.cc$'' + ''^src/libutil/error\.cc$'' + ''^src/libutil/error\.hh$'' + ''^src/libutil/exit\.hh$'' + ''^src/libutil/experimental-features\.cc$'' + ''^src/libutil/experimental-features\.hh$'' + ''^src/libutil/file-content-address\.cc$'' + ''^src/libutil/file-content-address\.hh$'' + ''^src/libutil/file-descriptor\.cc$'' + ''^src/libutil/file-descriptor\.hh$'' + ''^src/libutil/file-path-impl\.hh$'' + ''^src/libutil/file-path\.hh$'' + ''^src/libutil/file-system\.cc$'' + ''^src/libutil/file-system\.hh$'' + ''^src/libutil/finally\.hh$'' + ''^src/libutil/fmt\.hh$'' + ''^src/libutil/fs-sink\.cc$'' + ''^src/libutil/fs-sink\.hh$'' + ''^src/libutil/git\.cc$'' + ''^src/libutil/git\.hh$'' + ''^src/libutil/hash\.cc$'' + ''^src/libutil/hash\.hh$'' + ''^src/libutil/hilite\.cc$'' + ''^src/libutil/hilite\.hh$'' + ''^src/libutil/source-accessor\.hh$'' + ''^src/libutil/json-impls\.hh$'' + ''^src/libutil/json-utils\.cc$'' + ''^src/libutil/json-utils\.hh$'' + ''^src/libutil/linux/cgroup\.cc$'' + ''^src/libutil/linux/namespaces\.cc$'' + ''^src/libutil/logging\.cc$'' + ''^src/libutil/logging\.hh$'' + ''^src/libutil/lru-cache\.hh$'' + ''^src/libutil/memory-source-accessor\.cc$'' + ''^src/libutil/memory-source-accessor\.hh$'' + ''^src/libutil/pool\.hh$'' + ''^src/libutil/position\.cc$'' + ''^src/libutil/position\.hh$'' + ''^src/libutil/posix-source-accessor\.cc$'' + ''^src/libutil/posix-source-accessor\.hh$'' + ''^src/libutil/processes\.hh$'' + ''^src/libutil/ref\.hh$'' + ''^src/libutil/references\.cc$'' + ''^src/libutil/references\.hh$'' + ''^src/libutil/regex-combinators\.hh$'' + ''^src/libutil/serialise\.cc$'' + ''^src/libutil/serialise\.hh$'' + ''^src/libutil/signals\.hh$'' + ''^src/libutil/signature/local-keys\.cc$'' + ''^src/libutil/signature/local-keys\.hh$'' + ''^src/libutil/signature/signer\.cc$'' + ''^src/libutil/signature/signer\.hh$'' + ''^src/libutil/source-accessor\.cc$'' + ''^src/libutil/source-accessor\.hh$'' + ''^src/libutil/source-path\.cc$'' + ''^src/libutil/source-path\.hh$'' + ''^src/libutil/split\.hh$'' + ''^src/libutil/suggestions\.cc$'' + ''^src/libutil/suggestions\.hh$'' + ''^src/libutil/sync\.hh$'' + ''^src/libutil/terminal\.cc$'' + ''^src/libutil/terminal\.hh$'' + ''^src/libutil/thread-pool\.cc$'' + ''^src/libutil/thread-pool\.hh$'' + ''^src/libutil/topo-sort\.hh$'' + ''^src/libutil/types\.hh$'' + ''^src/libutil/unix/file-descriptor\.cc$'' + ''^src/libutil/unix/file-path\.cc$'' + ''^src/libutil/unix/monitor-fd\.hh$'' + ''^src/libutil/unix/processes\.cc$'' + ''^src/libutil/unix/signals-impl\.hh$'' + ''^src/libutil/unix/signals\.cc$'' + ''^src/libutil/unix-domain-socket\.cc$'' + ''^src/libutil/unix/users\.cc$'' + ''^src/libutil/url-parts\.hh$'' + ''^src/libutil/url\.cc$'' + ''^src/libutil/url\.hh$'' + ''^src/libutil/users\.cc$'' + ''^src/libutil/users\.hh$'' + ''^src/libutil/util\.cc$'' + ''^src/libutil/util\.hh$'' + ''^src/libutil/variant-wrapper\.hh$'' + ''^src/libutil/windows/environment-variables\.cc$'' + ''^src/libutil/windows/file-descriptor\.cc$'' + ''^src/libutil/windows/file-path\.cc$'' + ''^src/libutil/windows/processes\.cc$'' + ''^src/libutil/windows/users\.cc$'' + ''^src/libutil/windows/windows-error\.cc$'' + ''^src/libutil/windows/windows-error\.hh$'' + ''^src/libutil/xml-writer\.cc$'' + ''^src/libutil/xml-writer\.hh$'' + ''^src/nix-build/nix-build\.cc$'' + ''^src/nix-channel/nix-channel\.cc$'' + ''^src/nix-collect-garbage/nix-collect-garbage\.cc$'' + ''^src/nix-env/buildenv.nix$'' + ''^src/nix-env/nix-env\.cc$'' + ''^src/nix-env/user-env\.cc$'' + ''^src/nix-env/user-env\.hh$'' + ''^src/nix-instantiate/nix-instantiate\.cc$'' + ''^src/nix-store/dotgraph\.cc$'' + ''^src/nix-store/graphml\.cc$'' + ''^src/nix-store/nix-store\.cc$'' + ''^src/nix/add-to-store\.cc$'' + ''^src/nix/app\.cc$'' + ''^src/nix/build\.cc$'' + ''^src/nix/bundle\.cc$'' + ''^src/nix/cat\.cc$'' + ''^src/nix/config-check\.cc$'' + ''^src/nix/config\.cc$'' + ''^src/nix/copy\.cc$'' + ''^src/nix/derivation-add\.cc$'' + ''^src/nix/derivation-show\.cc$'' + ''^src/nix/derivation\.cc$'' + ''^src/nix/develop\.cc$'' + ''^src/nix/diff-closures\.cc$'' + ''^src/nix/dump-path\.cc$'' + ''^src/nix/edit\.cc$'' + ''^src/nix/eval\.cc$'' + ''^src/nix/flake\.cc$'' + ''^src/nix/fmt\.cc$'' + ''^src/nix/hash\.cc$'' + ''^src/nix/log\.cc$'' + ''^src/nix/ls\.cc$'' + ''^src/nix/main\.cc$'' + ''^src/nix/make-content-addressed\.cc$'' + ''^src/nix/nar\.cc$'' + ''^src/nix/optimise-store\.cc$'' + ''^src/nix/path-from-hash-part\.cc$'' + ''^src/nix/path-info\.cc$'' + ''^src/nix/prefetch\.cc$'' + ''^src/nix/profile\.cc$'' + ''^src/nix/realisation\.cc$'' + ''^src/nix/registry\.cc$'' + ''^src/nix/repl\.cc$'' + ''^src/nix/run\.cc$'' + ''^src/nix/run\.hh$'' + ''^src/nix/search\.cc$'' + ''^src/nix/sigs\.cc$'' + ''^src/nix/store-copy-log\.cc$'' + ''^src/nix/store-delete\.cc$'' + ''^src/nix/store-gc\.cc$'' + ''^src/nix/store-info\.cc$'' + ''^src/nix/store-repair\.cc$'' + ''^src/nix/store\.cc$'' + ''^src/nix/unix/daemon\.cc$'' + ''^src/nix/upgrade-nix\.cc$'' + ''^src/nix/verify\.cc$'' + ''^src/nix/why-depends\.cc$'' + + ''^tests/functional/plugins/plugintest\.cc'' + ''^tests/functional/test-libstoreconsumer/main\.cc'' + ''^tests/nixos/ca-fd-leak/sender\.c'' + ''^tests/nixos/ca-fd-leak/smuggler\.c'' + ''^tests/nixos/user-sandboxing/attacker\.c'' + ''^tests/unit/libexpr-support/tests/libexpr\.hh'' + ''^tests/unit/libexpr-support/tests/value/context\.cc'' + ''^tests/unit/libexpr-support/tests/value/context\.hh'' + ''^tests/unit/libexpr/derived-path\.cc'' + ''^tests/unit/libexpr/error_traces\.cc'' + ''^tests/unit/libexpr/eval\.cc'' + ''^tests/unit/libexpr/json\.cc'' + ''^tests/unit/libexpr/main\.cc'' + ''^tests/unit/libexpr/primops\.cc'' + ''^tests/unit/libexpr/search-path\.cc'' + ''^tests/unit/libexpr/trivial\.cc'' + ''^tests/unit/libexpr/value/context\.cc'' + ''^tests/unit/libexpr/value/print\.cc'' + ''^tests/unit/libfetchers/public-key\.cc'' + ''^tests/unit/libflake/flakeref\.cc'' + ''^tests/unit/libflake/url-name\.cc'' + ''^tests/unit/libstore-support/tests/derived-path\.cc'' + ''^tests/unit/libstore-support/tests/derived-path\.hh'' + ''^tests/unit/libstore-support/tests/nix_api_store\.hh'' + ''^tests/unit/libstore-support/tests/outputs-spec\.cc'' + ''^tests/unit/libstore-support/tests/outputs-spec\.hh'' + ''^tests/unit/libstore-support/tests/path\.cc'' + ''^tests/unit/libstore-support/tests/path\.hh'' + ''^tests/unit/libstore-support/tests/protocol\.hh'' + ''^tests/unit/libstore/common-protocol\.cc'' + ''^tests/unit/libstore/content-address\.cc'' + ''^tests/unit/libstore/derivation\.cc'' + ''^tests/unit/libstore/derived-path\.cc'' + ''^tests/unit/libstore/downstream-placeholder\.cc'' + ''^tests/unit/libstore/machines\.cc'' + ''^tests/unit/libstore/nar-info-disk-cache\.cc'' + ''^tests/unit/libstore/nar-info\.cc'' + ''^tests/unit/libstore/outputs-spec\.cc'' + ''^tests/unit/libstore/path-info\.cc'' + ''^tests/unit/libstore/path\.cc'' + ''^tests/unit/libstore/serve-protocol\.cc'' + ''^tests/unit/libstore/worker-protocol\.cc'' + ''^tests/unit/libutil-support/tests/characterization\.hh'' + ''^tests/unit/libutil-support/tests/hash\.cc'' + ''^tests/unit/libutil-support/tests/hash\.hh'' + ''^tests/unit/libutil/args\.cc'' + ''^tests/unit/libutil/canon-path\.cc'' + ''^tests/unit/libutil/chunked-vector\.cc'' + ''^tests/unit/libutil/closure\.cc'' + ''^tests/unit/libutil/compression\.cc'' + ''^tests/unit/libutil/config\.cc'' + ''^tests/unit/libutil/file-content-address\.cc'' + ''^tests/unit/libutil/git\.cc'' + ''^tests/unit/libutil/hash\.cc'' + ''^tests/unit/libutil/hilite\.cc'' + ''^tests/unit/libutil/json-utils\.cc'' + ''^tests/unit/libutil/logging\.cc'' + ''^tests/unit/libutil/lru-cache\.cc'' + ''^tests/unit/libutil/pool\.cc'' + ''^tests/unit/libutil/references\.cc'' + ''^tests/unit/libutil/suggestions\.cc'' + ''^tests/unit/libutil/tests\.cc'' + ''^tests/unit/libutil/url\.cc'' + ''^tests/unit/libutil/xml-writer\.cc'' + ]; + }; + shellcheck = { + enable = true; + excludes = [ + # We haven't linted these files yet + ''^config/install-sh$'' + ''^misc/bash/completion\.sh$'' + ''^misc/fish/completion\.fish$'' + ''^misc/zsh/completion\.zsh$'' + ''^scripts/create-darwin-volume\.sh$'' + ''^scripts/install-darwin-multi-user\.sh$'' + ''^scripts/install-multi-user\.sh$'' + ''^scripts/install-nix-from-closure\.sh$'' + ''^scripts/install-systemd-multi-user\.sh$'' + ''^src/nix/get-env\.sh$'' + ''^tests/functional/build\.sh$'' + ''^tests/functional/ca/build-dry\.sh$'' + ''^tests/functional/ca/build-with-garbage-path\.sh$'' + ''^tests/functional/ca/common\.sh$'' + ''^tests/functional/ca/concurrent-builds\.sh$'' + ''^tests/functional/ca/eval-store\.sh$'' + ''^tests/functional/ca/gc\.sh$'' + ''^tests/functional/ca/import-derivation\.sh$'' + ''^tests/functional/ca/new-build-cmd\.sh$'' + ''^tests/functional/ca/nix-shell\.sh$'' + ''^tests/functional/ca/post-hook\.sh$'' + ''^tests/functional/ca/recursive\.sh$'' + ''^tests/functional/ca/repl\.sh$'' + ''^tests/functional/ca/selfref-gc\.sh$'' + ''^tests/functional/ca/why-depends\.sh$'' + ''^tests/functional/characterisation-test-infra\.sh$'' + ''^tests/functional/check\.sh$'' + ''^tests/functional/common/vars-and-functions\.sh$'' + ''^tests/functional/completions\.sh$'' + ''^tests/functional/compute-levels\.sh$'' + ''^tests/functional/config\.sh$'' + ''^tests/functional/db-migration\.sh$'' + ''^tests/functional/debugger\.sh$'' + ''^tests/functional/dependencies\.builder0\.sh$'' + ''^tests/functional/dependencies\.sh$'' + ''^tests/functional/dump-db\.sh$'' + ''^tests/functional/dyn-drv/build-built-drv\.sh$'' + ''^tests/functional/dyn-drv/common\.sh$'' + ''^tests/functional/dyn-drv/dep-built-drv\.sh$'' + ''^tests/functional/dyn-drv/eval-outputOf\.sh$'' + ''^tests/functional/dyn-drv/old-daemon-error-hack\.sh$'' + ''^tests/functional/dyn-drv/recursive-mod-json\.sh$'' + ''^tests/functional/eval-store\.sh$'' + ''^tests/functional/eval\.sh$'' + ''^tests/functional/export-graph\.sh$'' + ''^tests/functional/export\.sh$'' + ''^tests/functional/extra-sandbox-profile\.sh$'' + ''^tests/functional/fetchClosure\.sh$'' + ''^tests/functional/fetchGit\.sh$'' + ''^tests/functional/fetchGitRefs\.sh$'' + ''^tests/functional/fetchGitSubmodules\.sh$'' + ''^tests/functional/fetchGitVerification\.sh$'' + ''^tests/functional/fetchMercurial\.sh$'' + ''^tests/functional/fetchurl\.sh$'' + ''^tests/functional/fixed\.builder1\.sh$'' + ''^tests/functional/fixed\.builder2\.sh$'' + ''^tests/functional/fixed\.sh$'' + ''^tests/functional/flakes/absolute-paths\.sh$'' + ''^tests/functional/flakes/check\.sh$'' + ''^tests/functional/flakes/common\.sh$'' + ''^tests/functional/flakes/config\.sh$'' + ''^tests/functional/flakes/develop\.sh$'' + ''^tests/functional/flakes/flakes\.sh$'' + ''^tests/functional/flakes/follow-paths\.sh$'' + ''^tests/functional/flakes/prefetch\.sh$'' + ''^tests/functional/flakes/run\.sh$'' + ''^tests/functional/flakes/show\.sh$'' + ''^tests/functional/fmt\.sh$'' + ''^tests/functional/fmt\.simple\.sh$'' + ''^tests/functional/gc-auto\.sh$'' + ''^tests/functional/gc-concurrent\.builder\.sh$'' + ''^tests/functional/gc-concurrent\.sh$'' + ''^tests/functional/gc-concurrent2\.builder\.sh$'' + ''^tests/functional/gc-non-blocking\.sh$'' + ''^tests/functional/gc\.sh$'' + ''^tests/functional/git-hashing/common\.sh$'' + ''^tests/functional/git-hashing/simple\.sh$'' + ''^tests/functional/hash-convert\.sh$'' + ''^tests/functional/help\.sh$'' + ''^tests/functional/impure-derivations\.sh$'' + ''^tests/functional/impure-env\.sh$'' + ''^tests/functional/impure-eval\.sh$'' + ''^tests/functional/install-darwin\.sh$'' + ''^tests/functional/lang\.sh$'' + ''^tests/functional/legacy-ssh-store\.sh$'' + ''^tests/functional/linux-sandbox\.sh$'' + ''^tests/functional/local-overlay-store/add-lower-inner\.sh$'' + ''^tests/functional/local-overlay-store/add-lower\.sh$'' + ''^tests/functional/local-overlay-store/bad-uris\.sh$'' + ''^tests/functional/local-overlay-store/build-inner\.sh$'' + ''^tests/functional/local-overlay-store/build\.sh$'' + ''^tests/functional/local-overlay-store/check-post-init-inner\.sh$'' + ''^tests/functional/local-overlay-store/check-post-init\.sh$'' + ''^tests/functional/local-overlay-store/common\.sh$'' + ''^tests/functional/local-overlay-store/delete-duplicate-inner\.sh$'' + ''^tests/functional/local-overlay-store/delete-duplicate\.sh$'' + ''^tests/functional/local-overlay-store/delete-refs-inner\.sh$'' + ''^tests/functional/local-overlay-store/delete-refs\.sh$'' + ''^tests/functional/local-overlay-store/gc-inner\.sh$'' + ''^tests/functional/local-overlay-store/gc\.sh$'' + ''^tests/functional/local-overlay-store/optimise-inner\.sh$'' + ''^tests/functional/local-overlay-store/optimise\.sh$'' + ''^tests/functional/local-overlay-store/redundant-add-inner\.sh$'' + ''^tests/functional/local-overlay-store/redundant-add\.sh$'' + ''^tests/functional/local-overlay-store/remount\.sh$'' + ''^tests/functional/local-overlay-store/stale-file-handle-inner\.sh$'' + ''^tests/functional/local-overlay-store/stale-file-handle\.sh$'' + ''^tests/functional/local-overlay-store/verify-inner\.sh$'' + ''^tests/functional/local-overlay-store/verify\.sh$'' + ''^tests/functional/logging\.sh$'' + ''^tests/functional/misc\.sh$'' + ''^tests/functional/multiple-outputs\.sh$'' + ''^tests/functional/nar-access\.sh$'' + ''^tests/functional/nested-sandboxing\.sh$'' + ''^tests/functional/nested-sandboxing/command\.sh$'' + ''^tests/functional/nix-build\.sh$'' + ''^tests/functional/nix-channel\.sh$'' + ''^tests/functional/nix-collect-garbage-d\.sh$'' + ''^tests/functional/nix-copy-ssh-common\.sh$'' + ''^tests/functional/nix-copy-ssh-ng\.sh$'' + ''^tests/functional/nix-copy-ssh\.sh$'' + ''^tests/functional/nix-daemon-untrusting\.sh$'' + ''^tests/functional/nix-profile\.sh$'' + ''^tests/functional/nix-shell\.sh$'' + ''^tests/functional/nix_path\.sh$'' + ''^tests/functional/optimise-store\.sh$'' + ''^tests/functional/output-normalization\.sh$'' + ''^tests/functional/parallel\.builder\.sh$'' + ''^tests/functional/parallel\.sh$'' + ''^tests/functional/pass-as-file\.sh$'' + ''^tests/functional/path-from-hash-part\.sh$'' + ''^tests/functional/path-info\.sh$'' + ''^tests/functional/placeholders\.sh$'' + ''^tests/functional/plugins\.sh$'' + ''^tests/functional/post-hook\.sh$'' + ''^tests/functional/pure-eval\.sh$'' + ''^tests/functional/push-to-store-old\.sh$'' + ''^tests/functional/push-to-store\.sh$'' + ''^tests/functional/read-only-store\.sh$'' + ''^tests/functional/readfile-context\.sh$'' + ''^tests/functional/recursive\.sh$'' + ''^tests/functional/referrers\.sh$'' + ''^tests/functional/remote-store\.sh$'' + ''^tests/functional/repair\.sh$'' + ''^tests/functional/restricted\.sh$'' + ''^tests/functional/search\.sh$'' + ''^tests/functional/secure-drv-outputs\.sh$'' + ''^tests/functional/selfref-gc\.sh$'' + ''^tests/functional/shell\.sh$'' + ''^tests/functional/shell\.shebang\.sh$'' + ''^tests/functional/signing\.sh$'' + ''^tests/functional/simple\.builder\.sh$'' + ''^tests/functional/simple\.sh$'' + ''^tests/functional/ssh-relay\.sh$'' + ''^tests/functional/store-info\.sh$'' + ''^tests/functional/structured-attrs\.sh$'' + ''^tests/functional/substitute-with-invalid-ca\.sh$'' + ''^tests/functional/suggestions\.sh$'' + ''^tests/functional/supplementary-groups\.sh$'' + ''^tests/functional/tarball\.sh$'' + ''^tests/functional/test-infra\.sh$'' + ''^tests/functional/test-libstoreconsumer\.sh$'' + ''^tests/functional/timeout\.sh$'' + ''^tests/functional/toString-path\.sh$'' + ''^tests/functional/user-envs-migration\.sh$'' + ''^tests/functional/user-envs-test-case\.sh$'' + ''^tests/functional/user-envs\.builder\.sh$'' + ''^tests/functional/user-envs\.sh$'' + ''^tests/functional/why-depends\.sh$'' + ''^tests/functional/zstd\.sh$'' + ''^tests/unit/libutil/data/git/check-data\.sh$'' + ]; + }; + # TODO: nixfmt, https://github.com/NixOS/nixfmt/issues/153 + }; + }; + }; + + # We'll be pulling from this in the main flake + flake.getSystem = getSystem; +} diff --git a/maintainers/local.mk b/maintainers/local.mk new file mode 100644 index 000000000..88d594d67 --- /dev/null +++ b/maintainers/local.mk @@ -0,0 +1,15 @@ + +.PHONY: format +print-top-help += echo ' format: Format source code' + +# This uses the cached .pre-commit-hooks.yaml file +format: + @if ! type -p pre-commit &>/dev/null; then \ + echo "make format: pre-commit not found. Please use \`nix develop\`."; \ + exit 1; \ + fi; \ + if test -z "$$_NIX_PRE_COMMIT_HOOKS_CONFIG"; then \ + echo "make format: _NIX_PRE_COMMIT_HOOKS_CONFIG not set. Please use \`nix develop\`."; \ + exit 1; \ + fi; \ + pre-commit run --config $$_NIX_PRE_COMMIT_HOOKS_CONFIG --all-files diff --git a/maintainers/release-credits b/maintainers/release-credits new file mode 100755 index 000000000..7a5c87d7d --- /dev/null +++ b/maintainers/release-credits @@ -0,0 +1,184 @@ +#!/usr/bin/env nix +# vim: set filetype=python: +#!nix develop --impure --expr +#!nix `` +#!nix let flake = builtins.getFlake ("git+file://" + toString ../.); +#!nix pkgs = flake.inputs.nixpkgs.legacyPackages.${builtins.currentSystem}; +#!nix in pkgs.mkShell { nativeBuildInputs = [ +#!nix (pkgs.python3.withPackages (ps: with ps; [ requests ])) +#!nix ]; } +#!nix `` --command python3 + +# This script lists out the contributors for a given release. +# It must be run from the root of the Nix repository. + +import os +import sys +import json +import requests + +github_token = os.environ.get("GITHUB_TOKEN") +if not github_token: + print("GITHUB_TOKEN is not set. If you hit the rate limit, set it", file=sys.stderr) + # Might be ok, as we have a cache. + # raise ValueError("GITHUB_TOKEN must be set") + +# 1. Read the current version in .version +version = os.environ.get("VERSION") +if not version: + version = open(".version").read().strip() + +print(f"Generating release credits for Nix {version}", file=sys.stderr) + +# 2. Compute previous version +vcomponents = version.split(".") +if len(vcomponents) >= 2: + prev_version = f"{vcomponents[0]}.{int(vcomponents[1])-1}.0" +else: + raise ValueError(".version must have at least two components") + +# For unreleased versions +endref = "HEAD" +# For older releases +# endref = version + +# 2. Find the merge base between the current version and the previous version +mergeBase = os.popen(f"git merge-base {prev_version} {endref}").read().strip() +print(f"Merge base between {prev_version} and {endref} is {mergeBase}", file=sys.stderr) + +# 3. Find the date of the merge base +mergeBaseDate = os.popen(f"git show -s --format=%ci {mergeBase}").read().strip()[0:10] +print(f"Merge base date is {mergeBaseDate}", file=sys.stderr) + +# 4. Get the commits between the merge base and the current version + +def get_commits(): + raw = os.popen(f"git log --pretty=format:'%H\t%an\t%ae' {mergeBase}..{endref}").read().strip() + lines = raw.split("\n") + return [ { "hash": items[0], "author": items[1], "email": items[2] } + for line in lines + for items in (line.split("\t"),) + ] + +def commits_to_first_commit_by_email(commits): + by_email = dict() + for commit in commits: + email = commit["email"] + if email not in by_email: + by_email[email] = commit + return by_email + + +samples = commits_to_first_commit_by_email(get_commits()) + +# For quick testing, only pick two samples from the dict +# samples = dict(list(samples.items())[:2]) + +# Query the GitHub API to get handle +def get_github_commit(commit): + url = f"https://api.github.com/repos/NixOS/nix/commits/{commit['hash']}" + headers = {'Authorization': f'token {github_token}'} + response = requests.get(url, headers=headers) + response.raise_for_status() + return response.json() + +class Cache: + def __init__(self, filename, require = True): + self.filename = filename + try: + with open(filename, "r") as f: + self.values = json.load(f) + except FileNotFoundError: + if require: + raise + self.values = dict() + def save(self): + with open(self.filename, "w") as f: + json.dump(self.values, f, indent=4) + print(f"Saved cache to {self.filename}", file=sys.stderr) + +# The email to handle cache maps email addresses to either +# - a handle (string) +# - None (if no handle was found) +email_to_handle_cache = Cache("maintainers/data/release-credits-email-to-handle.json") + +handles = set() +emails = dict() + +for sample in samples: + s = samples[sample] + email = s["email"] + if not email in email_to_handle_cache.values: + print(f"Querying GitHub API for {s['hash']}, to get handle for {s['email']}") + ghc = get_github_commit(samples[sample]) + gha = ghc["author"] + if gha and gha["login"]: + handle = gha["login"] + print(f"Handle: {handle}") + email_to_handle_cache.values[email] = handle + else: + print(f"Found no handle for {s['email']}") + email_to_handle_cache.values[email] = None + handle = email_to_handle_cache.values[email] + if handle is not None: + handles.add(handle) + else: + emails[email] = s["author"] + +# print(email_to_handle_cache.values) + +email_to_handle_cache.save() + +handle_to_name_cache = Cache("maintainers/data/release-credits-handle-to-name.json") + +print(f"Found {len(handles)} handles", file=sys.stderr) + +for handle in handles: + if not handle in handle_to_name_cache.values: + print(f"Querying GitHub API for {handle}, to get name", file=sys.stderr) + url = f"https://api.github.com/users/{handle}" + headers = {'Authorization': f'token {github_token}'} + response = requests.get(url, headers=headers) + response.raise_for_status() + user = response.json() + name = user["name"] + print(f"Name: {name}", file=sys.stderr) + handle_to_name_cache.values[handle] = name + +handle_to_name_cache.save() + +entries = list() + +def is_bot(handle): + return handle.endswith("[bot]") + +for handle in handles: + name = handle_to_name_cache.values[handle] + if name is None: + # This way it looks more regular + name = handle + + # Note to bots: I'm only doing this because it would distract humans, and + # therefore delay your creation. I'm sorry; it's for your benefit. + if not is_bot(handle): + entries += [ f"- {name} [**(@{handle})**](https://github.com/{handle})" ] + +def shuffle(entries): + salt = os.urandom(16) + return sorted(entries, key=lambda x: hash((x, salt))) + +# Fair ordering is undecidable +entries = shuffle(entries) + +# For a sanity check, we could sort the entries by handle instead. +# entries = sorted(entries) + +print("") +print(f"This release was made possible by the following {len(entries)} contributors:") +print("") + +for entry in entries: + print(entry) + +for email in emails: + print(f"- {emails[email]}") diff --git a/maintainers/release-notes b/maintainers/release-notes index 2d84485c1..c0c4ee734 100755 --- a/maintainers/release-notes +++ b/maintainers/release-notes @@ -1,5 +1,6 @@ #!/usr/bin/env nix -#!nix shell .#changelog-d-nix --command bash +# vim: set filetype=bash: +#!nix shell .#changelog-d --command bash # --- CONFIGURATION --- @@ -151,6 +152,13 @@ section_title="Release $version_full ($DATE)" echo "# $section_title" echo changelog-d doc/manual/rl-next | sed -e 's/ *$//' + + if ! $IS_PATCH; then + echo + echo "# Contributors" + echo + VERSION=$version_full ./maintainers/release-credits + fi ) | tee -a $file log "Wrote $file" diff --git a/maintainers/release-process.md b/maintainers/release-process.md index da6886ea9..7a2b3c0a7 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -39,6 +39,10 @@ release: * Proof-read / edit / rearrange the release notes if needed. Breaking changes and highlights should go to the top. +* Run `maintainers/release-credits` to make sure the credits script works + and produces a sensible output. Some emails might not automatically map to + a GitHub handle. + * Push. ```console diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 4e2c379f0..731988568 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -11,6 +11,8 @@ use JSON::PP; use LWP::UserAgent; use Net::Amazon::S3; +delete $ENV{'shell'}; # shut up a LWP::UserAgent.pm warning + my $evalId = $ARGV[0] or die "Usage: $0 EVAL-ID\n"; my $releasesBucketName = "nix-releases"; @@ -36,11 +38,11 @@ sub fetch { my $evalUrl = "https://hydra.nixos.org/eval/$evalId"; my $evalInfo = decode_json(fetch($evalUrl, 'application/json')); #print Dumper($evalInfo); -my $flakeUrl = $evalInfo->{flake} or die; -my $flakeInfo = decode_json(`nix flake metadata --json "$flakeUrl"` or die); -my $nixRev = $flakeInfo->{revision} or die; +my $flakeUrl = $evalInfo->{flake}; +my $flakeInfo = decode_json(`nix flake metadata --json "$flakeUrl"` or die) if $flakeUrl; +my $nixRev = ($flakeInfo ? $flakeInfo->{revision} : $evalInfo->{jobsetevalinputs}->{nix}->{revision}) or die; -my $buildInfo = decode_json(fetch("$evalUrl/job/build.x86_64-linux", 'application/json')); +my $buildInfo = decode_json(fetch("$evalUrl/job/build.nix.x86_64-linux", 'application/json')); #print Dumper($buildInfo); my $releaseName = $buildInfo->{nixname}; @@ -83,12 +85,19 @@ my $channelsBucket = $s3_us->bucket($channelsBucketName) or die; sub getStorePath { my ($jobName, $output) = @_; my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json')); - return $buildInfo->{buildoutputs}->{$output or "out"}->{path} or die "cannot get store path for '$jobName'"; + return $buildInfo->{buildoutputs}->{$output or "out"}->{path} // die "cannot get store path for '$jobName'"; } sub copyManual { - my $manual = getStorePath("build.x86_64-linux", "doc"); - print "$manual\n"; + my $manual; + eval { + $manual = getStorePath("build.nix.x86_64-linux", "doc"); + }; + if ($@) { + warn "$@"; + return; + } + print "Manual: $manual\n"; my $manualNar = "$tmpDir/$releaseName-manual.nar.xz"; print "$manualNar\n"; @@ -154,19 +163,34 @@ downloadFile("binaryTarball.x86_64-linux", "1"); downloadFile("binaryTarball.aarch64-linux", "1"); downloadFile("binaryTarball.x86_64-darwin", "1"); downloadFile("binaryTarball.aarch64-darwin", "1"); -downloadFile("binaryTarballCross.x86_64-linux.armv6l-unknown-linux-gnueabihf", "1"); -downloadFile("binaryTarballCross.x86_64-linux.armv7l-unknown-linux-gnueabihf", "1"); +eval { + downloadFile("binaryTarballCross.x86_64-linux.armv6l-unknown-linux-gnueabihf", "1"); +}; +warn "$@" if $@; +eval { + downloadFile("binaryTarballCross.x86_64-linux.armv7l-unknown-linux-gnueabihf", "1"); +}; +warn "$@" if $@; +eval { + downloadFile("binaryTarballCross.x86_64-linux.riscv64-unknown-linux-gnu", "1"); +}; +warn "$@" if $@; downloadFile("installerScript", "1"); # Upload docker images to dockerhub. my $dockerManifest = ""; my $dockerManifestLatest = ""; +my $haveDocker = 0; for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) { my $system = $platforms->[0]; my $dockerPlatform = $platforms->[1]; my $fn = "nix-$version-docker-image-$dockerPlatform.tar.gz"; - downloadFile("dockerImage.$system", "1", $fn); + eval { + downloadFile("dockerImage.$system", "1", $fn); + }; + die "$@" if $@; + $haveDocker = 1; print STDERR "loading docker image for $dockerPlatform...\n"; system("docker load -i $tmpDir/$fn") == 0 or die; @@ -194,31 +218,34 @@ for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) { $dockerManifestLatest .= " --amend $latestTag" } -print STDERR "creating multi-platform docker manifest...\n"; -system("docker manifest rm nixos/nix:$version"); -system("docker manifest create nixos/nix:$version $dockerManifest") == 0 or die; -if ($isLatest) { - print STDERR "creating latest multi-platform docker manifest...\n"; - system("docker manifest rm nixos/nix:latest"); - system("docker manifest create nixos/nix:latest $dockerManifestLatest") == 0 or die; -} +if ($haveDocker) { + print STDERR "creating multi-platform docker manifest...\n"; + system("docker manifest rm nixos/nix:$version"); + system("docker manifest create nixos/nix:$version $dockerManifest") == 0 or die; + if ($isLatest) { + print STDERR "creating latest multi-platform docker manifest...\n"; + system("docker manifest rm nixos/nix:latest"); + system("docker manifest create nixos/nix:latest $dockerManifestLatest") == 0 or die; + } -print STDERR "pushing multi-platform docker manifest...\n"; -system("docker manifest push nixos/nix:$version") == 0 or die; + print STDERR "pushing multi-platform docker manifest...\n"; + system("docker manifest push nixos/nix:$version") == 0 or die; -if ($isLatest) { - print STDERR "pushing latest multi-platform docker manifest...\n"; - system("docker manifest push nixos/nix:latest") == 0 or die; + if ($isLatest) { + print STDERR "pushing latest multi-platform docker manifest...\n"; + system("docker manifest push nixos/nix:latest") == 0 or die; + } } # Upload nix-fallback-paths.nix. write_file("$tmpDir/fallback-paths.nix", "{\n" . - " x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" . - " i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" . - " aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" . - " x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" . - " aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" . + " x86_64-linux = \"" . getStorePath("build.nix.x86_64-linux") . "\";\n" . + " i686-linux = \"" . getStorePath("build.nix.i686-linux") . "\";\n" . + " aarch64-linux = \"" . getStorePath("build.nix.aarch64-linux") . "\";\n" . + " riscv64-linux = \"" . getStorePath("buildCross.nix.riscv64-unknown-linux-gnu.x86_64-linux") . "\";\n" . + " x86_64-darwin = \"" . getStorePath("build.nix.x86_64-darwin") . "\";\n" . + " aarch64-darwin = \"" . getStorePath("build.nix.aarch64-darwin") . "\";\n" . "}\n"); # Upload release files to S3. diff --git a/meson.build b/meson.build new file mode 100644 index 000000000..1554244ab --- /dev/null +++ b/meson.build @@ -0,0 +1,44 @@ +# This is just a stub project to include all the others as subprojects +# for development shell purposes + +project('nix-dev-shell', 'cpp', + version : files('.version'), + subproject_dir : 'src', +) + +# Internal Libraries +subproject('libutil') +subproject('libstore') +subproject('libfetchers') +subproject('libexpr') +subproject('libflake') +subproject('libmain') +subproject('libcmd') + +# Executables +subproject('nix') + +# Docs +subproject('internal-api-docs') +subproject('external-api-docs') + +# External C wrapper libraries +subproject('libutil-c') +subproject('libstore-c') +subproject('libexpr-c') +subproject('libmain-c') + +# Language Bindings +if not meson.is_cross_build() + subproject('perl') +endif + +# Testing +subproject('nix-util-test-support') +subproject('nix-util-tests') +subproject('nix-store-test-support') +subproject('nix-store-tests') +subproject('nix-fetchers-tests') +subproject('nix-expr-test-support') +subproject('nix-expr-tests') +subproject('nix-flake-tests') diff --git a/misc/changelog-d.cabal.nix b/misc/changelog-d.cabal.nix deleted file mode 100644 index 76f9353cd..000000000 --- a/misc/changelog-d.cabal.nix +++ /dev/null @@ -1,31 +0,0 @@ -{ mkDerivation, aeson, base, bytestring, cabal-install-parsers -, Cabal-syntax, containers, directory, filepath, frontmatter -, generic-lens-lite, lib, mtl, optparse-applicative, parsec, pretty -, regex-applicative, text, pkgs -}: -let rev = "f30f6969e9cd8b56242309639d58acea21c99d06"; -in -mkDerivation { - pname = "changelog-d"; - version = "0.1"; - src = pkgs.fetchurl { - name = "changelog-d-${rev}.tar.gz"; - url = "https://codeberg.org/roberth/changelog-d/archive/${rev}.tar.gz"; - hash = "sha256-8a2+i5u7YoszAgd5OIEW0eYUcP8yfhtoOIhLJkylYJ4="; - } // { inherit rev; }; - isLibrary = false; - isExecutable = true; - libraryHaskellDepends = [ - aeson base bytestring cabal-install-parsers Cabal-syntax containers - directory filepath frontmatter generic-lens-lite mtl parsec pretty - regex-applicative text - ]; - executableHaskellDepends = [ - base bytestring Cabal-syntax directory filepath - optparse-applicative - ]; - doHaddock = false; - description = "Concatenate changelog entries into a single one"; - license = lib.licenses.gpl3Plus; - mainProgram = "changelog-d"; -} diff --git a/misc/changelog-d.nix b/misc/changelog-d.nix deleted file mode 100644 index 1b20f4596..000000000 --- a/misc/changelog-d.nix +++ /dev/null @@ -1,31 +0,0 @@ -# Taken temporarily from -{ - callPackage, - lib, - haskell, - haskellPackages, -}: - -let - hsPkg = haskellPackages.callPackage ./changelog-d.cabal.nix { }; - - addCompletions = haskellPackages.generateOptparseApplicativeCompletions ["changelog-d"]; - - haskellModifications = - lib.flip lib.pipe [ - addCompletions - haskell.lib.justStaticExecutables - ]; - - mkDerivationOverrides = finalAttrs: oldAttrs: { - - version = oldAttrs.version + "-git-${lib.strings.substring 0 7 oldAttrs.src.rev}"; - - meta = oldAttrs.meta // { - homepage = "https://codeberg.org/roberth/changelog-d"; - maintainers = [ lib.maintainers.roberth ]; - }; - - }; -in - (haskellModifications hsPkg).overrideAttrs mkDerivationOverrides diff --git a/misc/systemv/nix-daemon b/misc/systemv/nix-daemon index fea537167..e8326f947 100755 --- a/misc/systemv/nix-daemon +++ b/misc/systemv/nix-daemon @@ -34,6 +34,7 @@ else fi # Source function library. +# shellcheck source=/dev/null . /etc/init.d/functions LOCKFILE=/var/lock/subsys/nix-daemon @@ -41,14 +42,20 @@ RUNDIR=/var/run/nix PIDFILE=${RUNDIR}/nix-daemon.pid RETVAL=0 -base=${0##*/} +# https://www.shellcheck.net/wiki/SC3004 +# Check if gettext exists +if ! type gettext > /dev/null 2>&1 +then + # If not, create a dummy function that returns the input verbatim + gettext() { printf '%s' "$1"; } +fi start() { mkdir -p ${RUNDIR} chown ${NIX_DAEMON_USER}:${NIX_DAEMON_USER} ${RUNDIR} - echo -n $"Starting nix daemon... " + printf '%s' "$(gettext 'Starting nix daemon... ')" daemonize -u $NIX_DAEMON_USER -p ${PIDFILE} $NIX_DAEMON_BIN $NIX_DAEMON_OPTS RETVAL=$? @@ -58,7 +65,7 @@ start() { } stop() { - echo -n $"Shutting down nix daemon: " + printf '%s' "$(gettext 'Shutting down nix daemon: ')" killproc -p ${PIDFILE} $NIX_DAEMON_BIN RETVAL=$? [ $RETVAL -eq 0 ] && rm -f ${LOCKFILE} ${PIDFILE} @@ -67,7 +74,7 @@ stop() { } reload() { - echo -n $"Reloading nix daemon... " + printf '%s' "$(gettext 'Reloading nix daemon... ')" killproc -p ${PIDFILE} $NIX_DAEMON_BIN -HUP RETVAL=$? echo @@ -105,7 +112,7 @@ case "$1" in fi ;; *) - echo $"Usage: $0 {start|stop|status|restart|condrestart}" + printf '%s' "$(gettext "Usage: $0 {start|stop|status|restart|condrestart}")" exit 2 ;; esac diff --git a/mk/common-test.sh b/mk/common-test.sh index 2783d293b..817422c40 100644 --- a/mk/common-test.sh +++ b/mk/common-test.sh @@ -1,27 +1,23 @@ +# shellcheck shell=bash + # Remove overall test dir (at most one of the two should match) and # remove file extension. -test_name=$(echo -n "$test" | sed \ - -e "s|^tests/unit/[^/]*/data/||" \ + +test_name=$(echo -n "${test?must be defined by caller (test runner)}" | sed \ + -e "s|^src/[^/]*-test/data/||" \ -e "s|^tests/functional/||" \ -e "s|\.sh$||" \ ) +# shellcheck disable=SC2016 TESTS_ENVIRONMENT=( "TEST_NAME=$test_name" 'NIX_REMOTE=' 'PS4=+(${BASH_SOURCE[0]-$0}:$LINENO) ' ) -: ${BASH:=/usr/bin/env bash} +read -r -a bash <<< "${BASH:-/usr/bin/env bash}" run () { - cd "$(dirname $1)" && env "${TESTS_ENVIRONMENT[@]}" $BASH -x -e -u -o pipefail $(basename $1) -} - -init_test () { - run "$init" 2>/dev/null > /dev/null -} - -run_test_proper () { - run "$test" + cd "$(dirname "$1")" && env "${TESTS_ENVIRONMENT[@]}" "${bash[@]}" -x -e -u -o pipefail "$(basename "$1")" } diff --git a/mk/compilation-database.mk b/mk/compilation-database.mk new file mode 100644 index 000000000..f69dc0de0 --- /dev/null +++ b/mk/compilation-database.mk @@ -0,0 +1,11 @@ +compile-commands-json-files := + +define write-compile-commands + _srcs := $$(sort $$(foreach src, $$($(1)_SOURCES), $$(src))) + + $(1)_COMPILE_COMMANDS_JSON := $$(addprefix $(buildprefix), $$(addsuffix .compile_commands.json, $$(basename $$(_srcs)))) + + compile-commands-json-files += $$($(1)_COMPILE_COMMANDS_JSON) + + clean-files += $$($(1)_COMPILE_COMMANDS_JSON) +endef diff --git a/mk/cxx-big-literal.mk b/mk/cxx-big-literal.mk index 85365df8e..d64a171c8 100644 --- a/mk/cxx-big-literal.mk +++ b/mk/cxx-big-literal.mk @@ -1,5 +1,5 @@ %.gen.hh: % - @echo 'R"foo(' >> $@.tmp + @echo 'R"__NIX_STR(' >> $@.tmp $(trace-gen) cat $< >> $@.tmp - @echo ')foo"' >> $@.tmp + @echo ')__NIX_STR"' >> $@.tmp @mv $@.tmp $@ diff --git a/mk/debug-test.sh b/mk/debug-test.sh index 52482c01e..0dd4406c3 100755 --- a/mk/debug-test.sh +++ b/mk/debug-test.sh @@ -3,12 +3,8 @@ set -eu -o pipefail test=$1 -init=${2-} dir="$(dirname "${BASH_SOURCE[0]}")" source "$dir/common-test.sh" -if [ -n "$init" ]; then - (init_test) -fi -run_test_proper +run "$test" diff --git a/mk/lib.mk b/mk/lib.mk index fe0add1c9..1e7af6ad5 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -68,6 +68,7 @@ include mk/patterns.mk include mk/templates.mk include mk/cxx-big-literal.mk include mk/tests.mk +include mk/compilation-database.mk # Include all sub-Makefiles. @@ -86,17 +87,23 @@ $(foreach script, $(bin-scripts), $(eval $(call install-program-in,$(script),$(b $(foreach script, $(bin-scripts), $(eval programs-list += $(script))) $(foreach script, $(noinst-scripts), $(eval programs-list += $(script))) $(foreach template, $(template-files), $(eval $(call instantiate-template,$(template)))) -install_test_init=tests/functional/init.sh $(foreach test, $(install-tests), \ - $(eval $(call run-test,$(test),$(install_test_init))) \ + $(eval $(call run-test,$(test))) \ $(eval installcheck: $(test).test)) $(foreach test-group, $(install-tests-groups), \ - $(eval $(call run-test-group,$(test-group),$(install_test_init))) \ + $(eval $(call run-test-group,$(test-group))) \ $(eval installcheck: $(test-group).test-group) \ $(foreach test, $($(test-group)-tests), \ - $(eval $(call run-test,$(test),$(install_test_init))) \ + $(eval $(call run-test,$(test))) \ $(eval $(test-group).test-group: $(test).test))) +# Compilation database. +$(foreach lib, $(libraries), $(eval $(call write-compile-commands,$(lib)))) +$(foreach prog, $(programs), $(eval $(call write-compile-commands,$(prog)))) + +compile_commands.json: $(compile-commands-json-files) + @jq --slurp '.' $^ >$@ + # Include makefiles requiring built programs. $(foreach mf, $(makefiles-late), $(eval $(call include-sub-makefile,$(mf)))) diff --git a/mk/patterns.mk b/mk/patterns.mk index c81150260..4caa2039e 100644 --- a/mk/patterns.mk +++ b/mk/patterns.mk @@ -1,11 +1,41 @@ + +# These are the complete command lines we use to compile C and C++ files. +# - $< is the source file. +# - $1 is the object file to create. +CC_CMD=$(CC) -o $1 -c $< $(CPPFLAGS) $(GLOBAL_CFLAGS) $(CFLAGS) $($1_CFLAGS) -MMD -MF $(call filename-to-dep,$1) -MP +CXX_CMD=$(CXX) -o $1 -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($1_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep,$1) -MP + +# We use COMPILE_COMMANDS_JSON_CMD to turn a compilation command (like CC_CMD +# or CXX_CMD above) into a comple_commands.json file. We rely on bash native +# word splitting to define the positional arguments. +# - $< is the source file being compiled. +COMPILE_COMMANDS_JSON_CMD=jq --null-input '{ directory: $$ENV.PWD, file: "$<", arguments: $$ARGS.positional }' --args -- + + $(buildprefix)%.o: %.cc @mkdir -p "$(dir $@)" - $(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cxx) $(call CXX_CMD,$@) $(buildprefix)%.o: %.cpp @mkdir -p "$(dir $@)" - $(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cxx) $(call CXX_CMD,$@) $(buildprefix)%.o: %.c @mkdir -p "$(dir $@)" - $(trace-cc) $(CC) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CFLAGS) $(CFLAGS) $($@_CFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cc) $(call CC_CMD,$@) + +# In the following we need to replace the .compile_commands.json extension in $@ with .o +# to make the object file. This is needed because CC_CMD and CXX_CMD do further expansions +# based on the object file name (i.e. *_CXXFLAGS and filename-to-dep). + +$(buildprefix)%.compile_commands.json: %.cc + @mkdir -p "$(dir $@)" + $(trace-jq) $(COMPILE_COMMANDS_JSON_CMD) $(call CXX_CMD,$(@:.compile_commands.json=.o)) > $@ + +$(buildprefix)%.compile_commands.json: %.cpp + @mkdir -p "$(dir $@)" + $(trace-jq) $(COMPILE_COMMANDS_JSON_CMD) $(call CXX_CMD,$(@:.compile_commands.json=.o)) > $@ + +$(buildprefix)%.compile_commands.json: %.c + @mkdir -p "$(dir $@)" + $(trace-jq) $(COMPILE_COMMANDS_JSON_CMD) $(call CC_CMD,$(@:.compile_commands.json=.o)) > $@ diff --git a/mk/platform.mk b/mk/platform.mk index fe960dedf..22c114a20 100644 --- a/mk/platform.mk +++ b/mk/platform.mk @@ -29,4 +29,8 @@ ifdef HOST_OS HOST_SOLARIS = 1 HOST_UNIX = 1 endif + ifeq ($(HOST_KERNEL), gnu) + HOST_HURD = 1 + HOST_UNIX = 1 + endif endif diff --git a/mk/precompiled-headers.mk b/mk/precompiled-headers.mk index cdd3daecd..f2803eb79 100644 --- a/mk/precompiled-headers.mk +++ b/mk/precompiled-headers.mk @@ -8,7 +8,7 @@ GCH = $(buildprefix)precompiled-headers.h.gch $(GCH): precompiled-headers.h @rm -f $@ @mkdir -p "$(dir $@)" - $(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS) + $(trace-gen) $(CXX) -c -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS) clean-files += $(GCH) diff --git a/mk/run-test.sh b/mk/run-test.sh index da9c5a473..7f9f1d5f8 100755 --- a/mk/run-test.sh +++ b/mk/run-test.sh @@ -8,7 +8,6 @@ yellow="" normal="" test=$1 -init=${2-} dir="$(dirname "${BASH_SOURCE[0]}")" source "$dir/common-test.sh" @@ -22,20 +21,18 @@ if [ -t 1 ]; then fi run_test () { - if [ -n "$init" ]; then - (init_test 2>/dev/null > /dev/null) - fi - log="$(run_test_proper 2>&1)" && status=0 || status=$? + log="$(run "$test" 2>&1)" && status=0 || status=$? } run_test -if [ $status -eq 0 ]; then +if [[ "$status" = 0 ]]; then echo "$post_run_msg [${green}PASS$normal]" -elif [ $status -eq 99 ]; then +elif [[ "$status" = 77 ]]; then echo "$post_run_msg [${yellow}SKIP$normal]" else echo "$post_run_msg [${red}FAIL$normal]" + # shellcheck disable=SC2001 echo "$log" | sed 's/^/ /' exit "$status" fi diff --git a/mk/tests.mk b/mk/tests.mk index bac9b704a..0a10f6d3b 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -12,8 +12,8 @@ endef define run-test - $(eval $(call run-bash,$1.test,$1 $(test-deps),mk/run-test.sh $1 $2)) - $(eval $(call run-bash,$1.test-debug,$1 $(test-deps),mk/debug-test.sh $1 $2)) + $(eval $(call run-bash,$1.test,$1 $(test-deps),mk/run-test.sh $1)) + $(eval $(call run-bash,$1.test-debug,$1 $(test-deps),mk/debug-test.sh $1)) endef diff --git a/mk/tracing.mk b/mk/tracing.mk index 1fc5573d7..09db1e617 100644 --- a/mk/tracing.mk +++ b/mk/tracing.mk @@ -10,6 +10,8 @@ ifeq ($(V), 0) trace-install = @echo " INST " $@; trace-mkdir = @echo " MKDIR " $@; trace-test = @echo " TEST " $@; + trace-sh = @echo " SH " $@; + trace-jq = @echo " JQ " $@; suppress = @ diff --git a/package.nix b/package.nix index 20796a386..a7c8923e8 100644 --- a/package.nix +++ b/package.nix @@ -13,17 +13,16 @@ , curl , editline , readline -, fileset , flex , git , gtest , jq -, doxygen , libarchive , libcpuid , libgit2 , libseccomp , libsodium +, man , lowdown , mdbook , mdbook-linkcheck @@ -33,7 +32,8 @@ , pkg-config , rapidcheck , sqlite -, util-linux +, toml11 +, unixtools , xz , busybox-sandbox-shell ? null @@ -48,10 +48,8 @@ , pname ? "nix" , versionSuffix ? "" -, officialRelease ? false -# Whether to build Nix. Useful to skip for tasks like (a) just -# generating API docs or (b) testing existing pre-built versions of Nix +# Whether to build Nix. Useful to skip for tasks like testing existing pre-built versions of Nix , doBuild ? true # Run the unit tests as part of the build. See `installUnitTests` for an @@ -74,7 +72,10 @@ # sounds so long as evaluation just takes places within short-lived # processes. (When the process exits, the memory is reclaimed; it is # only leaked *within* the process.) -, enableGC ? true +# +# Temporarily disabled on Windows because the `GC_throw_bad_alloc` +# symbol is missing during linking. +, enableGC ? !stdenv.hostPlatform.isWindows # Whether to enable Markdown rendering in the Nix binary. , enableMarkdown ? !stdenv.hostPlatform.isWindows @@ -87,10 +88,6 @@ # - readline , readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline" -# Whether to build the internal API docs, can be done separately from -# everything else. -, enableInternalAPIDocs ? false - # Whether to install unit tests. This is useful when cross compiling # since we cannot run them natively during the build, but can do so # later. @@ -113,6 +110,8 @@ }: let + inherit (lib) fileset; + version = lib.fileContents ./.version + versionSuffix; # selected attributes with defaults, will be used to define some @@ -161,6 +160,8 @@ in { ./m4 # TODO: do we really need README.md? It doesn't seem used in the build. ./README.md + # This could be put behind a conditional + ./maintainers/local.mk # For make, regardless of what we are building ./local.mk ./Makefile @@ -171,17 +172,11 @@ in { ./doc ./misc ./precompiled-headers.h - ./src + (fileset.difference ./src ./src/perl) ./COPYING ./scripts/local.mk - ] ++ lib.optionals buildUnitTests [ + ] ++ lib.optionals enableManual [ ./doc/manual - ] ++ lib.optionals enableInternalAPIDocs [ - ./doc/internal-api - # Source might not be compiled, but still must be available - # for Doxygen to gather comments. - ./src - ./tests/unit ] ++ lib.optionals buildUnitTests [ ./tests/unit ] ++ lib.optionals doInstallCheck [ @@ -195,8 +190,10 @@ in { ++ lib.optional doBuild "dev" # If we are doing just build or just docs, the one thing will use # "out". We only need additional outputs if we are doing both. - ++ lib.optional (doBuild && (enableManual || enableInternalAPIDocs)) "doc" - ++ lib.optional installUnitTests "check"; + ++ lib.optional (doBuild && enableManual) "doc" + ++ lib.optional installUnitTests "check" + ++ lib.optional doCheck "testresults" + ; nativeBuildInputs = [ autoconf-archive @@ -213,14 +210,14 @@ in { git mercurial openssh + man # for testing `nix-* --help` ] ++ lib.optionals (doInstallCheck || enableManual) [ jq # Also for custom mdBook preprocessor. - ] ++ lib.optional stdenv.hostPlatform.isLinux util-linux - ++ lib.optional enableInternalAPIDocs doxygen + ] ++ lib.optional stdenv.hostPlatform.isStatic unixtools.hexdump ; - buildInputs = lib.optionals doBuild [ - boost + buildInputs = lib.optionals doBuild ( + [ brotli bzip2 curl @@ -229,6 +226,7 @@ in { libsodium openssl sqlite + toml11 xz ({ inherit readline editline; }.${readlineFlavor}) ] ++ lib.optionals enableMarkdown [ @@ -240,46 +238,22 @@ in { ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid # There have been issues building these dependencies ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin)) - (aws-sdk-cpp.override { - apis = ["s3" "transfer"]; - customMemoryManagement = false; - }) - ; + aws-sdk-cpp + ); - propagatedBuildInputs = [ + propagatedBuildInputs = lib.optionals doBuild ([ + boost nlohmann_json - ] ++ lib.optional enableGC boehmgc; + ] ++ lib.optional enableGC boehmgc + ); dontBuild = !attrs.doBuild; doCheck = attrs.doCheck; - disallowedReferences = [ boost ]; - - preConfigure = lib.optionalString (doBuild && ! stdenv.hostPlatform.isStatic) ( - '' - # Copy libboost_context so we don't get all of Boost in our closure. - # https://github.com/NixOS/nixpkgs/issues/45462 - mkdir -p $out/lib - cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib - rm -f $out/lib/*.a - '' + lib.optionalString stdenv.hostPlatform.isLinux '' - chmod u+w $out/lib/*.so.* - patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* - '' + lib.optionalString stdenv.hostPlatform.isDarwin '' - for LIB in $out/lib/*.dylib; do - chmod u+w $LIB - install_name_tool -id $LIB $LIB - install_name_tool -delete_rpath ${boost}/lib/ $LIB || true - done - install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib - '' - ); - configureFlags = [ (lib.enableFeature doBuild "build") (lib.enableFeature buildUnitTests "unit-tests") (lib.enableFeature doInstallCheck "functional-tests") - (lib.enableFeature enableInternalAPIDocs "internal-api-docs") (lib.enableFeature enableManual "doc-gen") (lib.enableFeature enableGC "gc") (lib.enableFeature enableMarkdown "markdown") @@ -303,8 +277,11 @@ in { makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1"; - installTargets = lib.optional doBuild "install" - ++ lib.optional enableInternalAPIDocs "internal-api-html"; + preCheck = '' + mkdir $testresults + ''; + + installTargets = lib.optional doBuild "install"; installFlags = "sysconfdir=$(out)/etc"; @@ -319,18 +296,16 @@ in { lib.optionalString stdenv.hostPlatform.isStatic '' mkdir -p $out/nix-support echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products - '' + lib.optionalString stdenv.isDarwin '' - install_name_tool \ - -change ${boost}/lib/libboost_context.dylib \ - $out/lib/libboost_context.dylib \ - $out/lib/libnixutil.dylib '' ) + lib.optionalString enableManual '' mkdir -p ''${!outputDoc}/nix-support echo "doc manual ''${!outputDoc}/share/doc/nix/manual" >> ''${!outputDoc}/nix-support/hydra-build-products - '' + lib.optionalString enableInternalAPIDocs '' - mkdir -p ''${!outputDoc}/nix-support - echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products + ''; + + # So the check output gets links for DLLs in the out output. + preFixup = lib.optionalString (stdenv.hostPlatform.isWindows && builtins.elem "check" finalAttrs.outputs) '' + ln -s "$check/lib/"*.dll "$check/bin" + ln -s "$out/bin/"*.dll "$check/bin" ''; doInstallCheck = attrs.doInstallCheck; @@ -341,20 +316,26 @@ in { # Work around weird bug where it doesn't think there is a Makefile. installCheckPhase = if (!doBuild && doInstallCheck) then '' + runHook preInstallCheck mkdir -p src/nix-channel make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES '' else null; # Needed for tests if we are not doing a build, but testing existing # built Nix. - preInstallCheck = lib.optionalString (! doBuild) '' - mkdir -p src/nix-channel - ''; + preInstallCheck = + lib.optionalString (! doBuild) '' + mkdir -p src/nix-channel + '' + # See https://github.com/NixOS/nix/issues/2523 + # Occurs often in tests since https://github.com/NixOS/nix/pull/9900 + + lib.optionalString stdenv.hostPlatform.isDarwin '' + export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES + ''; separateDebugInfo = !stdenv.hostPlatform.isStatic; - # TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated - # to work with `strictDeps`. + # TODO Always true after https://github.com/NixOS/nixpkgs/issues/318564 strictDeps = !withCoverageChecks; hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; diff --git a/packaging/components.nix b/packaging/components.nix new file mode 100644 index 000000000..870e9ae61 --- /dev/null +++ b/packaging/components.nix @@ -0,0 +1,43 @@ +scope: +let + inherit (scope) callPackage; +in + +# This becomes the pkgs.nixComponents attribute set +{ + nix = callPackage ../package.nix { }; + + nix-util = callPackage ../src/libutil/package.nix { }; + nix-util-c = callPackage ../src/libutil-c/package.nix { }; + nix-util-test-support = callPackage ../tests/unit/libutil-support/package.nix { }; + nix-util-tests = callPackage ../tests/unit/libutil/package.nix { }; + + nix-store = callPackage ../src/libstore/package.nix { }; + nix-store-c = callPackage ../src/libstore-c/package.nix { }; + nix-store-test-support = callPackage ../tests/unit/libstore-support/package.nix { }; + nix-store-tests = callPackage ../tests/unit/libstore/package.nix { }; + + nix-fetchers = callPackage ../src/libfetchers/package.nix { }; + nix-fetchers-tests = callPackage ../tests/unit/libfetchers/package.nix { }; + + nix-expr = callPackage ../src/libexpr/package.nix { }; + nix-expr-c = callPackage ../src/libexpr-c/package.nix { }; + nix-expr-test-support = callPackage ../tests/unit/libexpr-support/package.nix { }; + nix-expr-tests = callPackage ../tests/unit/libexpr/package.nix { }; + + nix-flake = callPackage ../src/libflake/package.nix { }; + nix-flake-tests = callPackage ../tests/unit/libflake/package.nix { }; + + nix-main = callPackage ../src/libmain/package.nix { }; + nix-main-c = callPackage ../src/libmain-c/package.nix { }; + + nix-cmd = callPackage ../src/libcmd/package.nix { }; + + # Will replace `nix` once the old build system is gone. + nix-ng = callPackage ../src/nix/package.nix { }; + + nix-internal-api-docs = callPackage ../src/internal-api-docs/package.nix { }; + nix-external-api-docs = callPackage ../src/external-api-docs/package.nix { }; + + nix-perl-bindings = callPackage ../src/perl/package.nix { }; +} diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix new file mode 100644 index 000000000..e0737593f --- /dev/null +++ b/packaging/dependencies.nix @@ -0,0 +1,165 @@ +# These overrides are applied to the dependencies of the Nix components. + +{ + # Flake inputs; used for sources + inputs, + + # The raw Nixpkgs, not affected by this scope + pkgs, + + stdenv, + versionSuffix, +}: + +let + prevStdenv = stdenv; +in + +let + inherit (pkgs) lib; + + root = ../.; + + stdenv = if prevStdenv.isDarwin && prevStdenv.isx86_64 + then darwinStdenv + else prevStdenv; + + # Fix the following error with the default x86_64-darwin SDK: + # + # error: aligned allocation function of type 'void *(std::size_t, std::align_val_t)' is only available on macOS 10.13 or newer + # + # Despite the use of the 10.13 deployment target here, the aligned + # allocation function Clang uses with this setting actually works + # all the way back to 10.6. + darwinStdenv = pkgs.overrideSDK prevStdenv { darwinMinVersion = "10.13"; }; + + # Nixpkgs implements this by returning a subpath into the fetched Nix sources. + resolvePath = p: p; + + # Indirection for Nixpkgs to override when package.nix files are vendored + filesetToSource = lib.fileset.toSource; + + localSourceLayer = finalAttrs: prevAttrs: + let + workDirPath = + # Ideally we'd pick finalAttrs.workDir, but for now `mkDerivation` has + # the requirement that everything except passthru and meta must be + # serialized by mkDerivation, which doesn't work for this. + prevAttrs.workDir; + + workDirSubpath = lib.path.removePrefix root workDirPath; + sources = assert prevAttrs.fileset._type == "fileset"; prevAttrs.fileset; + src = lib.fileset.toSource { fileset = sources; inherit root; }; + + in + { + sourceRoot = "${src.name}/" + workDirSubpath; + inherit src; + + # Clear what `derivation` can't/shouldn't serialize; see prevAttrs.workDir. + fileset = null; + workDir = null; + }; + + # Work around weird `--as-needed` linker behavior with BSD, see + # https://github.com/mesonbuild/meson/issues/3593 + bsdNoLinkAsNeeded = finalAttrs: prevAttrs: + lib.optionalAttrs stdenv.hostPlatform.isBSD { + mesonFlags = [ (lib.mesonBool "b_asneeded" false) ] ++ prevAttrs.mesonFlags or []; + }; + + miscGoodPractice = finalAttrs: prevAttrs: + { + strictDeps = prevAttrs.strictDeps or true; + enableParallelBuilding = true; + }; + +in +scope: { + inherit stdenv versionSuffix; + version = lib.fileContents ../.version + versionSuffix; + + aws-sdk-cpp = (pkgs.aws-sdk-cpp.override { + apis = [ "s3" "transfer" ]; + customMemoryManagement = false; + }).overrideAttrs { + # only a stripped down version is built, which takes a lot less resources + # to build, so we don't need a "big-parallel" machine. + requiredSystemFeatures = [ ]; + }; + + libseccomp = pkgs.libseccomp.overrideAttrs (_: rec { + version = "2.5.5"; + src = pkgs.fetchurl { + url = "https://github.com/seccomp/libseccomp/releases/download/v${version}/libseccomp-${version}.tar.gz"; + hash = "sha256-JIosik2bmFiqa69ScSw0r+/PnJ6Ut23OAsHJqiX7M3U="; + }; + }); + + boehmgc = pkgs.boehmgc.override { + enableLargeConfig = true; + }; + + # TODO Hack until https://github.com/NixOS/nixpkgs/issues/45462 is fixed. + boost = (pkgs.boost.override { + extraB2Args = [ + "--with-container" + "--with-context" + "--with-coroutine" + ]; + }).overrideAttrs (old: { + # Need to remove `--with-*` to use `--with-libraries=...` + buildPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.buildPhase; + installPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.installPhase; + }); + + libgit2 = pkgs.libgit2.overrideAttrs (attrs: { + src = inputs.libgit2; + version = inputs.libgit2.lastModifiedDate; + cmakeFlags = attrs.cmakeFlags or [] + ++ [ "-DUSE_SSH=exec" ]; + }); + + busybox-sandbox-shell = pkgs.busybox-sandbox-shell or (pkgs.busybox.override { + useMusl = true; + enableStatic = true; + enableMinimal = true; + extraConfig = '' + CONFIG_FEATURE_FANCY_ECHO y + CONFIG_FEATURE_SH_MATH y + CONFIG_FEATURE_SH_MATH_64 y + + CONFIG_ASH y + CONFIG_ASH_OPTIMIZE_FOR_SIZE y + + CONFIG_ASH_ALIAS y + CONFIG_ASH_BASH_COMPAT y + CONFIG_ASH_CMDCMD y + CONFIG_ASH_ECHO y + CONFIG_ASH_GETOPTS y + CONFIG_ASH_INTERNAL_GLOB y + CONFIG_ASH_JOB_CONTROL y + CONFIG_ASH_PRINTF y + CONFIG_ASH_TEST y + ''; + }); + + # TODO change in Nixpkgs, Windows works fine. First commit of + # https://github.com/NixOS/nixpkgs/pull/322977 backported will fix. + toml11 = pkgs.toml11.overrideAttrs (old: { + meta.platforms = lib.platforms.all; + }); + + inherit resolvePath filesetToSource; + + mkMesonDerivation = f: let + exts = [ + miscGoodPractice + bsdNoLinkAsNeeded + localSourceLayer + ]; + in stdenv.mkDerivation + (lib.extends + (lib.foldr lib.composeExtensions (_: _: {}) exts) + f); +} diff --git a/packaging/hydra.nix b/packaging/hydra.nix new file mode 100644 index 000000000..dbe992476 --- /dev/null +++ b/packaging/hydra.nix @@ -0,0 +1,200 @@ +{ inputs +, binaryTarball +, forAllCrossSystems +, forAllSystems +, lib +, linux64BitSystems +, nixpkgsFor +, self +}: +let + inherit (inputs) nixpkgs nixpkgs-regression; + + installScriptFor = tarballs: + nixpkgsFor.x86_64-linux.native.callPackage ../scripts/installer.nix { + inherit tarballs; + }; + + testNixVersions = pkgs: client: daemon: + pkgs.callPackage ../package.nix { + pname = + "nix-tests" + + lib.optionalString + (lib.versionAtLeast daemon.version "2.4pre20211005" && + lib.versionAtLeast client.version "2.4pre20211005") + "-${client.version}-against-${daemon.version}"; + + test-client = client; + test-daemon = daemon; + + doBuild = false; + }; + + # Technically we could just return `pkgs.nixComponents`, but for Hydra it's + # convention to transpose it, and to transpose it efficiently, we need to + # enumerate them manually, so that we don't evaluate unnecessary package sets. + forAllPackages = lib.genAttrs [ + "nix" + "nix-util" + "nix-util-c" + "nix-util-test-support" + "nix-util-tests" + "nix-store" + "nix-store-c" + "nix-store-test-support" + "nix-store-tests" + "nix-fetchers" + "nix-fetchers-tests" + "nix-expr" + "nix-expr-c" + "nix-expr-test-support" + "nix-expr-tests" + "nix-flake" + "nix-flake-tests" + "nix-main" + "nix-main-c" + "nix-cmd" + "nix-ng" + ]; +in +{ + # Binary package for various platforms. + build = forAllPackages (pkgName: + forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.${pkgName})); + + shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation); + + buildStatic = forAllPackages (pkgName: + lib.genAttrs linux64BitSystems (system: nixpkgsFor.${system}.static.nixComponents.${pkgName})); + + buildCross = forAllPackages (pkgName: + forAllCrossSystems (crossSystem: + lib.genAttrs [ "x86_64-linux" ] (system: nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName}))); + + buildNoGc = forAllSystems (system: + self.packages.${system}.nix.override { enableGC = false; } + ); + + buildNoTests = forAllSystems (system: nixpkgsFor.${system}.native.nix_noTests); + + # Toggles some settings for better coverage. Windows needs these + # library combinations, and Debian build Nix with GNU readline too. + buildReadlineNoMarkdown = forAllSystems (system: + self.packages.${system}.nix.override { + enableMarkdown = false; + readlineFlavor = "readline"; + } + ); + + # Perl bindings for various platforms. + perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.nix-perl-bindings); + + # Binary tarball for various platforms, containing a Nix store + # with the closure of 'nix' package, and the second half of + # the installation script. + binaryTarball = forAllSystems (system: binaryTarball nixpkgsFor.${system}.native.nix nixpkgsFor.${system}.native); + + binaryTarballCross = lib.genAttrs [ "x86_64-linux" ] (system: + forAllCrossSystems (crossSystem: + binaryTarball + nixpkgsFor.${system}.cross.${crossSystem}.nix + nixpkgsFor.${system}.cross.${crossSystem})); + + # The first half of the installation script. This is uploaded + # to https://nixos.org/nix/install. It downloads the binary + # tarball for the user's system and calls the second half of the + # installation script. + installerScript = installScriptFor [ + # Native + self.hydraJobs.binaryTarball."x86_64-linux" + self.hydraJobs.binaryTarball."i686-linux" + self.hydraJobs.binaryTarball."aarch64-linux" + self.hydraJobs.binaryTarball."x86_64-darwin" + self.hydraJobs.binaryTarball."aarch64-darwin" + # Cross + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu" + ]; + installerScriptForGHA = installScriptFor [ + # Native + self.hydraJobs.binaryTarball."x86_64-linux" + self.hydraJobs.binaryTarball."aarch64-darwin" + # Cross + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu" + ]; + + # docker image with Nix inside + dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); + + # Line coverage analysis. + coverage = nixpkgsFor.x86_64-linux.native.nix.override { + pname = "nix-coverage"; + withCoverageChecks = true; + }; + + # API docs for Nix's unstable internal C++ interfaces. + internal-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-internal-api-docs; + + # API docs for Nix's C bindings. + external-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-external-api-docs; + + # System tests. + tests = import ../tests/nixos { inherit lib nixpkgs nixpkgsFor self; } // { + + # Make sure that nix-env still produces the exact same result + # on a particular version of Nixpkgs. + evalNixpkgs = + let + inherit (nixpkgsFor.x86_64-linux.native) runCommand nix; + in + runCommand "eval-nixos" { buildInputs = [ nix ]; } + '' + type -p nix-env + # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593. + ( + set -x + time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages + [[ $(sha1sum < packages | cut -c1-40) = e01b031fc9785a572a38be6bc473957e3b6faad7 ]] + ) + mkdir $out + ''; + + nixpkgsLibTests = + forAllSystems (system: + import (nixpkgs + "/lib/tests/test-with-nix.nix") + { + lib = nixpkgsFor.${system}.native.lib; + nix = self.packages.${system}.nix; + pkgs = nixpkgsFor.${system}.native; + } + ); + }; + + metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" { + pkgs = nixpkgsFor.x86_64-linux.native; + nixpkgs = nixpkgs-regression; + }; + + installTests = forAllSystems (system: + let pkgs = nixpkgsFor.${system}.native; in + pkgs.runCommand "install-tests" + { + againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix; + againstCurrentLatest = + # FIXME: temporarily disable this on macOS because of #3605. + if system == "x86_64-linux" + then testNixVersions pkgs pkgs.nix pkgs.nixVersions.latest + else null; + # Disabled because the latest stable version doesn't handle + # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work + # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable; + } "touch $out"); + + installerTests = import ../tests/installer { + binaryTarballs = self.hydraJobs.binaryTarball; + inherit nixpkgsFor; + }; +} diff --git a/perl/.yath.rc b/perl/.yath.rc deleted file mode 100644 index 118bf80c8..000000000 --- a/perl/.yath.rc +++ /dev/null @@ -1,2 +0,0 @@ -[test] --I=rel(lib/Nix) diff --git a/perl/Makefile b/perl/Makefile deleted file mode 100644 index 832668dd1..000000000 --- a/perl/Makefile +++ /dev/null @@ -1,21 +0,0 @@ -makefiles = local.mk - -GLOBAL_CXXFLAGS += -g -Wall -std=c++2a - -# A convenience for concurrent development of Nix and its Perl bindings. -# Not needed in a standalone build of the Perl bindings. -ifneq ("$(wildcard ../src)", "") - GLOBAL_CXXFLAGS += -I ../src -endif - --include Makefile.config - -OPTIMIZE = 1 - -ifeq ($(OPTIMIZE), 1) - GLOBAL_CXXFLAGS += -O3 -else - GLOBAL_CXXFLAGS += -O0 -endif - -include mk/lib.mk diff --git a/perl/Makefile.config.in b/perl/Makefile.config.in deleted file mode 100644 index d856de3ad..000000000 --- a/perl/Makefile.config.in +++ /dev/null @@ -1,18 +0,0 @@ -HOST_OS = @host_os@ -CC = @CC@ -CFLAGS = @CFLAGS@ -CXX = @CXX@ -CXXFLAGS = @CXXFLAGS@ -PACKAGE_NAME = @PACKAGE_NAME@ -PACKAGE_VERSION = @PACKAGE_VERSION@ -SODIUM_LIBS = @SODIUM_LIBS@ -NIX_CFLAGS = @NIX_CFLAGS@ -NIX_LIBS = @NIX_LIBS@ -nixbindir = @nixbindir@ -curl = @curl@ -nixlibexecdir = @nixlibexecdir@ -nixlocalstatedir = @nixlocalstatedir@ -perl = @perl@ -perllibdir = @perllibdir@ -nixstoredir = @nixstoredir@ -nixsysconfdir = @nixsysconfdir@ diff --git a/perl/configure.ac b/perl/configure.ac deleted file mode 100644 index a02cb06c9..000000000 --- a/perl/configure.ac +++ /dev/null @@ -1,84 +0,0 @@ -AC_INIT(nix-perl, m4_esyscmd([bash -c "echo -n $(cat ../.version)$VERSION_SUFFIX"])) -AC_CONFIG_SRCDIR(MANIFEST) -AC_CONFIG_AUX_DIR(../config) - -CFLAGS= -CXXFLAGS= -AC_PROG_CC -AC_PROG_CXX - -AC_CANONICAL_HOST - -# Use 64-bit file system calls so that we can support files > 2 GiB. -AC_SYS_LARGEFILE - -AC_DEFUN([NEED_PROG], -[ -AC_PATH_PROG($1, $2) -if test -z "$$1"; then - AC_MSG_ERROR([$2 is required]) -fi -]) - -NEED_PROG(perl, perl) -NEED_PROG(curl, curl) -NEED_PROG(bzip2, bzip2) -NEED_PROG(xz, xz) - -# Test that Perl has the open/fork feature (Perl 5.8.0 and beyond). -AC_MSG_CHECKING([whether Perl is recent enough]) -if ! $perl -e 'open(FOO, "-|", "true"); while () { print; }; close FOO or die;'; then - AC_MSG_RESULT(no) - AC_MSG_ERROR([Your Perl version is too old. Nix requires Perl 5.8.0 or newer.]) -fi -AC_MSG_RESULT(yes) - - -# Figure out where to install Perl modules. -AC_MSG_CHECKING([for the Perl installation prefix]) -perlversion=$($perl -e 'use Config; print $Config{version};') -perlarchname=$($perl -e 'use Config; print $Config{archname};') -AC_SUBST(perllibdir, [${libdir}/perl5/site_perl/$perlversion/$perlarchname]) -AC_MSG_RESULT($perllibdir) - -# Look for libsodium. -PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"]) - -# Check for the required Perl dependencies (DBI and DBD::SQLite). -perlFlags="-I$perllibdir" - -AC_ARG_WITH(dbi, AC_HELP_STRING([--with-dbi=PATH], - [prefix of the Perl DBI library]), - perlFlags="$perlFlags -I$withval") - -AC_ARG_WITH(dbd-sqlite, AC_HELP_STRING([--with-dbd-sqlite=PATH], - [prefix of the Perl DBD::SQLite library]), - perlFlags="$perlFlags -I$withval") - -AC_MSG_CHECKING([whether DBD::SQLite works]) -if ! $perl $perlFlags -e 'use DBI; use DBD::SQLite;' 2>&5; then - AC_MSG_RESULT(no) - AC_MSG_FAILURE([The Perl modules DBI and/or DBD::SQLite are missing.]) -fi -AC_MSG_RESULT(yes) - -AC_SUBST(perlFlags) - -PKG_CHECK_MODULES([NIX], [nix-store]) - -NEED_PROG([NIX], [nix]) - -# Expand all variables in config.status. -test "$prefix" = NONE && prefix=$ac_default_prefix -test "$exec_prefix" = NONE && exec_prefix='${prefix}' -for name in $ac_subst_vars; do - declare $name="$(eval echo "${!name}")" - declare $name="$(eval echo "${!name}")" - declare $name="$(eval echo "${!name}")" -done - -rm -f Makefile.config -ln -sfn ../mk mk - -AC_CONFIG_FILES([]) -AC_OUTPUT diff --git a/perl/default.nix b/perl/default.nix deleted file mode 100644 index 7103574c9..000000000 --- a/perl/default.nix +++ /dev/null @@ -1,61 +0,0 @@ -{ lib, fileset -, stdenv -, perl, perlPackages -, autoconf-archive, autoreconfHook, pkg-config -, nix, curl, bzip2, xz, boost, libsodium, darwin -}: - -perl.pkgs.toPerlModule (stdenv.mkDerivation (finalAttrs: { - name = "nix-perl-${nix.version}"; - - src = fileset.toSource { - root = ../.; - fileset = fileset.unions ([ - ../.version - ../m4 - ../mk - ./MANIFEST - ./Makefile - ./Makefile.config.in - ./configure.ac - ./lib - ./local.mk - ] ++ lib.optionals finalAttrs.doCheck [ - ./.yath.rc - ./t - ]); - }; - - nativeBuildInputs = - [ autoconf-archive - autoreconfHook - pkg-config - ]; - - buildInputs = - [ nix - curl - bzip2 - xz - perl - boost - ] - ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium - ++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security; - - # `perlPackages.Test2Harness` is marked broken for Darwin - doCheck = !stdenv.isDarwin; - - nativeCheckInputs = [ - perlPackages.Test2Harness - ]; - - configureFlags = [ - "--with-dbi=${perlPackages.DBI}/${perl.libPrefix}" - "--with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix}" - ]; - - enableParallelBuilding = true; - - postUnpack = "sourceRoot=$sourceRoot/perl"; -})) diff --git a/perl/local.mk b/perl/local.mk deleted file mode 100644 index ed4764eb9..000000000 --- a/perl/local.mk +++ /dev/null @@ -1,46 +0,0 @@ -nix_perl_sources := \ - lib/Nix/Store.pm \ - lib/Nix/Manifest.pm \ - lib/Nix/SSH.pm \ - lib/Nix/CopyClosure.pm \ - lib/Nix/Config.pm.in \ - lib/Nix/Utils.pm - -nix_perl_modules := $(nix_perl_sources:.in=) - -$(foreach x, $(nix_perl_modules), $(eval $(call install-data-in, $(x), $(perllibdir)/Nix))) - -lib/Nix/Store.cc: lib/Nix/Store.xs - $(trace-gen) xsubpp $^ -output $@ - -libraries += Store - -Store_DIR := lib/Nix - -Store_SOURCES := $(Store_DIR)/Store.cc - -Store_CXXFLAGS = \ - $(NIX_CFLAGS) \ - -I$(shell perl -e 'use Config; print $$Config{archlibexp};')/CORE \ - -D_FILE_OFFSET_BITS=64 \ - -Wno-unknown-warning-option -Wno-unused-variable -Wno-literal-suffix \ - -Wno-reserved-user-defined-literal -Wno-duplicate-decl-specifier -Wno-pointer-bool-conversion - -Store_LDFLAGS := $(SODIUM_LIBS) $(NIX_LIBS) - -ifdef HOST_CYGWIN - archlib = $(shell perl -E 'use Config; print $$Config{archlib};') - libperl = $(shell perl -E 'use Config; print $$Config{libperl};') - Store_LDFLAGS += $(shell find ${archlib} -name ${libperl}) -endif - -Store_ALLOW_UNDEFINED = 1 - -Store_FORCE_INSTALL = 1 - -Store_INSTALL_DIR = $(perllibdir)/auto/Nix/Store - -clean-files += lib/Nix/Config.pm lib/Nix/Store.cc Makefile.config - -check: all - yath test diff --git a/precompiled-headers.h b/precompiled-headers.h index f52f1cab8..e1a3f8cc0 100644 --- a/precompiled-headers.h +++ b/precompiled-headers.h @@ -42,19 +42,22 @@ #include #include #include -#include -#include -#include #include -#include -#include -#include #include #include #include -#include -#include -#include #include +#ifndef _WIN32 +# include +# include +# include +# include +# include +# include +# include +# include +# include +#endif + #include diff --git a/scripts/bigsur-nixbld-user-migration.sh b/scripts/bigsur-nixbld-user-migration.sh index f1619fd56..0eb312e07 100755 --- a/scripts/bigsur-nixbld-user-migration.sh +++ b/scripts/bigsur-nixbld-user-migration.sh @@ -3,7 +3,7 @@ ((NEW_NIX_FIRST_BUILD_UID=301)) id_available(){ - dscl . list /Users UniqueID | grep -E '\b'$1'\b' >/dev/null + dscl . list /Users UniqueID | grep -E '\b'"$1"'\b' >/dev/null } change_nixbld_names_and_ids(){ @@ -26,18 +26,18 @@ change_nixbld_names_and_ids(){ fi done - if [[ $name == _* ]]; then + if [[ "$name" == _* ]]; then echo " It looks like $name has already been renamed--skipping." else # first 3 are cleanup, it's OK if they aren't here - sudo dscl . delete /Users/$name dsAttrTypeNative:_writers_passwd &>/dev/null || true - sudo dscl . change /Users/$name NFSHomeDirectory "/private/var/empty 1" "/var/empty" &>/dev/null || true + sudo dscl . delete "/Users/$name" dsAttrTypeNative:_writers_passwd &>/dev/null || true + sudo dscl . change "/Users/$name" NFSHomeDirectory "/private/var/empty 1" "/var/empty" &>/dev/null || true # remove existing user from group - sudo dseditgroup -o edit -t user -d $name nixbld || true - sudo dscl . change /Users/$name UniqueID $uid $next_id - sudo dscl . change /Users/$name RecordName $name _$name + sudo dseditgroup -o edit -t user -d "$name" nixbld || true + sudo dscl . change "/Users/$name" UniqueID "$uid" "$next_id" + sudo dscl . change "/Users/$name" RecordName "$name" "_$name" # add renamed user to group - sudo dseditgroup -o edit -t user -a _$name nixbld + sudo dseditgroup -o edit -t user -a "_$name" nixbld echo " $name migrated to _$name (uid: $next_id)" fi done < <(dscl . list /Users UniqueID | grep nixbld | sort -n -k2) diff --git a/scripts/check-hydra-status.sh b/scripts/check-hydra-status.sh deleted file mode 100644 index e62705e94..000000000 --- a/scripts/check-hydra-status.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail -# set -x - - -# mapfile BUILDS_FOR_LATEST_EVAL < <( -# curl -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master/evals | \ -# jq -r '.evals[0].builds[] | @sh') -BUILDS_FOR_LATEST_EVAL=$( -curl -sS -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master/evals | \ - jq -r '.evals[0].builds[]') - -someBuildFailed=0 - -for buildId in $BUILDS_FOR_LATEST_EVAL; do - buildInfo=$(curl --fail -sS -H 'Accept: application/json' "https://hydra.nixos.org/build/$buildId") - - finished=$(echo "$buildInfo" | jq -r '.finished') - - if [[ $finished = 0 ]]; then - continue - fi - - buildStatus=$(echo "$buildInfo" | jq -r '.buildstatus') - - if [[ $buildStatus != 0 ]]; then - someBuildFailed=1 - echo "Job “$(echo "$buildInfo" | jq -r '.job')†failed on hydra: $buildInfo" - fi -done - -exit "$someBuildFailed" diff --git a/scripts/flake-regressions.sh b/scripts/flake-regressions.sh new file mode 100755 index 000000000..d76531134 --- /dev/null +++ b/scripts/flake-regressions.sh @@ -0,0 +1,27 @@ +#! /usr/bin/env bash + +set -e + +echo "Nix version:" +nix --version + +cd flake-regressions + +status=0 + +flakes=$(find tests -mindepth 3 -maxdepth 3 -type d -not -path '*/.*' | sort | head -n25) + +echo "Running flake tests..." + +for flake in $flakes; do + + if ! REGENERATE=0 ./eval-flake.sh "$flake"; then + status=1 + echo "⌠$flake" + else + echo "✅ $flake" + fi + +done + +exit "$status" diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index ad3ee8881..6aee073e3 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -754,7 +754,7 @@ I will: (if it does, I will tell you how to clean them up.) - create local users (see the list above for the users I'll make) - create a local group ($NIX_BUILD_GROUP_NAME) - - install Nix in to $NIX_ROOT + - install Nix in $NIX_ROOT - create a configuration file in /etc/nix - set up the "default profile" by creating some Nix-related files in $ROOT_HOME diff --git a/scripts/install-systemd-multi-user.sh b/scripts/install-systemd-multi-user.sh index 202a9bb54..a62ed7e3a 100755 --- a/scripts/install-systemd-multi-user.sh +++ b/scripts/install-systemd-multi-user.sh @@ -35,7 +35,7 @@ escape_systemd_env() { # Gather all non-empty proxy environment variables into a string create_systemd_proxy_env() { - vars="http_proxy https_proxy ftp_proxy no_proxy HTTP_PROXY HTTPS_PROXY FTP_PROXY NO_PROXY" + vars="http_proxy https_proxy ftp_proxy all_proxy no_proxy HTTP_PROXY HTTPS_PROXY FTP_PROXY ALL_PROXY NO_PROXY" for v in $vars; do if [ "x${!v:-}" != "x" ]; then echo "Environment=${v}=$(escape_systemd_env ${!v})" diff --git a/scripts/install.in b/scripts/install.in index 7d2e52b26..b4e808d8e 100755 --- a/scripts/install.in +++ b/scripts/install.in @@ -50,6 +50,11 @@ case "$(uname -s).$(uname -m)" in path=@tarballPath_armv7l-linux@ system=armv7l-linux ;; + Linux.riscv64) + hash=@tarballHash_riscv64-linux@ + path=@tarballPath_riscv64-linux@ + system=riscv64-linux + ;; Darwin.x86_64) hash=@tarballHash_x86_64-darwin@ path=@tarballPath_x86_64-darwin@ diff --git a/scripts/nix-profile-daemon.fish.in b/scripts/nix-profile-daemon.fish.in index c23aa64f0..346dce5dd 100644 --- a/scripts/nix-profile-daemon.fish.in +++ b/scripts/nix-profile-daemon.fish.in @@ -28,7 +28,7 @@ else end # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. -if test -n "$NIX_SSH_CERT_FILE" +if test -n "$NIX_SSL_CERT_FILE" : # Allow users to override the NIX_SSL_CERT_FILE else if test -e /etc/ssl/certs/ca-certificates.crt # NixOS, Ubuntu, Debian, Gentoo, Arch set --export NIX_SSL_CERT_FILE /etc/ssl/certs/ca-certificates.crt @@ -44,7 +44,7 @@ else if test -e "$NIX_LINK/etc/ca-bundle.crt" # old cacert in Nix profile set --export NIX_SSL_CERT_FILE "$NIX_LINK/etc/ca-bundle.crt" else # Fall back to what is in the nix profiles, favouring whatever is defined last. - for i in $NIX_PROFILES + for i in (string split ' ' $NIX_PROFILES) if test -e "$i/etc/ssl/certs/ca-bundle.crt" set --export NIX_SSL_CERT_FILE "$i/etc/ssl/certs/ca-bundle.crt" end diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index d256b24ed..eb124c0b5 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -1,4 +1,5 @@ # Only execute this file once per shell. +# This file is tested by tests/installer/default.nix. if [ -n "${__ETC_PROFILE_NIX_SOURCED:-}" ]; then return; fi __ETC_PROFILE_NIX_SOURCED=1 @@ -9,11 +10,9 @@ else NIX_LINK_NEW=$HOME/.local/state/nix/profile fi if [ -e "$NIX_LINK_NEW" ]; then - NIX_LINK="$NIX_LINK_NEW" -else - if [ -t 2 ] && [ -e "$NIX_LINK_NEW" ]; then + if [ -t 2 ] && [ -e "$NIX_LINK" ]; then warning="\033[1;35mwarning:\033[0m" - printf "$warning Both %s and legacy %s exist; using the latter.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2 + printf "$warning Both %s and legacy %s exist; using the former.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2 if [ "$(realpath "$NIX_LINK")" = "$(realpath "$NIX_LINK_NEW")" ]; then printf " Since the profiles match, you can safely delete either of them.\n" 1>&2 else @@ -26,6 +25,7 @@ else printf "$warning Profiles do not match. You should manually migrate from %s to %s.\n" "$NIX_LINK" "$NIX_LINK_NEW" 1>&2 fi fi + NIX_LINK="$NIX_LINK_NEW" fi export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK" @@ -69,4 +69,4 @@ else fi export PATH="$NIX_LINK/bin:@localstatedir@/nix/profiles/default/bin:$PATH" -unset NIX_LINK +unset NIX_LINK NIX_LINK_NEW diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index 44bc96e89..e868399b1 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -1,3 +1,4 @@ +# This file is tested by tests/installer/default.nix. if [ -n "$HOME" ] && [ -n "$USER" ]; then # Set up the per-user profile. @@ -9,11 +10,9 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then NIX_LINK_NEW="$HOME/.local/state/nix/profile" fi if [ -e "$NIX_LINK_NEW" ]; then - NIX_LINK="$NIX_LINK_NEW" - else - if [ -t 2 ] && [ -e "$NIX_LINK_NEW" ]; then + if [ -t 2 ] && [ -e "$NIX_LINK" ]; then warning="\033[1;35mwarning:\033[0m" - printf "$warning Both %s and legacy %s exist; using the latter.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2 + printf "$warning Both %s and legacy %s exist; using the former.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2 if [ "$(realpath "$NIX_LINK")" = "$(realpath "$NIX_LINK_NEW")" ]; then printf " Since the profiles match, you can safely delete either of them.\n" 1>&2 else @@ -26,6 +25,7 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then printf "$warning Profiles do not match. You should manually migrate from %s to %s.\n" "$NIX_LINK" "$NIX_LINK_NEW" 1>&2 fi fi + NIX_LINK="$NIX_LINK_NEW" fi # Set up environment. diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 118468477..82ad7d862 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -11,11 +11,13 @@ #include "machines.hh" #include "shared.hh" +#include "plugin.hh" #include "pathlocks.hh" #include "globals.hh" #include "serialise.hh" #include "build-result.hh" #include "store-api.hh" +#include "strings.hh" #include "derivations.hh" #include "local-store.hh" #include "legacy.hh" @@ -37,7 +39,7 @@ static std::string currentLoad; static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot) { - return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true); + return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri.render()), slot), true); } static bool allSupportedLocally(Store & store, const std::set& requiredFeatures) { @@ -135,7 +137,7 @@ static int main_build_remote(int argc, char * * argv) Machine * bestMachine = nullptr; uint64_t bestLoad = 0; for (auto & m : machines) { - debug("considering building on remote machine '%s'", m.storeUri); + debug("considering building on remote machine '%s'", m.storeUri.render()); if (m.enabled && m.systemSupported(neededSystem) && @@ -202,7 +204,7 @@ static int main_build_remote(int argc, char * * argv) else drvstr = ""; - auto error = HintFmt(errorText); + auto error = HintFmt::fromFormatString(errorText); error % drvstr % neededSystem @@ -232,17 +234,16 @@ static int main_build_remote(int argc, char * * argv) lock = -1; try { + storeUri = bestMachine->storeUri.render(); - Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri)); + Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", storeUri)); sshStore = bestMachine->openStore(); sshStore->connect(); - storeUri = bestMachine->storeUri; - } catch (std::exception & e) { auto msg = chomp(drainFD(5, false)); printError("cannot build on '%s': %s%s", - bestMachine->storeUri, e.what(), + storeUri, e.what(), msg.empty() ? "" : ": " + msg); bestMachine->enabled = false; continue; @@ -262,7 +263,20 @@ connected: auto inputs = readStrings(source); auto wantedOutputs = readStrings(source); - AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true); + AutoCloseFD uploadLock; + { + auto setUpdateLock = [&](auto && fileName){ + uploadLock = openLockFile(currentLoad + "/" + escapeUri(fileName) + ".upload-lock", true); + }; + try { + setUpdateLock(storeUri); + } catch (SysError & e) { + if (e.errNo != ENAMETOOLONG) throw; + // Try again hashing the store URL so we have a shorter path + auto h = hashString(HashAlgorithm::MD5, storeUri); + setUpdateLock(h.to_string(HashFormat::Base64, false)); + } + } { Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri)); diff --git a/doc/internal-api/.gitignore b/src/external-api-docs/.gitignore similarity index 100% rename from doc/internal-api/.gitignore rename to src/external-api-docs/.gitignore diff --git a/src/external-api-docs/.version b/src/external-api-docs/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/external-api-docs/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/external-api-docs/README.md b/src/external-api-docs/README.md new file mode 100644 index 000000000..8760ac88b --- /dev/null +++ b/src/external-api-docs/README.md @@ -0,0 +1,121 @@ +# Getting started + +> **Warning** These bindings are **experimental**, which means they can change +> at any time or be removed outright; nevertheless the plan is to provide a +> stable external C API to the Nix language and the Nix store. + +The language library allows evaluating Nix expressions and interacting with Nix +language values. The Nix store API is still rudimentary, and only allows +initialising and connecting to a store for the Nix language evaluator to +interact with. + +Currently there are two ways to interface with the Nix language evaluator +programmatically: + +1. Embedding the evaluator +2. Writing language plug-ins + +Embedding means you link the Nix C libraries in your program and use them from +there. Adding a plug-in means you make a library that gets loaded by the Nix +language evaluator, specified through a configuration option. + +Many of the components and mechanisms involved are not yet documented, therefore +please refer to the [Nix source code](https://github.com/NixOS/nix/) for +details. Additions to in-code documentation and the reference manual are highly +appreciated. + +The following examples, for simplicity, don't include error handling. See the +[Handling errors](@ref errors) section for more information. + +# Embedding the Nix Evaluator{#nix_evaluator_example} + +In this example we programmatically start the Nix language evaluator with a +dummy store (that has no store paths and cannot be written to), and evaluate the +Nix expression `builtins.nixVersion`. + +**main.c:** + +```C +#include +#include +#include +#include +#include +#include + +// NOTE: This example lacks all error handling. Production code must check for +// errors, as some return values will be undefined. + +void my_get_string_cb(const char * start, unsigned int n, void * user_data) +{ + *((char **) user_data) = strdup(start); +} + +int main() +{ + nix_libexpr_init(NULL); + + Store * store = nix_store_open(NULL, "dummy://", NULL); + EvalState * state = nix_state_create(NULL, NULL, store); // empty search path (NIX_PATH) + Value * value = nix_alloc_value(NULL, state); + + nix_expr_eval_from_string(NULL, state, "builtins.nixVersion", ".", value); + nix_value_force(NULL, state, value); + + char * version; + nix_get_string(NULL, value, my_get_string_cb, &version); + printf("Nix version: %s\n", version); + + free(version); + nix_gc_decref(NULL, value); + nix_state_free(state); + nix_store_free(store); + return 0; +} +``` + +**Usage:** + +```ShellSession +$ gcc main.c $(pkg-config nix-expr-c --libs --cflags) -o main +$ ./main +Nix version: 2.17 +``` + +# Writing a Nix language plug-in + +In this example we add a custom primitive operation (_primop_) to `builtins`. It +will increment the argument if it is an integer and throw an error otherwise. + +**plugin.c:** + +```C +#include +#include +#include + +void increment(void* user_data, nix_c_context* ctx, EvalState* state, Value** args, Value* v) { + nix_value_force(NULL, state, args[0]); + if (nix_get_type(NULL, args[0]) == NIX_TYPE_INT) { + nix_init_int(NULL, v, nix_get_int(NULL, args[0]) + 1); + } else { + nix_set_err_msg(ctx, NIX_ERR_UNKNOWN, "First argument should be an integer."); + } +} + +void nix_plugin_entry() { + const char* args[] = {"n", NULL}; + PrimOp *p = nix_alloc_primop(NULL, increment, 1, "increment", args, "Example custom built-in function: increments an integer", NULL); + nix_register_primop(NULL, p); + nix_gc_decref(NULL, p); +} +``` + +**Usage:** + +```ShellSession +$ gcc plugin.c $(pkg-config nix-expr-c --libs --cflags) -shared -o plugin.so +$ nix --plugin-files ./plugin.so repl +nix-repl> builtins.increment 1 +2 +``` diff --git a/src/external-api-docs/doxygen.cfg.in b/src/external-api-docs/doxygen.cfg.in new file mode 100644 index 000000000..1be71d895 --- /dev/null +++ b/src/external-api-docs/doxygen.cfg.in @@ -0,0 +1,58 @@ +# Doxyfile 1.9.5 + +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by +# double-quotes, unless you are using Doxywizard) that should identify the +# project for which the documentation is generated. This name is used in the +# title of most generated pages and in a few other places. +# The default value is: My Project. + +PROJECT_NAME = "Nix" + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. This +# could be handy for archiving the generated documentation or if some version +# control system is used. + +PROJECT_NUMBER = @PROJECT_NUMBER@ + +OUTPUT_DIRECTORY = @OUTPUT_DIRECTORY@ + +# Using the PROJECT_BRIEF tag one can provide an optional one line description +# for a project that appears at the top of each page and should give viewer a +# quick idea about the purpose of the project. Keep the description short. + +PROJECT_BRIEF = "Nix, the purely functional package manager: C API (experimental)" + +# If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output. +# The default value is: YES. + +GENERATE_LATEX = NO + +# The INPUT tag is used to specify the files and/or directories that contain +# documented source files. You may enter file names like myfile.cpp or +# directories like /usr/src/myproject. Separate the files or directories with +# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING +# Note: If this tag is empty the current directory is searched. + +# FIXME Make this list more maintainable somehow. We could maybe generate this +# in the Makefile, but we would need to change how `.in` files are preprocessed +# so they can expand variables despite configure variables. + +INPUT = \ + @src@/src/libutil-c \ + @src@/src/libexpr-c \ + @src@/src/libstore-c \ + @src@/doc/external-api/README.md + +FILE_PATTERNS = nix_api_*.h *.md + +# The INCLUDE_PATH tag can be used to specify one or more directories that +# contain include files that are not input files but should be processed by the +# preprocessor. Note that the INCLUDE_PATH is not recursive, so the setting of +# RECURSIVE has no effect here. +# This tag requires that the tag SEARCH_INCLUDES is set to YES. + +EXCLUDE_PATTERNS = *_internal.h +GENERATE_TREEVIEW = YES +OPTIMIZE_OUTPUT_FOR_C = YES + +USE_MDFILE_AS_MAINPAGE = doc/external-api/README.md diff --git a/src/external-api-docs/meson.build b/src/external-api-docs/meson.build new file mode 100644 index 000000000..62474ffe4 --- /dev/null +++ b/src/external-api-docs/meson.build @@ -0,0 +1,31 @@ +project('nix-external-api-docs', + version : files('.version'), + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +fs = import('fs') + +doxygen_cfg = configure_file( + input : 'doxygen.cfg.in', + output : 'doxygen.cfg', + configuration : { + 'PROJECT_NUMBER': meson.project_version(), + 'OUTPUT_DIRECTORY' : meson.current_build_dir(), + 'src' : fs.parent(fs.parent(meson.project_source_root())), + }, +) + +doxygen = find_program('doxygen', native : true, required : true) + +custom_target( + 'external-api-docs', + command : [ doxygen , doxygen_cfg ], + input : [ + doxygen_cfg, + ], + output : 'html', + install : true, + install_dir : get_option('datadir') / 'doc/nix/external-api', + build_always_stale : true, +) diff --git a/src/external-api-docs/package.nix b/src/external-api-docs/package.nix new file mode 100644 index 000000000..743b3e9b7 --- /dev/null +++ b/src/external-api-docs/package.nix @@ -0,0 +1,59 @@ +{ lib +, mkMesonDerivation + +, meson +, ninja +, doxygen + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-external-api-docs"; + inherit version; + + workDir = ./.; + fileset = + let + cpp = fileset.fileFilter (file: file.hasExt "cc" || file.hasExt "h"); + in + fileset.unions [ + ./.version + ../../.version + ./meson.build + ./doxygen.cfg.in + ./README.md + # Source is not compiled, but still must be available for Doxygen + # to gather comments. + (cpp ../libexpr-c) + (cpp ../libstore-c) + (cpp ../libutil-c) + ]; + + nativeBuildInputs = [ + meson + ninja + doxygen + ]; + + preConfigure = + '' + chmod u+w ./.version + echo ${finalAttrs.version} > ./.version + ''; + + postInstall = '' + mkdir -p ''${!outputDoc}/nix-support + echo "doc external-api-docs $out/share/doc/nix/external-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products + ''; + + meta = { + platforms = lib.platforms.all; + }; +}) diff --git a/src/internal-api-docs/.gitignore b/src/internal-api-docs/.gitignore new file mode 100644 index 000000000..dab28b6b0 --- /dev/null +++ b/src/internal-api-docs/.gitignore @@ -0,0 +1,3 @@ +/doxygen.cfg +/html +/latex diff --git a/src/internal-api-docs/.version b/src/internal-api-docs/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/internal-api-docs/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/doc/internal-api/doxygen.cfg.in b/src/internal-api-docs/doxygen.cfg.in similarity index 84% rename from doc/internal-api/doxygen.cfg.in rename to src/internal-api-docs/doxygen.cfg.in index 6c6c325bd..f1ef75b38 100644 --- a/doc/internal-api/doxygen.cfg.in +++ b/src/internal-api-docs/doxygen.cfg.in @@ -12,7 +12,9 @@ PROJECT_NAME = "Nix" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = @PACKAGE_VERSION@ +PROJECT_NUMBER = @PROJECT_NUMBER@ + +OUTPUT_DIRECTORY = @OUTPUT_DIRECTORY@ # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a @@ -36,27 +38,27 @@ GENERATE_LATEX = NO # so they can expand variables despite configure variables. INPUT = \ - src/libcmd \ - src/libexpr \ - src/libexpr/flake \ - tests/unit/libexpr \ - tests/unit/libexpr/value \ - tests/unit/libexpr/test \ - tests/unit/libexpr/test/value \ - src/libexpr/value \ - src/libfetchers \ - src/libmain \ - src/libstore \ - src/libstore/build \ - src/libstore/builtins \ - tests/unit/libstore \ - tests/unit/libstore/test \ - src/libutil \ - tests/unit/libutil \ - tests/unit/libutil/test \ - src/nix \ - src/nix-env \ - src/nix-store + @src@/libcmd \ + @src@/libexpr \ + @src@/libexpr/flake \ + @src@/nix-expr-tests \ + @src@/nix-expr-tests/value \ + @src@/nix-expr-test-support/test \ + @src@/nix-expr-test-support/test/value \ + @src@/libexpr/value \ + @src@/libfetchers \ + @src@/libmain \ + @src@/libstore \ + @src@/libstore/build \ + @src@/libstore/builtins \ + @src@/nix-store-tests \ + @src@/nix-store-test-support/test \ + @src@/libutil \ + @src@/nix-util-tests \ + @src@/nix-util-test-support/test \ + @src@/nix \ + @src@/nix-env \ + @src@/nix-store # If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names # in the source code. If set to NO, only conditional compilation will be diff --git a/src/internal-api-docs/meson.build b/src/internal-api-docs/meson.build new file mode 100644 index 000000000..54eb7e5dd --- /dev/null +++ b/src/internal-api-docs/meson.build @@ -0,0 +1,31 @@ +project('nix-internal-api-docs', + version : files('.version'), + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +fs = import('fs') + +doxygen_cfg = configure_file( + input : 'doxygen.cfg.in', + output : 'doxygen.cfg', + configuration : { + 'PROJECT_NUMBER': meson.project_version(), + 'OUTPUT_DIRECTORY' : meson.current_build_dir(), + 'src' : fs.parent(fs.parent(meson.project_source_root())) / 'src', + }, +) + +doxygen = find_program('doxygen', native : true, required : true) + +custom_target( + 'internal-api-docs', + command : [ doxygen , doxygen_cfg ], + input : [ + doxygen_cfg, + ], + output : 'html', + install : true, + install_dir : get_option('datadir') / 'doc/nix/internal-api', + build_always_stale : true, +) diff --git a/src/internal-api-docs/package.nix b/src/internal-api-docs/package.nix new file mode 100644 index 000000000..07ca6d4d9 --- /dev/null +++ b/src/internal-api-docs/package.nix @@ -0,0 +1,54 @@ +{ lib +, mkMesonDerivation + +, meson +, ninja +, doxygen + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-internal-api-docs"; + inherit version; + + workDir = ./.; + fileset = let + cpp = fileset.fileFilter (file: file.hasExt "cc" || file.hasExt "hh"); + in fileset.unions [ + ./.version + ../../.version + ./meson.build + ./doxygen.cfg.in + # Source is not compiled, but still must be available for Doxygen + # to gather comments. + (cpp ../.) + ]; + + nativeBuildInputs = [ + meson + ninja + doxygen + ]; + + preConfigure = + '' + chmod u+w ./.version + echo ${finalAttrs.version} > ./.version + ''; + + postInstall = '' + mkdir -p ''${!outputDoc}/nix-support + echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products + ''; + + meta = { + platforms = lib.platforms.all; + }; +}) diff --git a/src/libcmd/.version b/src/libcmd/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libcmd/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libcmd/build-utils-meson b/src/libcmd/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libcmd/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc index c5eb93c5d..905e70f32 100644 --- a/src/libcmd/built-path.cc +++ b/src/libcmd/built-path.cc @@ -1,6 +1,7 @@ #include "built-path.hh" #include "derivations.hh" #include "store-api.hh" +#include "comparator.hh" #include @@ -8,30 +9,24 @@ namespace nix { -#define CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, COMPARATOR) \ - bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \ - { \ - const MY_TYPE* me = this; \ - auto fields1 = std::tie(*me->drvPath, me->FIELD); \ - me = &other; \ - auto fields2 = std::tie(*me->drvPath, me->FIELD); \ - return fields1 COMPARATOR fields2; \ - } -#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, ==) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <) +// Custom implementation to avoid `ref` ptr equality +GENERATE_CMP_EXT( + , + std::strong_ordering, + SingleBuiltPathBuilt, + *me->drvPath, + me->output); -#define FIELD_TYPE std::pair -CMP(SingleBuiltPath, SingleBuiltPathBuilt, output) -#undef FIELD_TYPE +// Custom implementation to avoid `ref` ptr equality -#define FIELD_TYPE std::map -CMP(SingleBuiltPath, BuiltPathBuilt, outputs) -#undef FIELD_TYPE - -#undef CMP -#undef CMP_ONE +// TODO no `GENERATE_CMP_EXT` because no `std::set::operator<=>` on +// Darwin, per header. +GENERATE_EQUAL( + , + BuiltPathBuilt ::, + BuiltPathBuilt, + *me->drvPath, + me->outputs); StorePath SingleBuiltPath::outPath() const { diff --git a/src/libcmd/built-path.hh b/src/libcmd/built-path.hh index 99917e0ee..dc78d3e59 100644 --- a/src/libcmd/built-path.hh +++ b/src/libcmd/built-path.hh @@ -18,7 +18,8 @@ struct SingleBuiltPathBuilt { static SingleBuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view); nlohmann::json toJSON(const StoreDirConfig & store) const; - DECLARE_CMP(SingleBuiltPathBuilt); + bool operator ==(const SingleBuiltPathBuilt &) const noexcept; + std::strong_ordering operator <=>(const SingleBuiltPathBuilt &) const noexcept; }; using _SingleBuiltPathRaw = std::variant< @@ -33,6 +34,9 @@ struct SingleBuiltPath : _SingleBuiltPathRaw { using Opaque = DerivedPathOpaque; using Built = SingleBuiltPathBuilt; + bool operator == (const SingleBuiltPath &) const = default; + auto operator <=> (const SingleBuiltPath &) const = default; + inline const Raw & raw() const { return static_cast(*this); } @@ -59,11 +63,13 @@ struct BuiltPathBuilt { ref drvPath; std::map outputs; + bool operator == (const BuiltPathBuilt &) const noexcept; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //std::strong_ordering operator <=> (const BuiltPathBuilt &) const noexcept; + std::string to_string(const StoreDirConfig & store) const; static BuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view); nlohmann::json toJSON(const StoreDirConfig & store) const; - - DECLARE_CMP(BuiltPathBuilt); }; using _BuiltPathRaw = std::variant< @@ -82,6 +88,10 @@ struct BuiltPath : _BuiltPathRaw { using Opaque = DerivedPathOpaque; using Built = BuiltPathBuilt; + bool operator == (const BuiltPath &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=> (const BuiltPath &) const = default; + inline const Raw & raw() const { return static_cast(*this); } diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 369fa6004..67fef1909 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -1,3 +1,5 @@ +#include + #include "command.hh" #include "markdown.hh" #include "store-api.hh" @@ -6,8 +8,7 @@ #include "nixexpr.hh" #include "profiles.hh" #include "repl.hh" - -#include +#include "strings.hh" extern char * * environ __attribute__((weak)); @@ -127,12 +128,12 @@ ref EvalCommand::getEvalState() if (!evalState) { evalState = #if HAVE_BOEHMGC - std::allocate_shared(traceable_allocator(), - searchPath, getEvalStore(), getStore()) + std::allocate_shared( + traceable_allocator(), #else std::make_shared( - searchPath, getEvalStore(), getStore()) #endif + lookupPath, getEvalStore(), fetchSettings, evalSettings, getStore()) ; evalState->repair = repair; @@ -148,7 +149,7 @@ MixOperateOnOptions::MixOperateOnOptions() { addFlag({ .longName = "derivation", - .description = "Operate on the [store derivation](../../glossary.md#gloss-store-derivation) rather than its outputs.", + .description = "Operate on the [store derivation](@docroot@/glossary.md#gloss-store-derivation) rather than its outputs.", .category = installablesCategory, .handler = {&operateOn, OperateOn::Derivation}, }); diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 444ff81c9..fcef92487 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -1,18 +1,57 @@ +#include "fetch-settings.hh" #include "eval-settings.hh" #include "common-eval-args.hh" #include "shared.hh" +#include "config-global.hh" #include "filetransfer.hh" #include "eval.hh" #include "fetchers.hh" #include "registry.hh" #include "flake/flakeref.hh" +#include "flake/settings.hh" #include "store-api.hh" #include "command.hh" #include "tarball.hh" #include "fetch-to-store.hh" +#include "compatibility-settings.hh" +#include "eval-settings.hh" namespace nix { +fetchers::Settings fetchSettings; + +static GlobalConfig::Register rFetchSettings(&fetchSettings); + +EvalSettings evalSettings { + settings.readOnlyMode, + { + { + "flake", + [](ref store, std::string_view rest) { + experimentalFeatureSettings.require(Xp::Flakes); + // FIXME `parseFlakeRef` should take a `std::string_view`. + auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false); + debug("fetching flake search path element '%s''", rest); + auto storePath = flakeRef.resolve(store).fetchTree(store).first; + return store->toRealPath(storePath); + }, + }, + }, +}; + +static GlobalConfig::Register rEvalSettings(&evalSettings); + + +flake::Settings flakeSettings; + +static GlobalConfig::Register rFlakeSettings(&flakeSettings); + + +CompatibilitySettings compatibilitySettings {}; + +static GlobalConfig::Register rCompatibilitySettings(&compatibilitySettings); + + MixEvalArgs::MixEvalArgs() { addFlag({ @@ -20,7 +59,7 @@ MixEvalArgs::MixEvalArgs() .description = "Pass the value *expr* as the argument *name* to Nix functions.", .category = category, .labels = {"name", "expr"}, - .handler = {[&](std::string name, std::string expr) { autoArgs[name] = 'E' + expr; }} + .handler = {[&](std::string name, std::string expr) { autoArgs.insert_or_assign(name, AutoArg{AutoArgExpr{expr}}); }} }); addFlag({ @@ -28,87 +67,40 @@ MixEvalArgs::MixEvalArgs() .description = "Pass the string *string* as the argument *name* to Nix functions.", .category = category, .labels = {"name", "string"}, - .handler = {[&](std::string name, std::string s) { autoArgs[name] = 'S' + s; }}, + .handler = {[&](std::string name, std::string s) { autoArgs.insert_or_assign(name, AutoArg{AutoArgString{s}}); }}, + }); + + addFlag({ + .longName = "arg-from-file", + .description = "Pass the contents of file *path* as the argument *name* to Nix functions.", + .category = category, + .labels = {"name", "path"}, + .handler = {[&](std::string name, std::string path) { autoArgs.insert_or_assign(name, AutoArg{AutoArgFile{path}}); }}, + .completer = completePath + }); + + addFlag({ + .longName = "arg-from-stdin", + .description = "Pass the contents of stdin as the argument *name* to Nix functions.", + .category = category, + .labels = {"name"}, + .handler = {[&](std::string name) { autoArgs.insert_or_assign(name, AutoArg{AutoArgStdin{}}); }}, }); addFlag({ .longName = "include", .shortName = 'I', .description = R"( - Add *path* to the Nix search path. The Nix search path is - initialized from the colon-separated [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) environment - variable, and is used to look up the location of Nix expressions using [paths](@docroot@/language/values.md#type-path) enclosed in angle - brackets (i.e., ``). + Add *path* to search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md) - For instance, passing + This option may be given multiple times. - ``` - -I /home/eelco/Dev - -I /etc/nixos - ``` - - will cause Nix to look for paths relative to `/home/eelco/Dev` and - `/etc/nixos`, in that order. This is equivalent to setting the - `NIX_PATH` environment variable to - - ``` - /home/eelco/Dev:/etc/nixos - ``` - - It is also possible to match paths against a prefix. For example, - passing - - ``` - -I nixpkgs=/home/eelco/Dev/nixpkgs-branch - -I /etc/nixos - ``` - - will cause Nix to search for `` in - `/home/eelco/Dev/nixpkgs-branch/path` and `/etc/nixos/nixpkgs/path`. - - If a path in the Nix search path starts with `http://` or `https://`, - it is interpreted as the URL of a tarball that will be downloaded and - unpacked to a temporary location. The tarball must consist of a single - top-level directory. For example, passing - - ``` - -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/master.tar.gz - ``` - - tells Nix to download and use the current contents of the `master` - branch in the `nixpkgs` repository. - - The URLs of the tarballs from the official `nixos.org` channels - (see [the manual page for `nix-channel`](../nix-channel.md)) can be - abbreviated as `channel:`. For instance, the - following two flags are equivalent: - - ``` - -I nixpkgs=channel:nixos-21.05 - -I nixpkgs=https://nixos.org/channels/nixos-21.05/nixexprs.tar.xz - ``` - - You can also fetch source trees using [flake URLs](./nix3-flake.md#url-like-syntax) and add them to the - search path. For instance, - - ``` - -I nixpkgs=flake:nixpkgs - ``` - - specifies that the prefix `nixpkgs` shall refer to the source tree - downloaded from the `nixpkgs` entry in the flake registry. Similarly, - - ``` - -I nixpkgs=flake:github:NixOS/nixpkgs/nixos-22.05 - ``` - - makes `` refer to a particular branch of the - `NixOS/nixpkgs` repository on GitHub. + Paths added through `-I` take precedence over the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path) and the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH). )", .category = category, .labels = {"path"}, .handler = {[&](std::string s) { - searchPath.elements.emplace_back(SearchPath::Elem::parse(s)); + lookupPath.elements.emplace_back(LookupPath::Elem::parse(s)); }} }); @@ -127,8 +119,8 @@ MixEvalArgs::MixEvalArgs() .category = category, .labels = {"original-ref", "resolved-ref"}, .handler = {[&](std::string _from, std::string _to) { - auto from = parseFlakeRef(_from, absPath(".")); - auto to = parseFlakeRef(_to, absPath(".")); + auto from = parseFlakeRef(fetchSettings, _from, absPath(".")); + auto to = parseFlakeRef(fetchSettings, _to, absPath(".")); fetchers::Attrs extraAttrs; if (to.subdir != "") extraAttrs["dir"] = to.subdir; fetchers::overrideRegistry(from.input, to.input, extraAttrs); @@ -154,13 +146,23 @@ MixEvalArgs::MixEvalArgs() Bindings * MixEvalArgs::getAutoArgs(EvalState & state) { auto res = state.buildBindings(autoArgs.size()); - for (auto & i : autoArgs) { + for (auto & [name, arg] : autoArgs) { auto v = state.allocValue(); - if (i.second[0] == 'E') - state.mkThunk_(*v, state.parseExprFromString(i.second.substr(1), state.rootPath("."))); - else - v->mkString(((std::string_view) i.second).substr(1)); - res.insert(state.symbols.create(i.first), v); + std::visit(overloaded { + [&](const AutoArgExpr & arg) { + state.mkThunk_(*v, state.parseExprFromString(arg.expr, compatibilitySettings.nixShellShebangArgumentsRelativeToScript ? state.rootPath(absPath(getCommandBaseDir())) : state.rootPath("."))); + }, + [&](const AutoArgString & arg) { + v->mkString(arg.s); + }, + [&](const AutoArgFile & arg) { + v->mkString(readFile(arg.path.string())); + }, + [&](const AutoArgStdin & arg) { + v->mkString(readFile(STDIN_FILENO)); + } + }, arg); + res.insert(state.symbols.create(name), v); } return res.finish(); } @@ -176,7 +178,7 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas else if (hasPrefix(s, "flake:")) { experimentalFeatureSettings.require(Xp::Flakes); - auto flakeRef = parseFlakeRef(std::string(s.substr(6)), {}, true, false); + auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false); auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first; return state.rootPath(CanonPath(state.store->toRealPath(storePath))); } diff --git a/src/libcmd/common-eval-args.hh b/src/libcmd/common-eval-args.hh index 2eb63e15d..c62365b32 100644 --- a/src/libcmd/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh @@ -6,13 +6,42 @@ #include "common-args.hh" #include "search-path.hh" +#include + namespace nix { class Store; + +namespace fetchers { struct Settings; } + class EvalState; +struct EvalSettings; +struct CompatibilitySettings; class Bindings; struct SourcePath; +namespace flake { struct Settings; } + +/** + * @todo Get rid of global setttings variables + */ +extern fetchers::Settings fetchSettings; + +/** + * @todo Get rid of global setttings variables + */ +extern EvalSettings evalSettings; + +/** + * @todo Get rid of global setttings variables + */ +extern flake::Settings flakeSettings; + +/** + * Settings that control behaviors that have changed since Nix 2.3. + */ +extern CompatibilitySettings compatibilitySettings; + struct MixEvalArgs : virtual Args, virtual MixRepair { static constexpr auto category = "Common evaluation options"; @@ -21,14 +50,24 @@ struct MixEvalArgs : virtual Args, virtual MixRepair Bindings * getAutoArgs(EvalState & state); - SearchPath searchPath; + LookupPath lookupPath; std::optional evalStoreUrl; private: - std::map autoArgs; + struct AutoArgExpr { std::string expr; }; + struct AutoArgString { std::string s; }; + struct AutoArgFile { std::filesystem::path path; }; + struct AutoArgStdin { }; + + using AutoArg = std::variant; + + std::map autoArgs; }; +/** + * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) + */ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * baseDir = nullptr); } diff --git a/src/libcmd/compatibility-settings.hh b/src/libcmd/compatibility-settings.hh new file mode 100644 index 000000000..a129a957a --- /dev/null +++ b/src/libcmd/compatibility-settings.hh @@ -0,0 +1,36 @@ +#pragma once +#include "config.hh" + +namespace nix { +struct CompatibilitySettings : public Config +{ + + CompatibilitySettings() = default; + + // Added in Nix 2.24, July 2024. + Setting nixShellAlwaysLooksForShellNix{this, true, "nix-shell-always-looks-for-shell-nix", R"( + Before Nix 2.24, [`nix-shell`](@docroot@/command-ref/nix-shell.md) would only look at `shell.nix` if it was in the working directory - when no file was specified. + + Since Nix 2.24, `nix-shell` always looks for a `shell.nix`, whether that's in the working directory, or in a directory that was passed as an argument. + + You may set this to `false` to temporarily revert to the behavior of Nix 2.23 and older. + + Using this setting is not recommended. + It will be deprecated and removed. + )"}; + + // Added in Nix 2.24, July 2024. + Setting nixShellShebangArgumentsRelativeToScript{ + this, true, "nix-shell-shebang-arguments-relative-to-script", R"( + Before Nix 2.24, relative file path expressions in arguments in a `nix-shell` shebang were resolved relative to the working directory. + + Since Nix 2.24, `nix-shell` resolves these paths in a manner that is relative to the [base directory](@docroot@/glossary.md#gloss-base-directory), defined as the script's directory. + + You may set this to `false` to temporarily revert to the behavior of Nix 2.23 and older. + + Using this setting is not recommended. + It will be deprecated and removed. + )"}; +}; + +}; diff --git a/src/libcmd/installable-attr-path.cc b/src/libcmd/installable-attr-path.cc index 3ec1c1614..8917e7a01 100644 --- a/src/libcmd/installable-attr-path.cc +++ b/src/libcmd/installable-attr-path.cc @@ -75,6 +75,8 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths() std::set outputsToInstall; for (auto & output : packageInfo.queryOutputs(false, true)) outputsToInstall.insert(output.first); + if (outputsToInstall.empty()) + outputsToInstall.insert("out"); return OutputsSpec::Names { std::move(outputsToInstall) }; }, [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec { diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index ddec7537b..8796ad5ba 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -43,20 +43,6 @@ std::vector InstallableFlake::getActualAttrPaths() return res; } -Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake) -{ - auto vFlake = state.allocValue(); - - callFlake(state, lockedFlake, *vFlake); - - auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); - assert(aOutputs); - - state.forceValue(*aOutputs->value, aOutputs->value->determinePos(noPos)); - - return aOutputs->value; -} - static std::string showAttrPaths(const std::vector & paths) { std::string s; @@ -106,9 +92,14 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() fmt("while evaluating the flake output attribute '%s'", attrPath))) { return { *derivedPathWithInfo }; + } else { + throw Error( + "expected flake output attribute '%s' to be a derivation or path but found %s: %s", + attrPath, + showType(v), + ValuePrinter(*this->state, v, errorPrintOptions) + ); } - else - throw Error("flake output attribute '%s' is not a derivation or path", attrPath); } auto drvPath = attr->forceDerivation(); @@ -205,7 +196,8 @@ std::shared_ptr InstallableFlake::getLockedFlake() const flake::LockFlags lockFlagsApplyConfig = lockFlags; // FIXME why this side effect? lockFlagsApplyConfig.applyNixConfig = true; - _lockedFlake = std::make_shared(lockFlake(*state, flakeRef, lockFlagsApplyConfig)); + _lockedFlake = std::make_shared(lockFlake( + flakeSettings, *state, flakeRef, lockFlagsApplyConfig)); } return _lockedFlake; } diff --git a/src/libcmd/installable-flake.hh b/src/libcmd/installable-flake.hh index 314918c14..8e0a232ef 100644 --- a/src/libcmd/installable-flake.hh +++ b/src/libcmd/installable-flake.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "common-eval-args.hh" #include "installable-value.hh" namespace nix { @@ -52,8 +53,6 @@ struct InstallableFlake : InstallableValue std::vector getActualAttrPaths(); - Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake); - DerivedPathsWithInfo toDerivedPaths() override; std::pair toValue(EvalState & state) override; @@ -80,7 +79,7 @@ struct InstallableFlake : InstallableValue */ static inline FlakeRef defaultNixpkgsFlakeRef() { - return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}}); + return FlakeRef::fromAttrs(fetchSettings, {{"type","indirect"}, {"id", "nixpkgs"}}); } ref openEvalCache( diff --git a/src/libcmd/installable-value.hh b/src/libcmd/installable-value.hh index f300d392b..798cb5e1a 100644 --- a/src/libcmd/installable-value.hh +++ b/src/libcmd/installable-value.hh @@ -66,7 +66,7 @@ struct ExtraPathInfoValue : ExtraPathInfo }; /** - * An Installable which corresponds a Nix langauge value, in addition to + * An Installable which corresponds a Nix language value, in addition to * a collection of \ref DerivedPath "derived paths". */ struct InstallableValue : Installable diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 16d25d3cf..0fe956ec0 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -27,6 +27,8 @@ #include +#include "strings-inline.hh" + namespace nix { void completeFlakeInputPath( @@ -129,7 +131,7 @@ MixFlakeOptions::MixFlakeOptions() lockFlags.writeLockFile = false; lockFlags.inputOverrides.insert_or_assign( flake::parseInputPath(inputPath), - parseFlakeRef(flakeRef, absPath(getCommandBaseDir()), true)); + parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir()), true)); }}, .completer = {[&](AddCompletions & completions, size_t n, std::string_view prefix) { if (n == 0) { @@ -146,7 +148,7 @@ MixFlakeOptions::MixFlakeOptions() .category = category, .labels = {"flake-lock-path"}, .handler = {[&](std::string lockFilePath) { - lockFlags.referenceLockFilePath = lockFilePath; + lockFlags.referenceLockFilePath = {getFSSourceAccessor(), CanonPath(absPath(lockFilePath))}; }}, .completer = completePath }); @@ -170,14 +172,15 @@ MixFlakeOptions::MixFlakeOptions() .handler = {[&](std::string flakeRef) { auto evalState = getEvalState(); auto flake = flake::lockFlake( + flakeSettings, *evalState, - parseFlakeRef(flakeRef, absPath(getCommandBaseDir())), + parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir())), { .writeLockFile = false }); for (auto & [inputName, input] : flake.lockFile.root->inputs) { auto input2 = flake.lockFile.findInput({inputName}); // resolve 'follows' nodes if (auto input3 = std::dynamic_pointer_cast(input2)) { overrideRegistry( - fetchers::Input::fromAttrs({{"type","indirect"}, {"id", inputName}}), + fetchers::Input::fromAttrs(fetchSettings, {{"type","indirect"}, {"id", inputName}}), input3->lockedRef.input, {}); } @@ -288,11 +291,11 @@ void SourceExprCommand::completeInstallable(AddCompletions & completions, std::s state->autoCallFunction(*autoArgs, v1, v2); if (v2.type() == nAttrs) { - for (auto & i : *v2.attrs) { - std::string name = state->symbols[i.name]; + for (auto & i : *v2.attrs()) { + std::string_view name = state->symbols[i.name]; if (name.find(searchWord) == 0) { if (prefix_ == "") - completions.add(name); + completions.add(std::string(name)); else completions.add(prefix_ + "." + name); } @@ -338,10 +341,11 @@ void completeFlakeRefWithFragment( auto flakeRefS = std::string(prefix.substr(0, hash)); // TODO: ideally this would use the command base directory instead of assuming ".". - auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath(".")); + auto flakeRef = parseFlakeRef(fetchSettings, expandTilde(flakeRefS), absPath(".")); auto evalCache = openEvalCache(*evalState, - std::make_shared(lockFlake(*evalState, flakeRef, lockFlags))); + std::make_shared(lockFlake( + flakeSettings, *evalState, flakeRef, lockFlags))); auto root = evalCache->getRoot(); @@ -372,6 +376,7 @@ void completeFlakeRefWithFragment( auto attrPath2 = (*attr)->getAttrPath(attr2); /* Strip the attrpath prefix. */ attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size()); + // FIXME: handle names with dots completions.add(flakeRefS + "#" + prefixRoot + concatStringsSep(".", evalState->symbols.resolve(attrPath2))); } } @@ -403,7 +408,7 @@ void completeFlakeRef(AddCompletions & completions, ref store, std::strin Args::completeDir(completions, 0, prefix); /* Look for registry entries that match the prefix. */ - for (auto & registry : fetchers::getRegistries(store)) { + for (auto & registry : fetchers::getRegistries(fetchSettings, store)) { for (auto & entry : registry->entries) { auto from = entry.from.to_string(); if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) { @@ -442,13 +447,10 @@ ref openEvalCache( EvalState & state, std::shared_ptr lockedFlake) { - auto fingerprint = lockedFlake->getFingerprint(); - return make_ref( - evalSettings.useEvalCache && evalSettings.pureEval - ? std::optional { std::cref(fingerprint) } - : std::nullopt, - state, - [&state, lockedFlake]() + auto fingerprint = evalSettings.useEvalCache && evalSettings.pureEval + ? lockedFlake->getFingerprint(state.store) + : std::nullopt; + auto rootLoader = [&state, lockedFlake]() { /* For testing whether the evaluation cache is complete. */ @@ -460,11 +462,21 @@ ref openEvalCache( state.forceAttrs(*vFlake, noPos, "while parsing cached flake data"); - auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); + auto aOutputs = vFlake->attrs()->get(state.symbols.create("outputs")); assert(aOutputs); return aOutputs->value; - }); + }; + + if (fingerprint) { + auto search = state.evalCaches.find(fingerprint.value()); + if (search == state.evalCaches.end()) { + search = state.evalCaches.emplace(fingerprint.value(), make_ref(fingerprint, state, rootLoader)).first; + } + return search->second; + } else { + return make_ref(std::nullopt, state, rootLoader); + } } Installables SourceExprCommand::parseInstallables( @@ -527,7 +539,8 @@ Installables SourceExprCommand::parseInstallables( } try { - auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, absPath(getCommandBaseDir())); + auto [flakeRef, fragment] = parseFlakeRefWithFragment( + fetchSettings, std::string { prefix }, absPath(getCommandBaseDir())); result.push_back(make_ref( this, getEvalState(), @@ -594,6 +607,37 @@ std::vector Installable::build( return res; } +static void throwBuildErrors( + std::vector & buildResults, + const Store & store) +{ + std::vector failed; + for (auto & buildResult : buildResults) { + if (!buildResult.success()) { + failed.push_back(buildResult); + } + } + + auto failedResult = failed.begin(); + if (failedResult != failed.end()) { + if (failed.size() == 1) { + failedResult->rethrow(); + } else { + StringSet failedPaths; + for (; failedResult != failed.end(); failedResult++) { + if (!failedResult->errorMsg.empty()) { + logError(ErrorInfo{ + .level = lvlError, + .msg = failedResult->errorMsg, + }); + } + failedPaths.insert(failedResult->path.to_string(store)); + } + throw Error("build of %s failed", concatStringsSep(", ", quoteStrings(failedPaths))); + } + } +} + std::vector, BuiltPathWithResult>> Installable::build2( ref evalStore, ref store, @@ -655,10 +699,9 @@ std::vector, BuiltPathWithResult>> Installable::build if (settings.printMissing) printMissing(store, pathsToBuild, lvlInfo); - for (auto & buildResult : store->buildPathsWithResults(pathsToBuild, bMode, evalStore)) { - if (!buildResult.success()) - buildResult.rethrow(); - + auto buildResults = store->buildPathsWithResults(pathsToBuild, bMode, evalStore); + throwBuildErrors(buildResults, *store); + for (auto & buildResult : buildResults) { for (auto & aux : backmap[buildResult.path]) { std::visit(overloaded { [&](const DerivedPath::Built & bfd) { @@ -814,6 +857,7 @@ std::vector RawInstallablesCommand::getFlakeRefsForCompletion() std::vector res; for (auto i : rawInstallables) res.push_back(parseFlakeRefWithFragment( + fetchSettings, expandTilde(i), absPath(getCommandBaseDir())).first); return res; @@ -836,6 +880,7 @@ std::vector InstallableCommand::getFlakeRefsForCompletion() { return { parseFlakeRefWithFragment( + fetchSettings, expandTilde(_installable), absPath(getCommandBaseDir())).first }; diff --git a/src/libcmd/local.mk b/src/libcmd/local.mk index abb7459a7..a270333f4 100644 --- a/src/libcmd/local.mk +++ b/src/libcmd/local.mk @@ -6,10 +6,10 @@ libcmd_DIR := $(d) libcmd_SOURCES := $(wildcard $(d)/*.cc) -libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers +libcmd_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) $(INCLUDE_libflake) $(INCLUDE_libmain) libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) $(THREAD_LDFLAGS) -libcmd_LIBS = libstore libutil libexpr libmain libfetchers +libcmd_LIBS = libutil libstore libfetchers libflake libexpr libmain $(eval $(call install-file-in, $(buildprefix)$(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644)) diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc index a4e3c5a77..6a0d05d9f 100644 --- a/src/libcmd/markdown.cc +++ b/src/libcmd/markdown.cc @@ -1,21 +1,23 @@ #include "markdown.hh" -#include "util.hh" +#include "environment-variables.hh" +#include "error.hh" #include "finally.hh" #include "terminal.hh" -#include #if HAVE_LOWDOWN -#include +# include +# include #endif namespace nix { -std::string renderMarkdownToTerminal(std::string_view markdown) -{ #if HAVE_LOWDOWN +static std::string doRenderMarkdownToTerminal(std::string_view markdown) +{ int windowWidth = getWindowSize().second; - struct lowdown_opts opts { + struct lowdown_opts opts + { .type = LOWDOWN_TERM, .maxdepth = 20, .cols = (size_t) std::max(windowWidth - 5, 60), @@ -50,10 +52,22 @@ std::string renderMarkdownToTerminal(std::string_view markdown) if (!rndr_res) throw Error("allocation error while rendering Markdown"); - return filterANSIEscapes(std::string(buf->data, buf->size), !shouldANSI()); -#else - return std::string(markdown); -#endif + return filterANSIEscapes(std::string(buf->data, buf->size), !isTTY()); } +std::string renderMarkdownToTerminal(std::string_view markdown) +{ + if (auto e = getEnv("_NIX_TEST_RAW_MARKDOWN"); e && *e == "1") + return std::string(markdown); + else + return doRenderMarkdownToTerminal(markdown); } + +#else +std::string renderMarkdownToTerminal(std::string_view markdown) +{ + return std::string(markdown); +} +#endif + +} // namespace nix diff --git a/src/libcmd/markdown.hh b/src/libcmd/markdown.hh index a04d32a4f..66db1736c 100644 --- a/src/libcmd/markdown.hh +++ b/src/libcmd/markdown.hh @@ -1,10 +1,17 @@ #pragma once ///@file -#include "types.hh" +#include namespace nix { +/** + * Render the given Markdown text to the terminal. + * + * If Nix is compiled without Markdown support, this function will return the input text as-is. + * + * The renderer takes into account the terminal width, and wraps text accordingly. + */ std::string renderMarkdownToTerminal(std::string_view markdown); } diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build new file mode 100644 index 000000000..c484cf998 --- /dev/null +++ b/src/libcmd/meson.build @@ -0,0 +1,130 @@ +project('nix-cmd', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ +] +deps_public_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), + dependency('nix-fetchers'), + dependency('nix-expr'), + dependency('nix-flake'), + dependency('nix-main'), +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') +deps_public += nlohmann_json + +lowdown = dependency('lowdown', version : '>= 0.9.0', required : get_option('markdown')) +deps_private += lowdown +configdata.set('HAVE_LOWDOWN', lowdown.found().to_int()) + +readline_flavor = get_option('readline-flavor') +if readline_flavor == 'editline' + editline = dependency('libeditline', 'editline', version : '>=1.14') + deps_private += editline +elif readline_flavor == 'readline' + readline = dependency('readline') + deps_private += readline + configdata.set( + 'USE_READLINE', + 1, + description: 'Use readline instead of editline', + ) +else + error('illegal editline flavor', readline_flavor) +endif + +config_h = configure_file( + configuration : configdata, + output : 'config-cmd.hh', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + # '-include', 'config-fetchers.h', + '-include', 'config-expr.hh', + '-include', 'config-main.hh', + '-include', 'config-cmd.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'built-path.cc', + 'command-installable-value.cc', + 'command.cc', + 'common-eval-args.cc', + 'editor-for.cc', + 'installable-attr-path.cc', + 'installable-derived-path.cc', + 'installable-flake.cc', + 'installable-value.cc', + 'installables.cc', + 'legacy.cc', + 'markdown.cc', + 'misc-store-flags.cc', + 'network-proxy.cc', + 'repl-interacter.cc', + 'repl.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'built-path.hh', + 'command-installable-value.hh', + 'command.hh', + 'common-eval-args.hh', + 'compatibility-settings.hh', + 'editor-for.hh', + 'installable-attr-path.hh', + 'installable-derived-path.hh', + 'installable-flake.hh', + 'installable-value.hh', + 'installables.hh', + 'legacy.hh', + 'markdown.hh', + 'misc-store-flags.hh', + 'network-proxy.hh', + 'repl-interacter.hh', + 'repl.hh', +) + +this_library = library( + 'nixcmd', + sources, + dependencies : deps_public + deps_private + deps_other, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libcmd/meson.options b/src/libcmd/meson.options new file mode 100644 index 000000000..79ae4fa55 --- /dev/null +++ b/src/libcmd/meson.options @@ -0,0 +1,15 @@ +# vim: filetype=meson + +option( + 'markdown', + type: 'feature', + description: 'Enable Markdown rendering in the Nix binary (requires lowdown)', +) + +option( + 'readline-flavor', + type : 'combo', + choices : ['editline', 'readline'], + value : 'editline', + description : 'Which library to use for nice line editing with the Nix language REPL', +) diff --git a/src/libcmd/misc-store-flags.cc b/src/libcmd/misc-store-flags.cc index e66d3f63b..06552c032 100644 --- a/src/libcmd/misc-store-flags.cc +++ b/src/libcmd/misc-store-flags.cc @@ -81,9 +81,15 @@ Args::Flag fileIngestionMethod(FileIngestionMethod * method) How to compute the hash of the input. One of: - - `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function. + - `nar` (the default): + Serialises the input as a + [Nix Archive](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) + and passes that to the hash function. - - `flat`: Assumes that the input is a single file and directly passes it to the hash function; + - `flat`: + Assumes that the input is a single file and + [directly passes](@docroot@/store/file-system-object/content-address.md#serial-flat) + it to the hash function. )", .labels = {"file-ingestion-method"}, .handler = {[method](std::string s) { @@ -101,15 +107,23 @@ Args::Flag contentAddressMethod(ContentAddressMethod * method) How to compute the content-address of the store object. One of: - - `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function. + - [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive) + (the default): + Serialises the input as a + [Nix Archive](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) + and passes that to the hash function. - - `flat`: Assumes that the input is a single file and directly passes it to the hash function; + - [`flat`](@docroot@/store/store-object/content-address.md#method-flat): + Assumes that the input is a single file and + [directly passes](@docroot@/store/file-system-object/content-address.md#serial-flat) + it to the hash function. - - `text`: Like `flat`, but used for - [derivations](@docroot@/glossary.md#store-derivation) serialized in store object and + - [`text`](@docroot@/store/store-object/content-address.md#method-text): + Like `flat`, but used for + [derivations](@docroot@/glossary.md#store-derivation) serialized in store object and [`builtins.toFile`](@docroot@/language/builtins.html#builtins-toFile). For advanced use-cases only; - for regular usage prefer `nar` and `flat. + for regular usage prefer `nar` and `flat`. )", .labels = {"content-address-method"}, .handler = {[method](std::string s) { diff --git a/src/libcmd/network-proxy.cc b/src/libcmd/network-proxy.cc new file mode 100644 index 000000000..738bf6147 --- /dev/null +++ b/src/libcmd/network-proxy.cc @@ -0,0 +1,50 @@ +#include "network-proxy.hh" + +#include + +#include "environment-variables.hh" + +namespace nix { + +static const StringSet lowercaseVariables{"http_proxy", "https_proxy", "ftp_proxy", "all_proxy", "no_proxy"}; + +static StringSet getAllVariables() +{ + StringSet variables = lowercaseVariables; + for (const auto & variable : lowercaseVariables) { + std::string upperVariable; + std::transform( + variable.begin(), variable.end(), upperVariable.begin(), [](unsigned char c) { return std::toupper(c); }); + variables.insert(std::move(upperVariable)); + } + return variables; +} + +const StringSet networkProxyVariables = getAllVariables(); + +static StringSet getExcludingNoProxyVariables() +{ + static const StringSet excludeVariables{"no_proxy", "NO_PROXY"}; + StringSet variables; + std::set_difference( + networkProxyVariables.begin(), + networkProxyVariables.end(), + excludeVariables.begin(), + excludeVariables.end(), + std::inserter(variables, variables.begin())); + return variables; +} + +static const StringSet excludingNoProxyVariables = getExcludingNoProxyVariables(); + +bool haveNetworkProxyConnection() +{ + for (const auto & variable : excludingNoProxyVariables) { + if (getEnv(variable).has_value()) { + return true; + } + } + return false; +} + +} diff --git a/src/libcmd/network-proxy.hh b/src/libcmd/network-proxy.hh new file mode 100644 index 000000000..0b6856acb --- /dev/null +++ b/src/libcmd/network-proxy.hh @@ -0,0 +1,22 @@ +#pragma once +///@file + +#include "types.hh" + +namespace nix { + +/** + * Environment variables relating to network proxying. These are used by + * a few misc commands. + * + * See the Environment section of https://curl.se/docs/manpage.html for details. + */ +extern const StringSet networkProxyVariables; + +/** + * Heuristically check if there is a proxy connection by checking for defined + * proxy variables. + */ +bool haveNetworkProxyConnection(); + +} diff --git a/src/libcmd/package.nix b/src/libcmd/package.nix new file mode 100644 index 000000000..cde494901 --- /dev/null +++ b/src/libcmd/package.nix @@ -0,0 +1,104 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-util +, nix-store +, nix-fetchers +, nix-expr +, nix-flake +, nix-main +, editline +, readline +, lowdown +, nlohmann_json + +# Configuration Options + +, version + +# Whether to enable Markdown rendering in the Nix binary. +, enableMarkdown ? !stdenv.hostPlatform.isWindows + +# Which interactive line editor library to use for Nix's repl. +# +# Currently supported choices are: +# +# - editline (default) +# - readline +, readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline" +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-cmd"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + ({ inherit editline readline; }.${readlineFlavor}) + ] ++ lib.optional enableMarkdown lowdown; + + propagatedBuildInputs = [ + nix-util + nix-store + nix-fetchers + nix-expr + nix-flake + nix-main + nlohmann_json + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + mesonFlags = [ + (lib.mesonEnable "markdown" enableMarkdown) + (lib.mesonOption "readline-flavor" readlineFlavor) + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libcmd/repl-interacter.cc b/src/libcmd/repl-interacter.cc new file mode 100644 index 000000000..187af46ea --- /dev/null +++ b/src/libcmd/repl-interacter.cc @@ -0,0 +1,206 @@ +#include + +#ifdef USE_READLINE +#include +#include +#else +// editline < 1.15.2 don't wrap their API for C++ usage +// (added in https://github.com/troglobit/editline/commit/91398ceb3427b730995357e9d120539fb9bb7461). +// This results in linker errors due to to name-mangling of editline C symbols. +// For compatibility with these versions, we wrap the API here +// (wrapping multiple times on newer versions is no problem). +extern "C" { +#include +} +#endif + +#include "signals.hh" +#include "finally.hh" +#include "repl-interacter.hh" +#include "file-system.hh" +#include "repl.hh" +#include "environment-variables.hh" + +namespace nix { + +namespace { +// Used to communicate to NixRepl::getLine whether a signal occurred in ::readline. +volatile sig_atomic_t g_signal_received = 0; + +void sigintHandler(int signo) +{ + g_signal_received = signo; +} +}; + +static detail::ReplCompleterMixin * curRepl; // ugly + +#ifndef USE_READLINE +static char * completionCallback(char * s, int * match) +{ + auto possible = curRepl->completePrefix(s); + if (possible.size() == 1) { + *match = 1; + auto * res = strdup(possible.begin()->c_str() + strlen(s)); + if (!res) + throw Error("allocation failure"); + return res; + } else if (possible.size() > 1) { + auto checkAllHaveSameAt = [&](size_t pos) { + auto & first = *possible.begin(); + for (auto & p : possible) { + if (p.size() <= pos || p[pos] != first[pos]) + return false; + } + return true; + }; + size_t start = strlen(s); + size_t len = 0; + while (checkAllHaveSameAt(start + len)) + ++len; + if (len > 0) { + *match = 1; + auto * res = strdup(std::string(*possible.begin(), start, len).c_str()); + if (!res) + throw Error("allocation failure"); + return res; + } + } + + *match = 0; + return nullptr; +} + +static int listPossibleCallback(char * s, char *** avp) +{ + auto possible = curRepl->completePrefix(s); + + if (possible.size() > (std::numeric_limits::max() / sizeof(char *))) + throw Error("too many completions"); + + int ac = 0; + char ** vp = nullptr; + + auto check = [&](auto * p) { + if (!p) { + if (vp) { + while (--ac >= 0) + free(vp[ac]); + free(vp); + } + throw Error("allocation failure"); + } + return p; + }; + + vp = check((char **) malloc(possible.size() * sizeof(char *))); + + for (auto & p : possible) + vp[ac++] = check(strdup(p.c_str())); + + *avp = vp; + + return ac; +} +#endif + +ReadlineLikeInteracter::Guard ReadlineLikeInteracter::init(detail::ReplCompleterMixin * repl) +{ + // Allow nix-repl specific settings in .inputrc + rl_readline_name = "nix-repl"; + try { + createDirs(dirOf(historyFile)); + } catch (SystemError & e) { + logWarning(e.info()); + } +#ifndef USE_READLINE + el_hist_size = 1000; +#endif + read_history(historyFile.c_str()); + auto oldRepl = curRepl; + curRepl = repl; + Guard restoreRepl([oldRepl] { curRepl = oldRepl; }); +#ifndef USE_READLINE + rl_set_complete_func(completionCallback); + rl_set_list_possib_func(listPossibleCallback); +#endif + return restoreRepl; +} + +static constexpr const char * promptForType(ReplPromptType promptType) +{ + switch (promptType) { + case ReplPromptType::ReplPrompt: + return "nix-repl> "; + case ReplPromptType::ContinuationPrompt: + return " "; + } + assert(false); +} + +bool ReadlineLikeInteracter::getLine(std::string & input, ReplPromptType promptType) +{ +#ifndef _WIN32 // TODO use more signals.hh for this + struct sigaction act, old; + sigset_t savedSignalMask, set; + + auto setupSignals = [&]() { + act.sa_handler = sigintHandler; + sigfillset(&act.sa_mask); + act.sa_flags = 0; + if (sigaction(SIGINT, &act, &old)) + throw SysError("installing handler for SIGINT"); + + sigemptyset(&set); + sigaddset(&set, SIGINT); + if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask)) + throw SysError("unblocking SIGINT"); + }; + auto restoreSignals = [&]() { + if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) + throw SysError("restoring signals"); + + if (sigaction(SIGINT, &old, 0)) + throw SysError("restoring handler for SIGINT"); + }; + + setupSignals(); +#endif + char * s = readline(promptForType(promptType)); + Finally doFree([&]() { free(s); }); +#ifndef _WIN32 // TODO use more signals.hh for this + restoreSignals(); +#endif + + if (g_signal_received) { + g_signal_received = 0; + input.clear(); + return true; + } + + // editline doesn't echo the input to the output when non-interactive, unlike readline + // this results in a different behavior when running tests. The echoing is + // quite useful for reading the test output, so we add it here. + if (auto e = getEnv("_NIX_TEST_REPL_ECHO"); s && e && *e == "1") + { +#ifndef USE_READLINE + // This is probably not right for multi-line input, but we don't use that + // in the characterisation tests, so it's fine. + std::cout << promptForType(promptType) << s << std::endl; +#endif + } + + if (!s) + return false; + input += s; + input += '\n'; + + return true; +} + +ReadlineLikeInteracter::~ReadlineLikeInteracter() +{ + write_history(historyFile.c_str()); +} + +}; diff --git a/src/libcmd/repl-interacter.hh b/src/libcmd/repl-interacter.hh new file mode 100644 index 000000000..cc70efd07 --- /dev/null +++ b/src/libcmd/repl-interacter.hh @@ -0,0 +1,48 @@ +#pragma once +/// @file + +#include "finally.hh" +#include "types.hh" +#include +#include + +namespace nix { + +namespace detail { +/** Provides the completion hooks for the repl, without exposing its complete + * internals. */ +struct ReplCompleterMixin { + virtual StringSet completePrefix(const std::string & prefix) = 0; +}; +}; + +enum class ReplPromptType { + ReplPrompt, + ContinuationPrompt, +}; + +class ReplInteracter +{ +public: + using Guard = Finally>; + + virtual Guard init(detail::ReplCompleterMixin * repl) = 0; + /** Returns a boolean of whether the interacter got EOF */ + virtual bool getLine(std::string & input, ReplPromptType promptType) = 0; + virtual ~ReplInteracter(){}; +}; + +class ReadlineLikeInteracter : public virtual ReplInteracter +{ + std::string historyFile; +public: + ReadlineLikeInteracter(std::string historyFile) + : historyFile(historyFile) + { + } + virtual Guard init(detail::ReplCompleterMixin * repl) override; + virtual bool getLine(std::string & input, ReplPromptType promptType) override; + virtual ~ReadlineLikeInteracter() override; +}; + +}; diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 8b83608fa..efc04b029 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -1,34 +1,17 @@ #include #include #include -#include - -#include - -#ifdef USE_READLINE -#include -#include -#else -// editline < 1.15.2 don't wrap their API for C++ usage -// (added in https://github.com/troglobit/editline/commit/91398ceb3427b730995357e9d120539fb9bb7461). -// This results in linker errors due to to name-mangling of editline C symbols. -// For compatibility with these versions, we wrap the API here -// (wrapping multiple times on newer versions is no problem). -extern "C" { -#include -} -#endif +#include "repl-interacter.hh" #include "repl.hh" #include "ansicolor.hh" -#include "signals.hh" #include "shared.hh" +#include "config-global.hh" #include "eval.hh" -#include "eval-cache.hh" -#include "eval-inline.hh" #include "eval-settings.hh" #include "attr-path.hh" +#include "signals.hh" #include "store-api.hh" #include "log-store.hh" #include "common-eval-args.hh" @@ -38,18 +21,21 @@ extern "C" { #include "flake/flake.hh" #include "flake/lockfile.hh" #include "users.hh" -#include "terminal.hh" #include "editor-for.hh" #include "finally.hh" #include "markdown.hh" #include "local-fs-store.hh" #include "print.hh" +#include "ref.hh" +#include "value.hh" #if HAVE_BOEHMGC #define GC_INCLUDE_NEW #include #endif +#include "strings.hh" + namespace nix { /** @@ -75,6 +61,7 @@ enum class ProcessLineResult { struct NixRepl : AbstractNixRepl + , detail::ReplCompleterMixin #if HAVE_BOEHMGC , gc #endif @@ -90,17 +77,16 @@ struct NixRepl int displ; StringSet varNames; - const Path historyFile; + std::unique_ptr interacter; - NixRepl(const SearchPath & searchPath, nix::ref store,ref state, + NixRepl(const LookupPath & lookupPath, nix::ref store,ref state, std::function getValues); - virtual ~NixRepl(); + virtual ~NixRepl() = default; ReplExitStatus mainLoop() override; void initEnv() override; - StringSet completePrefix(const std::string & prefix); - bool getLine(std::string & input, const std::string & prompt); + virtual StringSet completePrefix(const std::string & prefix) override; StorePath getDerivationPath(Value & v); ProcessLineResult processLine(std::string line); @@ -123,7 +109,8 @@ struct NixRepl .force = true, .derivationPaths = true, .maxDepth = maxDepth, - .prettyIndent = 2 + .prettyIndent = 2, + .errors = ErrorPrintBehavior::ThrowTopLevel, }); } }; @@ -137,111 +124,33 @@ std::string removeWhitespace(std::string s) } -NixRepl::NixRepl(const SearchPath & searchPath, nix::ref store, ref state, +NixRepl::NixRepl(const LookupPath & lookupPath, nix::ref store, ref state, std::function getValues) : AbstractNixRepl(state) , debugTraceIndex(0) , getValues(getValues) , staticEnv(new StaticEnv(nullptr, state->staticBaseEnv.get())) - , historyFile(getDataDir() + "/nix/repl-history") + , interacter(make_unique(getDataDir() + "/nix/repl-history")) { } - -NixRepl::~NixRepl() -{ - write_history(historyFile.c_str()); -} - void runNix(Path program, const Strings & args, const std::optional & input = {}) { auto subprocessEnv = getEnv(); subprocessEnv["NIX_CONFIG"] = globalConfig.toKeyValue(); - + //isInteractive avoid grabling interactive commands runProgram2(RunOptions { .program = settings.nixBinDir+ "/" + program, .args = args, .environment = subprocessEnv, .input = input, + .isInteractive = true, }); return; } -static NixRepl * curRepl; // ugly - -static char * completionCallback(char * s, int *match) { - auto possible = curRepl->completePrefix(s); - if (possible.size() == 1) { - *match = 1; - auto *res = strdup(possible.begin()->c_str() + strlen(s)); - if (!res) throw Error("allocation failure"); - return res; - } else if (possible.size() > 1) { - auto checkAllHaveSameAt = [&](size_t pos) { - auto &first = *possible.begin(); - for (auto &p : possible) { - if (p.size() <= pos || p[pos] != first[pos]) - return false; - } - return true; - }; - size_t start = strlen(s); - size_t len = 0; - while (checkAllHaveSameAt(start + len)) ++len; - if (len > 0) { - *match = 1; - auto *res = strdup(std::string(*possible.begin(), start, len).c_str()); - if (!res) throw Error("allocation failure"); - return res; - } - } - - *match = 0; - return nullptr; -} - -static int listPossibleCallback(char *s, char ***avp) { - auto possible = curRepl->completePrefix(s); - - if (possible.size() > (INT_MAX / sizeof(char*))) - throw Error("too many completions"); - - int ac = 0; - char **vp = nullptr; - - auto check = [&](auto *p) { - if (!p) { - if (vp) { - while (--ac >= 0) - free(vp[ac]); - free(vp); - } - throw Error("allocation failure"); - } - return p; - }; - - vp = check((char **)malloc(possible.size() * sizeof(char*))); - - for (auto & p : possible) - vp[ac++] = check(strdup(p.c_str())); - - *avp = vp; - - return ac; -} - -namespace { - // Used to communicate to NixRepl::getLine whether a signal occurred in ::readline. - volatile sig_atomic_t g_signal_received = 0; - - void sigintHandler(int signo) { - g_signal_received = signo; - } -} - static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positions, const DebugTrace & dt) { if (dt.isError) @@ -281,24 +190,7 @@ ReplExitStatus NixRepl::mainLoop() loadFiles(); - // Allow nix-repl specific settings in .inputrc - rl_readline_name = "nix-repl"; - try { - createDirs(dirOf(historyFile)); - } catch (SystemError & e) { - logWarning(e.info()); - } -#ifndef USE_READLINE - el_hist_size = 1000; -#endif - read_history(historyFile.c_str()); - auto oldRepl = curRepl; - curRepl = this; - Finally restoreRepl([&] { curRepl = oldRepl; }); -#ifndef USE_READLINE - rl_set_complete_func(completionCallback); - rl_set_list_possib_func(listPossibleCallback); -#endif + auto _guard = interacter->init(static_cast(this)); std::string input; @@ -307,7 +199,7 @@ ReplExitStatus NixRepl::mainLoop() logger->pause(); // When continuing input from previous lines, don't print a prompt, just align to the same // number of chars as the prompt. - if (!getLine(input, input.empty() ? "nix-repl> " : " ")) { + if (!interacter->getLine(input, input.empty() ? ReplPromptType::ReplPrompt : ReplPromptType::ContinuationPrompt)) { // Ctrl-D should exit the debugger. state->debugStop = false; logger->cout(""); @@ -325,7 +217,7 @@ ReplExitStatus NixRepl::mainLoop() case ProcessLineResult::PromptAgain: break; default: - abort(); + unreachable(); } } catch (ParseError & e) { if (e.msg().find("unexpected end of file") != std::string::npos) { @@ -336,13 +228,7 @@ ReplExitStatus NixRepl::mainLoop() printMsg(lvlError, e.msg()); } } catch (EvalError & e) { - // in debugger mode, an EvalError should trigger another repl session. - // when that session returns the exception will land here. No need to show it again; - // show the error for this repl session instead. - if (state->debugRepl && !state->debugTraces.empty()) - showDebugTrace(std::cout, state->positions, state->debugTraces.front()); - else - printMsg(lvlError, e.msg()); + printMsg(lvlError, e.msg()); } catch (Error & e) { printMsg(lvlError, e.msg()); } catch (Interrupted & e) { @@ -356,51 +242,6 @@ ReplExitStatus NixRepl::mainLoop() } } - -bool NixRepl::getLine(std::string & input, const std::string & prompt) -{ - struct sigaction act, old; - sigset_t savedSignalMask, set; - - auto setupSignals = [&]() { - act.sa_handler = sigintHandler; - sigfillset(&act.sa_mask); - act.sa_flags = 0; - if (sigaction(SIGINT, &act, &old)) - throw SysError("installing handler for SIGINT"); - - sigemptyset(&set); - sigaddset(&set, SIGINT); - if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask)) - throw SysError("unblocking SIGINT"); - }; - auto restoreSignals = [&]() { - if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) - throw SysError("restoring signals"); - - if (sigaction(SIGINT, &old, 0)) - throw SysError("restoring handler for SIGINT"); - }; - - setupSignals(); - char * s = readline(prompt.c_str()); - Finally doFree([&]() { free(s); }); - restoreSignals(); - - if (g_signal_received) { - g_signal_received = 0; - input.clear(); - return true; - } - - if (!s) - return false; - input += s; - input += '\n'; - return true; -} - - StringSet NixRepl::completePrefix(const std::string & prefix) { StringSet completions; @@ -421,11 +262,14 @@ StringSet NixRepl::completePrefix(const std::string & prefix) try { auto dir = std::string(cur, 0, slash); auto prefix2 = std::string(cur, slash + 1); - for (auto & entry : readDirectory(dir == "" ? "/" : dir)) { - if (entry.name[0] != '.' && hasPrefix(entry.name, prefix2)) - completions.insert(prev + dir + "/" + entry.name); + for (auto & entry : std::filesystem::directory_iterator{dir == "" ? "/" : dir}) { + checkInterrupt(); + auto name = entry.path().filename().string(); + if (name[0] != '.' && hasPrefix(name, prefix2)) + completions.insert(prev + entry.path().string()); } } catch (Error &) { + } catch (std::filesystem::filesystem_error &) { } } else if ((dot = cur.rfind('.')) == std::string::npos) { /* This is a variable name; look it up in the current scope. */ @@ -452,7 +296,7 @@ StringSet NixRepl::completePrefix(const std::string & prefix) e->eval(*state, *env, v); state->forceAttrs(v, noPos, "while evaluating an attrset for the purpose of completion (this error should not be displayed; file an issue?)"); - for (auto & i : *v.attrs) { + for (auto & i : *v.attrs()) { std::string_view name = state->symbols[i.name]; if (name.substr(0, cur2.size()) != cur2) continue; completions.insert(concatStrings(prev, expr, ".", name)); @@ -464,6 +308,8 @@ StringSet NixRepl::completePrefix(const std::string & prefix) // Quietly ignore evaluation errors. } catch (BadURL & e) { // Quietly ignore BadURL flake-related errors. + } catch (FileNotFound & e) { + // Quietly ignore non-existent file beeing `import`-ed. } } @@ -519,7 +365,7 @@ ProcessLineResult NixRepl::processLine(std::string line) if (line.empty()) return ProcessLineResult::PromptAgain; - _isInterrupted = false; + setInterrupted(false); std::string command, arg; @@ -548,6 +394,7 @@ ProcessLineResult NixRepl::processLine(std::string line) << " :l, :load Load Nix expression and add it to scope\n" << " :lf, :load-flake Load Nix flake and add it to scope\n" << " :p, :print Evaluate and print expression recursively\n" + << " Strings are printed directly, without escaping.\n" << " :q, :quit Exit nix-repl\n" << " :r, :reload Reload all files\n" << " :sh Build dependencies of derivation, then start\n" @@ -651,7 +498,7 @@ ProcessLineResult NixRepl::processLine(std::string line) auto path = state->coerceToPath(noPos, v, context, "while evaluating the filename to edit"); return {path, 0}; } else if (v.isLambda()) { - auto pos = state->positions[v.lambda.fun->pos]; + auto pos = state->positions[v.payload.lambda.fun->pos]; if (auto path = std::get_if(&pos.origin)) return {*path, pos.line}; else @@ -669,7 +516,7 @@ ProcessLineResult NixRepl::processLine(std::string line) // runProgram redirects stdout to a StringSink, // using runProgram2 to allow editors to display their UI - runProgram2(RunOptions { .program = editor, .searchPath = true, .args = args }); + runProgram2(RunOptions { .program = editor, .lookupPath = true, .args = args , .isInteractive = true }); // Reload right after exiting the editor state->resetFileCache(); @@ -755,7 +602,11 @@ ProcessLineResult NixRepl::processLine(std::string line) else if (command == ":p" || command == ":print") { Value v; evalString(arg, v); - printValue(std::cout, v); + if (v.type() == nString) { + std::cout << v.string_view(); + } else { + printValue(std::cout, v); + } std::cout << std::endl; } @@ -766,6 +617,35 @@ ProcessLineResult NixRepl::processLine(std::string line) else if (command == ":doc") { Value v; + + auto expr = parseString(arg); + std::string fallbackName; + PosIdx fallbackPos; + DocComment fallbackDoc; + if (auto select = dynamic_cast(expr)) { + Value vAttrs; + auto name = select->evalExceptFinalSelect(*state, *env, vAttrs); + fallbackName = state->symbols[name]; + + state->forceAttrs(vAttrs, noPos, "while evaluating an attribute set to look for documentation"); + auto attrs = vAttrs.attrs(); + assert(attrs); + auto attr = attrs->get(name); + if (!attr) { + // When missing, trigger the normal exception + // e.g. :doc builtins.foo + // behaves like + // nix-repl> builtins.foo + // error: attribute 'foo' missing + evalString(arg, v); + assert(false); + } + if (attr->pos) { + fallbackPos = attr->pos; + fallbackDoc = state->getDocCommentForPos(fallbackPos); + } + } + evalString(arg, v); if (auto doc = state->getDoc(v)) { std::string markdown; @@ -783,6 +663,19 @@ ProcessLineResult NixRepl::processLine(std::string line) markdown += stripIndentation(doc->doc); logger->cout(trim(renderMarkdownToTerminal(markdown))); + } else if (fallbackPos) { + std::stringstream ss; + ss << "Attribute `" << fallbackName << "`\n\n"; + ss << " … defined at " << state->positions[fallbackPos] << "\n\n"; + if (fallbackDoc) { + ss << fallbackDoc.getInnerText(state->positions); + } else { + ss << "No documentation found.\n\n"; + } + + auto markdown = ss.str(); + logger->cout(trim(renderMarkdownToTerminal(markdown))); + } else throw Error("value does not have documentation"); } @@ -840,14 +733,14 @@ void NixRepl::loadFlake(const std::string & flakeRefS) if (flakeRefS.empty()) throw Error("cannot use ':load-flake' without a path specified. (Use '.' for the current working directory.)"); - auto flakeRef = parseFlakeRef(flakeRefS, absPath("."), true); + auto flakeRef = parseFlakeRef(fetchSettings, flakeRefS, absPath("."), true); if (evalSettings.pureEval && !flakeRef.input.isLocked()) throw Error("cannot use ':load-flake' on locked flake reference '%s' (use --impure to override)", flakeRefS); Value v; flake::callFlake(*state, - flake::lockFlake(*state, flakeRef, + flake::lockFlake(flakeSettings, *state, flakeRef, flake::LockFlags { .updateLockFile = false, .useRegistries = !evalSettings.pureEval, @@ -899,17 +792,17 @@ void NixRepl::loadFiles() void NixRepl::addAttrsToScope(Value & attrs) { state->forceAttrs(attrs, [&]() { return attrs.determinePos(noPos); }, "while evaluating an attribute set to be merged in the global scope"); - if (displ + attrs.attrs->size() >= envSize) + if (displ + attrs.attrs()->size() >= envSize) throw Error("environment full; cannot add more variables"); - for (auto & i : *attrs.attrs) { + for (auto & i : *attrs.attrs()) { staticEnv->vars.emplace_back(i.name, displ); env->values[displ++] = i.value; varNames.emplace(state->symbols[i.name]); } staticEnv->sort(); staticEnv->deduplicate(); - notice("Added %1% variables.", attrs.attrs->size()); + notice("Added %1% variables.", attrs.attrs()->size()); } @@ -941,11 +834,11 @@ void NixRepl::evalString(std::string s, Value & v) std::unique_ptr AbstractNixRepl::create( - const SearchPath & searchPath, nix::ref store, ref state, + const LookupPath & lookupPath, nix::ref store, ref state, std::function getValues) { return std::make_unique( - searchPath, + lookupPath, openStore(), state, getValues @@ -961,9 +854,9 @@ ReplExitStatus AbstractNixRepl::runSimple( NixRepl::AnnotatedValues values; return values; }; - SearchPath searchPath = {}; + LookupPath lookupPath = {}; auto repl = std::make_unique( - searchPath, + lookupPath, openStore(), evalState, getValues diff --git a/src/libcmd/repl.hh b/src/libcmd/repl.hh index 21aa8bfc7..3fd4b2c39 100644 --- a/src/libcmd/repl.hh +++ b/src/libcmd/repl.hh @@ -3,11 +3,6 @@ #include "eval.hh" -#if HAVE_BOEHMGC -#define GC_INCLUDE_NEW -#include -#endif - namespace nix { struct AbstractNixRepl @@ -25,7 +20,7 @@ struct AbstractNixRepl typedef std::vector> AnnotatedValues; static std::unique_ptr create( - const SearchPath & searchPath, nix::ref store, ref state, + const LookupPath & lookupPath, nix::ref store, ref state, std::function getValues); static ReplExitStatus runSimple( diff --git a/src/libexpr-c/.version b/src/libexpr-c/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libexpr-c/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libexpr-c/build-utils-meson b/src/libexpr-c/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libexpr-c/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libexpr-c/local.mk b/src/libexpr-c/local.mk new file mode 100644 index 000000000..227a4095b --- /dev/null +++ b/src/libexpr-c/local.mk @@ -0,0 +1,25 @@ +libraries += libexprc + +libexprc_NAME = libnixexprc + +libexprc_DIR := $(d) + +libexprc_SOURCES := \ + $(wildcard $(d)/*.cc) \ + +# Not just for this library itself, but also for downstream libraries using this library + +INCLUDE_libexprc := -I $(d) +libexprc_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libutilc) \ + $(INCLUDE_libfetchers) \ + $(INCLUDE_libstore) $(INCLUDE_libstorec) \ + $(INCLUDE_libexpr) $(INCLUDE_libexprc) + +libexprc_LIBS = libutil libutilc libstore libstorec libfetchers libexpr + +libexprc_LDFLAGS += $(THREAD_LDFLAGS) + +$(eval $(call install-file-in, $(d)/nix-expr-c.pc, $(libdir)/pkgconfig, 0644)) + +libexprc_FORCE_INSTALL := 1 + diff --git a/src/libexpr-c/meson.build b/src/libexpr-c/meson.build new file mode 100644 index 000000000..6db5b83b8 --- /dev/null +++ b/src/libexpr-c/meson.build @@ -0,0 +1,93 @@ +project('nix-expr-c', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), + dependency('nix-expr'), +] +deps_public_maybe_subproject = [ + dependency('nix-util-c'), + dependency('nix-store-c'), +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +# TODO rename, because it will conflict with downstream projects +configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) + +config_h = configure_file( + configuration : configdata, + output : 'config-expr.h', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + + # From C++ libraries, only for internals + '-include', 'config-util.hh', + '-include', 'config-store.hh', + '-include', 'config-expr.hh', + + # From C libraries, for our public, installed headers too + '-include', 'config-util.h', + '-include', 'config-store.h', + '-include', 'config-expr.h', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'nix_api_expr.cc', + 'nix_api_external.cc', + 'nix_api_value.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'nix_api_expr.h', + 'nix_api_external.h', + 'nix_api_value.h', +) + +# TODO move this header to libexpr, maybe don't use it in tests? +headers += files('nix_api_expr_internal.h') + +subdir('build-utils-meson/export-all-symbols') + +this_library = library( + 'nixexprc', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + link_args: linker_export_flags, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libexpr-c/nix-expr-c.pc.in b/src/libexpr-c/nix-expr-c.pc.in new file mode 100644 index 000000000..06897064d --- /dev/null +++ b/src/libexpr-c/nix-expr-c.pc.in @@ -0,0 +1,10 @@ +prefix=@prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: Nix +Description: Nix Language Evaluator - C API +Version: @PACKAGE_VERSION@ +Requires: nix-store-c +Libs: -L${libdir} -lnixexprc +Cflags: -I${includedir}/nix diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc new file mode 100644 index 000000000..8f21d7022 --- /dev/null +++ b/src/libexpr-c/nix_api_expr.cc @@ -0,0 +1,213 @@ +#include +#include +#include + +#include "eval.hh" +#include "eval-gc.hh" +#include "globals.hh" +#include "eval-settings.hh" + +#include "nix_api_expr.h" +#include "nix_api_expr_internal.h" +#include "nix_api_store.h" +#include "nix_api_store_internal.h" +#include "nix_api_util.h" +#include "nix_api_util_internal.h" + +#if HAVE_BOEHMGC +# include +# define GC_INCLUDE_NEW 1 +# include "gc_cpp.h" +#endif + +nix_err nix_libexpr_init(nix_c_context * context) +{ + if (context) + context->last_err_code = NIX_OK; + { + auto ret = nix_libutil_init(context); + if (ret != NIX_OK) + return ret; + } + { + auto ret = nix_libstore_init(context); + if (ret != NIX_OK) + return ret; + } + try { + nix::initGC(); + } + NIXC_CATCH_ERRS +} + +nix_err nix_expr_eval_from_string( + nix_c_context * context, EvalState * state, const char * expr, const char * path, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + nix::Expr * parsedExpr = state->state.parseExprFromString(expr, state->state.rootPath(nix::CanonPath(path))); + state->state.eval(parsedExpr, value->value); + state->state.forceValue(value->value, nix::noPos); + } + NIXC_CATCH_ERRS +} + +nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, nix_value * arg, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + state->state.callFunction(fn->value, arg->value, value->value, nix::noPos); + state->state.forceValue(value->value, nix::noPos); + } + NIXC_CATCH_ERRS +} + +nix_err nix_value_call_multi(nix_c_context * context, EvalState * state, nix_value * fn, size_t nargs, nix_value ** args, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + state->state.callFunction(fn->value, nargs, (nix::Value * *)args, value->value, nix::noPos); + state->state.forceValue(value->value, nix::noPos); + } + NIXC_CATCH_ERRS +} + +nix_err nix_value_force(nix_c_context * context, EvalState * state, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + state->state.forceValue(value->value, nix::noPos); + } + NIXC_CATCH_ERRS +} + +nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + state->state.forceValueDeep(value->value); + } + NIXC_CATCH_ERRS +} + +EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c, Store * store) +{ + if (context) + context->last_err_code = NIX_OK; + try { + nix::Strings lookupPath; + if (lookupPath_c != nullptr) + for (size_t i = 0; lookupPath_c[i] != nullptr; i++) + lookupPath.push_back(lookupPath_c[i]); + + void * p = ::operator new( + sizeof(EvalState), + static_cast(alignof(EvalState))); + auto * p2 = static_cast(p); + new (p) EvalState { + .fetchSettings = nix::fetchers::Settings{}, + .settings = nix::EvalSettings{ + nix::settings.readOnlyMode, + }, + .state = nix::EvalState( + nix::LookupPath::parse(lookupPath), + store->ptr, + p2->fetchSettings, + p2->settings), + }; + loadConfFile(p2->settings); + return p2; + } + NIXC_CATCH_ERRS_NULL +} + +void nix_state_free(EvalState * state) +{ + delete state; +} + +#if HAVE_BOEHMGC +std::unordered_map< + const void *, + unsigned int, + std::hash, + std::equal_to, + traceable_allocator
; }' - > - > - > - > - > - > - > - > - > ``` + Just parse the input files, and print their abstract syntax trees on + standard output as a Nix expression. - - `--find-file`\ - Look up the given files in Nix’s search path (as specified by the - `NIX_PATH` environment variable). If found, print the corresponding - absolute paths on standard output. For instance, if `NIX_PATH` is - `nixpkgs=/home/alice/nixpkgs`, then `nix-instantiate --find-file - nixpkgs/default.nix` will print `/home/alice/nixpkgs/default.nix`. +- `--eval` - - `--strict`\ - When used with `--eval`, recursively evaluate list elements and - attributes. Normally, such sub-expressions are left unevaluated - (since the Nix language is lazy). + Just parse and evaluate the input files, and print the resulting + values on standard output. No instantiation of store derivations + takes place. - > **Warning** - > - > This option can cause non-termination, because lazy data - > structures can be infinitely large. + > **Warning** + > + > This option produces output which can be parsed as a Nix expression which + > will produce a different result than the input expression when evaluated. + > For example, these two Nix expressions print the same result despite + > having different meaning: + > + > ```console + > $ nix-instantiate --eval --expr '{ a = {}; }' + > { a = ; } + > $ nix-instantiate --eval --expr '{ a = ; }' + > { a = ; } + > ``` + > + > For human-readable output, `nix eval` (experimental) is more informative: + > + > ```console + > $ nix-instantiate --eval --expr 'a: a' + > + > $ nix eval --expr 'a: a' + > «lambda @ «string»:1:1» + > ``` + > + > For machine-readable output, the `--xml` option produces unambiguous + > output: + > + > ```console + > $ nix-instantiate --eval --xml --expr '{ foo = ; }' + > + > + > + > + > + > + > + > + > ``` - - `--json`\ - When used with `--eval`, print the resulting value as an JSON - representation of the abstract syntax tree rather than as a Nix expression. +- `--find-file` - - `--xml`\ - When used with `--eval`, print the resulting value as an XML - representation of the abstract syntax tree rather than as a Nix expression. - The schema is the same as that used by the [`toXML` - built-in](../language/builtins.md). + Look up the given files in Nix’s search path (as specified by the + `NIX_PATH` environment variable). If found, print the corresponding + absolute paths on standard output. For instance, if `NIX_PATH` is + `nixpkgs=/home/alice/nixpkgs`, then `nix-instantiate --find-file + nixpkgs/default.nix` will print `/home/alice/nixpkgs/default.nix`. - - `--read-write-mode`\ - When used with `--eval`, perform evaluation in read/write mode so - nix language features that require it will still work (at the cost - of needing to do instantiation of every evaluated derivation). If - this option is not enabled, there may be uninstantiated store paths - in the final output. +- `--strict` + + When used with `--eval`, recursively evaluate list elements and + attributes. Normally, such sub-expressions are left unevaluated + (since the Nix language is lazy). + + > **Warning** + > + > This option can cause non-termination, because lazy data + > structures can be infinitely large. + +- `--json` + + When used with `--eval`, print the resulting value as an JSON + representation of the abstract syntax tree rather than as a Nix expression. + +- `--xml` + + When used with `--eval`, print the resulting value as an XML + representation of the abstract syntax tree rather than as a Nix expression. + The schema is the same as that used by the [`toXML` + built-in](../language/builtins.md). + +- `--read-write-mode` + + When used with `--eval`, perform evaluation in read/write mode so + nix language features that require it will still work (at the cost + of needing to do instantiation of every evaluated derivation). If + this option is not enabled, there may be uninstantiated store paths + in the final output. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-prefetch-url.md b/doc/manual/src/command-ref/nix-prefetch-url.md index 45ef01e02..ffab94b8a 100644 --- a/doc/manual/src/command-ref/nix-prefetch-url.md +++ b/doc/manual/src/command-ref/nix-prefetch-url.md @@ -39,27 +39,32 @@ the path of the downloaded file in the Nix store is also printed. # Options - - `--type` *hashAlgo*\ - Use the specified cryptographic hash algorithm, - which can be one of `md5`, `sha1`, `sha256`, and `sha512`. - The default is `sha256`. +- `--type` *hashAlgo* - - `--print-path`\ - Print the store path of the downloaded file on standard output. + Use the specified cryptographic hash algorithm, + which can be one of `md5`, `sha1`, `sha256`, and `sha512`. + The default is `sha256`. - - `--unpack`\ - Unpack the archive (which must be a tarball or zip file) and add the - result to the Nix store. The resulting hash can be used with - functions such as Nixpkgs’s `fetchzip` or `fetchFromGitHub`. +- `--print-path` - - `--executable`\ - Set the executable bit on the downloaded file. + Print the store path of the downloaded file on standard output. - - `--name` *name*\ - Override the name of the file in the Nix store. By default, this is - `hash-basename`, where *basename* is the last component of *url*. - Overriding the name is necessary when *basename* contains characters - that are not allowed in Nix store paths. +- `--unpack` + + Unpack the archive (which must be a tarball or zip file) and add the + result to the Nix store. The resulting hash can be used with + functions such as Nixpkgs’s `fetchzip` or `fetchFromGitHub`. + +- `--executable` + + Set the executable bit on the downloaded file. + +- `--name` *name* + + Override the name of the file in the Nix store. By default, this is + `hash-basename`, where *basename* is the last component of *url*. + Overriding the name is necessary when *basename* contains characters + that are not allowed in Nix store paths. # Examples diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index 1eaf3c36a..69a711bd5 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -60,55 +60,63 @@ All options not listed here are passed to `nix-store --realise`, except for `--arg` and `--attr` / `-A` which are passed to `nix-instantiate`. - - `--command` *cmd*\ - In the environment of the derivation, run the shell command *cmd*. - This command is executed in an interactive shell. (Use `--run` to - use a non-interactive shell instead.) However, a call to `exit` is - implicitly added to the command, so the shell will exit after - running the command. To prevent this, add `return` at the end; - e.g. `--command "echo Hello; return"` will print `Hello` and then - drop you into the interactive shell. This can be useful for doing - any additional initialisation. +- `--command` *cmd* - - `--run` *cmd*\ - Like `--command`, but executes the command in a non-interactive - shell. This means (among other things) that if you hit Ctrl-C while - the command is running, the shell exits. + In the environment of the derivation, run the shell command *cmd*. + This command is executed in an interactive shell. (Use `--run` to + use a non-interactive shell instead.) However, a call to `exit` is + implicitly added to the command, so the shell will exit after + running the command. To prevent this, add `return` at the end; + e.g. `--command "echo Hello; return"` will print `Hello` and then + drop you into the interactive shell. This can be useful for doing + any additional initialisation. - - `--exclude` *regexp*\ - Do not build any dependencies whose store path matches the regular - expression *regexp*. This option may be specified multiple times. +- `--run` *cmd* - - `--pure`\ - If this flag is specified, the environment is almost entirely - cleared before the interactive shell is started, so you get an - environment that more closely corresponds to the “real†Nix build. A - few variables, in particular `HOME`, `USER` and `DISPLAY`, are - retained. + Like `--command`, but executes the command in a non-interactive + shell. This means (among other things) that if you hit Ctrl-C while + the command is running, the shell exits. - - `--packages` / `-p` *packages*…\ - Set up an environment in which the specified packages are present. - The command line arguments are interpreted as attribute names inside - the Nix Packages collection. Thus, `nix-shell --packages libjpeg openjdk` - will start a shell in which the packages denoted by the attribute - names `libjpeg` and `openjdk` are present. +- `--exclude` *regexp* - - `-i` *interpreter*\ - The chained script interpreter to be invoked by `nix-shell`. Only - applicable in `#!`-scripts (described below). + Do not build any dependencies whose store path matches the regular + expression *regexp*. This option may be specified multiple times. - - `--keep` *name*\ - When a `--pure` shell is started, keep the listed environment - variables. +- `--pure` + + If this flag is specified, the environment is almost entirely + cleared before the interactive shell is started, so you get an + environment that more closely corresponds to the “real†Nix build. A + few variables, in particular `HOME`, `USER` and `DISPLAY`, are + retained. + +- `--packages` / `-p` *packages*… + + Set up an environment in which the specified packages are present. + The command line arguments are interpreted as attribute names inside + the Nix Packages collection. Thus, `nix-shell --packages libjpeg openjdk` + will start a shell in which the packages denoted by the attribute + names `libjpeg` and `openjdk` are present. + +- `-i` *interpreter* + + The chained script interpreter to be invoked by `nix-shell`. Only + applicable in `#!`-scripts (described below). + +- `--keep` *name* + + When a `--pure` shell is started, keep the listed environment + variables. {{#include ./opt-common.md}} # Environment variables - - `NIX_BUILD_SHELL`\ - Shell used to start the interactive environment. Defaults to the - `bash` found in ``, falling back to the `bash` found in - `PATH` if not found. +- `NIX_BUILD_SHELL` + + Shell used to start the interactive environment. Defaults to the + `bash` found in ``, falling back to the `bash` found in + `PATH` if not found. {{#include ./env-common.md}} @@ -202,14 +210,14 @@ For example, here is a Python script that depends on Python and the ```python #! /usr/bin/env nix-shell -#! nix-shell -i python --packages python pythonPackages.prettytable +#! nix-shell -i python3 --packages python3 python3Packages.prettytable import prettytable # Print a simple table. t = prettytable.PrettyTable(["N", "N^2"]) for n in range(1, 10): t.add_row([n, n * n]) -print t +print(t) ``` Similarly, the following is a Perl script that specifies that it @@ -289,3 +297,8 @@ with import {}; runCommand "dummy" { buildInputs = [ python pythonPackages.prettytable ]; } "" ``` + +The script's file name is passed as the first argument to the interpreter specified by the `-i` flag. + +Aside from the very first line, which is a directive to the operating system, the additional `#! nix-shell` lines do not need to be at the beginning of the file. +This allows wrapping them in block comments for languages where `#` does not start a comment, such as ECMAScript, Erlang, PHP, or Ruby. diff --git a/doc/manual/src/command-ref/nix-store/add-fixed.md b/doc/manual/src/command-ref/nix-store/add-fixed.md index d25db091c..bebf15026 100644 --- a/doc/manual/src/command-ref/nix-store/add-fixed.md +++ b/doc/manual/src/command-ref/nix-store/add-fixed.md @@ -16,9 +16,10 @@ public url or broke since the download expression was written. This operation has the following options: - - `--recursive`\ - Use recursive instead of flat hashing mode, used when adding - directories to the store. +- `--recursive` + + Use recursive instead of flat hashing mode, used when adding + directories to the store. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/dump.md b/doc/manual/src/command-ref/nix-store/dump.md index c2f3c42ef..3de0e27b0 100644 --- a/doc/manual/src/command-ref/nix-store/dump.md +++ b/doc/manual/src/command-ref/nix-store/dump.md @@ -1,6 +1,6 @@ # Name -`nix-store --dump` - write a single path to a Nix Archive +`nix-store --dump` - write a single path to a [Nix Archive] ## Synopsis @@ -8,7 +8,7 @@ ## Description -The operation `--dump` produces a NAR (Nix ARchive) file containing the +The operation `--dump` produces a [Nix archive](@docroot@/glossary.md#gloss-nar) (NAR) file containing the contents of the file system tree rooted at *path*. The archive is written to standard output. @@ -30,8 +30,9 @@ NAR archives support filenames of unlimited length and 64-bit file sizes. They can contain regular files, directories, and symbolic links, but not other types of files (such as device nodes). -A Nix archive can be unpacked using `nix-store ---restore`. +A Nix archive can be unpacked using [`nix-store --restore`](@docroot@/command-ref/nix-store/restore.md). + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/export.md b/doc/manual/src/command-ref/nix-store/export.md index 1bc46f53b..ba772eb43 100644 --- a/doc/manual/src/command-ref/nix-store/export.md +++ b/doc/manual/src/command-ref/nix-store/export.md @@ -1,6 +1,6 @@ # Name -`nix-store --export` - export store paths to a Nix Archive +`nix-store --export` - export store paths to a [Nix Archive] ## Synopsis @@ -8,16 +8,22 @@ ## Description -The operation `--export` writes a serialisation of the specified store -paths to standard output in a format that can be imported into another -Nix store with `nix-store --import`. This is like `nix-store ---dump`, except that the NAR archive produced by that command doesn’t -contain the necessary meta-information to allow it to be imported into -another Nix store (namely, the set of references of the path). +The operation `--export` writes a serialisation of the given [store objects](@docroot@/glossary.md#gloss-store-object) to standard output in a format that can be imported into another [Nix store](@docroot@/store/index.md) with [`nix-store --import`](./import.md). -This command does not produce a *closure* of the specified paths, so if -a store path references other store paths that are missing in the target -Nix store, the import will fail. +> **Warning** +> +> This command *does not* produce a [closure](@docroot@/glossary.md#gloss-closure) of the specified store paths. +> Trying to import a store object that refers to store paths not available in the target Nix store will fail. +> +> Use [`nix-store --query`](@docroot@/command-ref/nix-store/query.md) to obtain the closure of a store path. + +This command is different from [`nix-store --dump`](./dump.md), which produces a [Nix archive](@docroot@/glossary.md#gloss-nar) that *does not* contain the set of [references](@docroot@/glossary.md#gloss-reference) of a given store path. + +> **Note** +> +> For efficient transfer of closures to remote machines over SSH, use [`nix-copy-closure`](@docroot@/command-ref/nix-copy-closure.md). + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive {{#include ./opt-common.md}} @@ -27,15 +33,21 @@ Nix store, the import will fail. # Examples -To copy a whole closure, do something -like: - -```console -$ nix-store --export $(nix-store --query --requisites paths) > out -``` - -To import the whole closure again, run: - -```console -$ nix-store --import < out -``` +> **Example** +> +> Deploy GNU Hello to an airgapped machine via USB stick. +> +> Write the closure to the block device on a machine with internet connection: +> +> ```shell-session +> [alice@itchy]$ storePath=$(nix-build '' -I nixpkgs=channel:nixpkgs-unstable -A hello --no-out-link) +> [alice@itchy]$ nix-store --export $(nix-store --query --requisites $storePath) | sudo dd of=/dev/usb +> ``` +> +> Read the closure from the block device on the machine without internet connection: +> +> ```shell-session +> [bob@scratchy]$ hello=$(sudo dd if=/dev/usb | nix-store --import | tail -1) +> [bob@scratchy]$ $hello/bin/hello +> Hello, world! +> ``` diff --git a/doc/manual/src/command-ref/nix-store/gc.md b/doc/manual/src/command-ref/nix-store/gc.md index 7be0d559a..f432e00eb 100644 --- a/doc/manual/src/command-ref/nix-store/gc.md +++ b/doc/manual/src/command-ref/nix-store/gc.md @@ -14,30 +14,34 @@ reachable via file system references from a set of “rootsâ€, are deleted. The following suboperations may be specified: - - `--print-roots`\ - This operation prints on standard output the set of roots used by - the garbage collector. +- `--print-roots` - - `--print-live`\ - This operation prints on standard output the set of “live†store - paths, which are all the store paths reachable from the roots. Live - paths should never be deleted, since that would break consistency — - it would become possible that applications are installed that - reference things that are no longer present in the store. + This operation prints on standard output the set of roots used by + the garbage collector. - - `--print-dead`\ - This operation prints out on standard output the set of “dead†store - paths, which is just the opposite of the set of live paths: any path - in the store that is not live (with respect to the roots) is dead. +- `--print-live` + + This operation prints on standard output the set of “live†store + paths, which are all the store paths reachable from the roots. Live + paths should never be deleted, since that would break consistency — + it would become possible that applications are installed that + reference things that are no longer present in the store. + +- `--print-dead` + + This operation prints out on standard output the set of “dead†store + paths, which is just the opposite of the set of live paths: any path + in the store that is not live (with respect to the roots) is dead. By default, all unreachable paths are deleted. The following options control what gets deleted and in what order: - - `--max-freed` *bytes*\ - Keep deleting paths until at least *bytes* bytes have been deleted, - then stop. The argument *bytes* can be followed by the - multiplicative suffix `K`, `M`, `G` or `T`, denoting KiB, MiB, GiB - or TiB units. +- `--max-freed` *bytes* + + Keep deleting paths until at least *bytes* bytes have been deleted, + then stop. The argument *bytes* can be followed by the + multiplicative suffix `K`, `M`, `G` or `T`, denoting KiB, MiB, GiB + or TiB units. The behaviour of the collector is also influenced by the `keep-outputs` and `keep-derivations` settings in the Nix diff --git a/doc/manual/src/command-ref/nix-store/import.md b/doc/manual/src/command-ref/nix-store/import.md index 2711316a7..3f6b3d076 100644 --- a/doc/manual/src/command-ref/nix-store/import.md +++ b/doc/manual/src/command-ref/nix-store/import.md @@ -1,6 +1,8 @@ # Name -`nix-store --import` - import Nix Archive into the store +`nix-store --import` - import [Nix Archive] into the store + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive # Synopsis @@ -8,14 +10,34 @@ # Description -The operation `--import` reads a serialisation of a set of store paths -produced by `nix-store --export` from standard input and adds those -store paths to the Nix store. Paths that already exist in the Nix store -are ignored. If a path refers to another path that doesn’t exist in the -Nix store, the import fails. +The operation `--import` reads a serialisation of a set of [store objects](@docroot@/glossary.md#gloss-store-object) produced by [`nix-store --export`](./export.md) from standard input, and adds those store objects to the specified [Nix store](@docroot@/store/index.md). +Paths that already exist in the target Nix store are ignored. +If a path [refers](@docroot@/glossary.md#gloss-reference) to another path that doesn’t exist in the target Nix store, the import fails. + +> **Note** +> +> For efficient transfer of closures to remote machines over SSH, use [`nix-copy-closure`](@docroot@/command-ref/nix-copy-closure.md). {{#include ./opt-common.md}} {{#include ../opt-common.md}} {{#include ../env-common.md}} + +# Examples + +> **Example** +> +> Given a closure of GNU Hello as a file: +> +> ```shell-session +> $ storePath="$(nix-build '' -I nixpkgs=channel:nixpkgs-unstable -A hello --no-out-link)" +> $ nix-store --export $(nix-store --query --requisites $storePath) > hello.closure +> ``` +> +> Import the closure into a [remote SSH store](@docroot@/store/types/ssh-store.md) using the [`--store`](@docroot@/command-ref/conf-file.md#conf-store) option: +> +> ```console +> $ nix-store --import --store ssh://alice@itchy.example.org < hello.closure +> ``` + diff --git a/doc/manual/src/command-ref/nix-store/optimise.md b/doc/manual/src/command-ref/nix-store/optimise.md index dc392aeb8..b257466b2 100644 --- a/doc/manual/src/command-ref/nix-store/optimise.md +++ b/doc/manual/src/command-ref/nix-store/optimise.md @@ -12,7 +12,7 @@ The operation `--optimise` reduces Nix store disk space usage by finding identical files in the store and hard-linking them to each other. It typically reduces the size of the store by something like 25-35%. Only regular files and symlinks are hard-linked in this manner. Files are -considered identical when they have the same NAR archive serialisation: +considered identical when they have the same [Nix Archive (NAR)][Nix Archive] serialisation: that is, regular files must have the same contents and permission (executable or non-executable), and symlinks must have the same contents. @@ -38,3 +38,4 @@ hashing files in `/nix/store/qhqx7l2f1kmwihc9bnxs7rc159hsxnf3-gcc-4.1.1' there are 114486 files with equal contents out of 215894 files in total ``` +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive diff --git a/doc/manual/src/command-ref/nix-store/query.md b/doc/manual/src/command-ref/nix-store/query.md index a158c76aa..b4efa734e 100644 --- a/doc/manual/src/command-ref/nix-store/query.md +++ b/doc/manual/src/command-ref/nix-store/query.md @@ -24,122 +24,138 @@ symlink. # Common query options - - `--use-output`; `-u`\ - For each argument to the query that is a [store derivation], apply the - query to the output path of the derivation instead. +- `--use-output` / `-u` - - `--force-realise`; `-f`\ - Realise each argument to the query first (see [`nix-store --realise`](./realise.md)). + For each argument to the query that is a [store derivation], apply the + query to the output path of the derivation instead. + +- `--force-realise` / `-f` + + Realise each argument to the query first (see [`nix-store --realise`](./realise.md)). [store derivation]: @docroot@/glossary.md#gloss-store-derivation # Queries - - `--outputs`\ - Prints out the [output paths] of the store - derivations *paths*. These are the paths that will be produced when - the derivation is built. +- `--outputs` - [output paths]: ../../glossary.md#gloss-output-path + Prints out the [output paths] of the store + derivations *paths*. These are the paths that will be produced when + the derivation is built. - - `--requisites`; `-R`\ - Prints out the [closure] of the store path *paths*. + [output paths]: @docroot@/glossary.md#gloss-output-path - [closure]: ../../glossary.md#gloss-closure +- `--requisites` / `-R` - This query has one option: + Prints out the [closure] of the store path *paths*. - - `--include-outputs` - Also include the existing output paths of [store derivation]s, - and their closures. + [closure]: @docroot@/glossary.md#gloss-closure - This query can be used to implement various kinds of deployment. A - *source deployment* is obtained by distributing the closure of a - store derivation. A *binary deployment* is obtained by distributing - the closure of an output path. A *cache deployment* (combined - source/binary deployment, including binaries of build-time-only - dependencies) is obtained by distributing the closure of a store - derivation and specifying the option `--include-outputs`. + This query has one option: - - `--references`\ - Prints the set of [references] of the store paths - *paths*, that is, their immediate dependencies. (For *all* - dependencies, use `--requisites`.) + - `--include-outputs` + Also include the existing output paths of [store derivation]s, + and their closures. - [references]: ../../glossary.md#gloss-reference + This query can be used to implement various kinds of deployment. A + *source deployment* is obtained by distributing the closure of a + store derivation. A *binary deployment* is obtained by distributing + the closure of an output path. A *cache deployment* (combined + source/binary deployment, including binaries of build-time-only + dependencies) is obtained by distributing the closure of a store + derivation and specifying the option `--include-outputs`. - - `--referrers`\ - Prints the set of *referrers* of the store paths *paths*, that is, - the store paths currently existing in the Nix store that refer to - one of *paths*. Note that contrary to the references, the set of - referrers is not constant; it can change as store paths are added or - removed. +- `--references` - - `--referrers-closure`\ - Prints the closure of the set of store paths *paths* under the - referrers relation; that is, all store paths that directly or - indirectly refer to one of *paths*. These are all the path currently - in the Nix store that are dependent on *paths*. + Prints the set of [references] of the store paths + *paths*, that is, their immediate dependencies. (For *all* + dependencies, use `--requisites`.) - - `--deriver`; `-d`\ - Prints the [deriver] that was used to build the store paths *paths*. If - the path has no deriver (e.g., if it is a source file), or if the - deriver is not known (e.g., in the case of a binary-only - deployment), the string `unknown-deriver` is printed. - The returned deriver is not guaranteed to exist in the local store, for - example when *paths* were substituted from a binary cache. - Use `--valid-derivers` instead to obtain valid paths only. + [references]: @docroot@/glossary.md#gloss-reference - [deriver]: ../../glossary.md#gloss-deriver +- `--referrers` - - `--valid-derivers`\ - Prints a set of derivation files (`.drv`) which are supposed produce - said paths when realized. Might print nothing, for example for source paths - or paths subsituted from a binary cache. + Prints the set of *referrers* of the store paths *paths*, that is, + the store paths currently existing in the Nix store that refer to + one of *paths*. Note that contrary to the references, the set of + referrers is not constant; it can change as store paths are added or + removed. - - `--graph`\ - Prints the references graph of the store paths *paths* in the format - of the `dot` tool of AT\&T's [Graphviz - package](http://www.graphviz.org/). This can be used to visualise - dependency graphs. To obtain a build-time dependency graph, apply - this to a store derivation. To obtain a runtime dependency graph, - apply it to an output path. +- `--referrers-closure` - - `--tree`\ - Prints the references graph of the store paths *paths* as a nested - ASCII tree. References are ordered by descending closure size; this - tends to flatten the tree, making it more readable. The query only - recurses into a store path when it is first encountered; this - prevents a blowup of the tree representation of the graph. + Prints the closure of the set of store paths *paths* under the + referrers relation; that is, all store paths that directly or + indirectly refer to one of *paths*. These are all the path currently + in the Nix store that are dependent on *paths*. - - `--graphml`\ - Prints the references graph of the store paths *paths* in the - [GraphML](http://graphml.graphdrawing.org/) file format. This can be - used to visualise dependency graphs. To obtain a build-time - dependency graph, apply this to a [store derivation]. To obtain a - runtime dependency graph, apply it to an output path. +- `--deriver` / `-d` - - `--binding` *name*; `-b` *name*\ - Prints the value of the attribute *name* (i.e., environment - variable) of the [store derivation]s *paths*. It is an error for a - derivation to not have the specified attribute. + Prints the [deriver] that was used to build the store paths *paths*. If + the path has no deriver (e.g., if it is a source file), or if the + deriver is not known (e.g., in the case of a binary-only + deployment), the string `unknown-deriver` is printed. + The returned deriver is not guaranteed to exist in the local store, for + example when *paths* were substituted from a binary cache. + Use `--valid-derivers` instead to obtain valid paths only. - - `--hash`\ - Prints the SHA-256 hash of the contents of the store paths *paths* - (that is, the hash of the output of `nix-store --dump` on the given - paths). Since the hash is stored in the Nix database, this is a fast - operation. + [deriver]: @docroot@/glossary.md#gloss-deriver - - `--size`\ - Prints the size in bytes of the contents of the store paths *paths* - — to be precise, the size of the output of `nix-store --dump` on - the given paths. Note that the actual disk space required by the - store paths may be higher, especially on filesystems with large - cluster sizes. +- `--valid-derivers` - - `--roots`\ - Prints the garbage collector roots that point, directly or - indirectly, at the store paths *paths*. + Prints a set of derivation files (`.drv`) which are supposed produce + said paths when realized. Might print nothing, for example for source paths + or paths subsituted from a binary cache. + +- `--graph` + + Prints the references graph of the store paths *paths* in the format + of the `dot` tool of AT\&T's [Graphviz + package](http://www.graphviz.org/). This can be used to visualise + dependency graphs. To obtain a build-time dependency graph, apply + this to a store derivation. To obtain a runtime dependency graph, + apply it to an output path. + +- `--tree` + + Prints the references graph of the store paths *paths* as a nested + ASCII tree. References are ordered by descending closure size; this + tends to flatten the tree, making it more readable. The query only + recurses into a store path when it is first encountered; this + prevents a blowup of the tree representation of the graph. + +- `--graphml` + + Prints the references graph of the store paths *paths* in the + [GraphML](http://graphml.graphdrawing.org/) file format. This can be + used to visualise dependency graphs. To obtain a build-time + dependency graph, apply this to a [store derivation]. To obtain a + runtime dependency graph, apply it to an output path. + +- `--binding` *name* / `-b` *name* + + Prints the value of the attribute *name* (i.e., environment + variable) of the [store derivation]s *paths*. It is an error for a + derivation to not have the specified attribute. + +- `--hash` + + Prints the SHA-256 hash of the contents of the store paths *paths* + (that is, the hash of the output of `nix-store --dump` on the given + paths). Since the hash is stored in the Nix database, this is a fast + operation. + +- `--size` + + Prints the size in bytes of the contents of the store paths *paths* + — to be precise, the size of the output of `nix-store --dump` on + the given paths. Note that the actual disk space required by the + store paths may be higher, especially on filesystems with large + cluster sizes. + +- `--roots` + + Prints the garbage collector roots that point, directly or + indirectly, at the store paths *paths*. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index 5428d57fa..a899758df 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -25,14 +25,14 @@ Each of *paths* is processed as follows: If no substitutes are available and no store derivation is given, realisation fails. -[store paths]: @docroot@/glossary.md#gloss-store-path +[store paths]: @docroot@/store/store-path.md [valid]: @docroot@/glossary.md#gloss-validity [store derivation]: @docroot@/glossary.md#gloss-store-derivation [output paths]: @docroot@/glossary.md#gloss-output-path -[store objects]: @docroot@/glossary.md#gloss-store-object +[store objects]: @docroot@/store/store-object.md [closure]: @docroot@/glossary.md#gloss-closure [substituters]: @docroot@/command-ref/conf-file.md#conf-substituters -[content-addressed derivations]: @docroot@/contributing/experimental-features.md#xp-feature-ca-derivations +[content-addressed derivations]: @docroot@/development/experimental-features.md#xp-feature-ca-derivations [Nix database]: @docroot@/glossary.md#gloss-nix-database The resulting paths are printed on standard output. @@ -42,23 +42,26 @@ For non-derivation arguments, the argument itself is printed. # Options - - `--dry-run`\ - Print on standard error a description of what packages would be - built or downloaded, without actually performing the operation. +- `--dry-run` - - `--ignore-unknown`\ - If a non-derivation path does not have a substitute, then silently - ignore it. + Print on standard error a description of what packages would be + built or downloaded, without actually performing the operation. - - `--check`\ - This option allows you to check whether a derivation is - deterministic. It rebuilds the specified derivation and checks - whether the result is bitwise-identical with the existing outputs, - printing an error if that’s not the case. The outputs of the - specified derivation must already exist. When used with `-K`, if an - output path is not identical to the corresponding output from the - previous build, the new output path is left in - `/nix/store/name.check.` +- `--ignore-unknown` + + If a non-derivation path does not have a substitute, then silently + ignore it. + +- `--check` + + This option allows you to check whether a derivation is + deterministic. It rebuilds the specified derivation and checks + whether the result is bitwise-identical with the existing outputs, + printing an error if that’s not the case. The outputs of the + specified derivation must already exist. When used with `-K`, if an + output path is not identical to the corresponding output from the + previous build, the new output path is left in + `/nix/store/name.check.` {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/restore.md b/doc/manual/src/command-ref/nix-store/restore.md index fcba43df4..2d0aa3127 100644 --- a/doc/manual/src/command-ref/nix-store/restore.md +++ b/doc/manual/src/command-ref/nix-store/restore.md @@ -8,9 +8,11 @@ ## Description -The operation `--restore` unpacks a NAR archive to *path*, which must +The operation `--restore` unpacks a [Nix Archive (NAR)][Nix Archive] to *path*, which must not already exist. The archive is read from standard input. +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive + {{#include ./opt-common.md}} {{#include ../opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/serve.md b/doc/manual/src/command-ref/nix-store/serve.md index 0f90f65ae..9a4cf5216 100644 --- a/doc/manual/src/command-ref/nix-store/serve.md +++ b/doc/manual/src/command-ref/nix-store/serve.md @@ -14,10 +14,11 @@ access to a restricted ssh user. The following flags are available: - - `--write`\ - Allow the connected client to request the realization of - derivations. In effect, this can be used to make the host act as a - remote builder. +- `--write` + + Allow the connected client to request the realization of + derivations. In effect, this can be used to make the host act as a + remote builder. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/verify.md b/doc/manual/src/command-ref/nix-store/verify.md index 2695b3361..40c9180db 100644 --- a/doc/manual/src/command-ref/nix-store/verify.md +++ b/doc/manual/src/command-ref/nix-store/verify.md @@ -16,18 +16,20 @@ being modified by non-Nix tools, or of bugs in Nix itself. This operation has the following options: - - `--check-contents`\ - Checks that the contents of every valid store path has not been - altered by computing a SHA-256 hash of the contents and comparing it - with the hash stored in the Nix database at build time. Paths that - have been modified are printed out. For large stores, - `--check-contents` is obviously quite slow. +- `--check-contents` - - `--repair`\ - If any valid path is missing from the store, or (if - `--check-contents` is given) the contents of a valid path has been - modified, then try to repair the path by redownloading it. See - `nix-store --repair-path` for details. + Checks that the contents of every valid store path has not been + altered by computing a SHA-256 hash of the contents and comparing it + with the hash stored in the Nix database at build time. Paths that + have been modified are printed out. For large stores, + `--check-contents` is obviously quite slow. + +- `--repair` + + If any valid path is missing from the store, or (if + `--check-contents` is given) the contents of a valid path has been + modified, then try to repair the path by redownloading it. See + `nix-store --repair-path` for details. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index 114b292f9..69a700207 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -37,7 +37,7 @@ Most Nix commands accept the following command-line options: Print even more informational messages. - `4` “Debug†- + Print debug information. - `5` “Vomit†@@ -143,7 +143,7 @@ Most Nix commands accept the following command-line options: This option is accepted by `nix-env`, `nix-instantiate`, `nix-shell` and `nix-build`. When evaluating Nix expressions, the expression evaluator will automatically try to call functions that it encounters. - It can automatically call functions for which every argument has a [default value](@docroot@/language/constructs.md#functions) (e.g., `{ argName ? defaultValue }: ...`). + It can automatically call functions for which every argument has a [default value](@docroot@/language/syntax.md#functions) (e.g., `{ argName ? defaultValue }: ...`). With `--arg`, you can also call functions that have arguments without a default value (or override a default value). That is, if the evaluator encounters a function with an argument named *name*, it will call it with value *value*. @@ -187,11 +187,12 @@ Most Nix commands accept the following command-line options: For `nix-shell`, this option is commonly used to give you a shell in which you can build the packages returned by the expression. If you want to get a shell which contain the *built* packages ready for use, give your expression to the `nix-shell --packages ` convenience flag instead. -- [`-I`](#opt-I) *path* +- [`-I` / `--include`](#opt-I) *path* - Add an entry to the [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path). + Add an entry to the list of search paths used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). This option may be given multiple times. - Paths added through `-I` take precedence over [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH). + + Paths added through `-I` take precedence over the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path) and the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH). - [`--option`](#opt-option) *name* *value* diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/development/building.md similarity index 67% rename from doc/manual/src/contributing/hacking.md rename to doc/manual/src/development/building.md index 916ec3077..5a5fb3368 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/development/building.md @@ -1,24 +1,67 @@ -# Hacking +# Building Nix -This section provides some notes on how to hack on Nix. To get the -latest version of Nix from GitHub: +This section provides some notes on how to start hacking on Nix. +To get the latest version of Nix from GitHub: ```console $ git clone https://github.com/NixOS/nix.git $ cd nix ``` -The following instructions assume you already have some version of Nix installed locally, so that you can use it to set up the development environment. If you don't have it installed, follow the [installation instructions]. +> **Note** +> +> The following instructions assume you already have some version of Nix installed locally, so that you can use it to set up the development environment. +> If you don't have it installed, follow the [installation instructions](../installation/index.md). -[installation instructions]: ../installation/index.md + +To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: + +```console +$ nix-shell +``` + +To get a shell with one of the other [supported compilation environments](#compilation-environments): + +```console +$ nix-shell --attr devShells.x86_64-linux.native-clangStdenvPackages +``` + +> **Note** +> +> You can use `native-ccacheStdenvPackages` to drastically improve rebuild time. +> By default, [ccache](https://ccache.dev) keeps artifacts in `~/.cache/ccache/`. + +To build Nix itself in this shell: + +```console +[nix-shell]$ autoreconfPhase +[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out +[nix-shell]$ make -j $NIX_BUILD_CORES +``` + +To install it in `$(pwd)/outputs` and test it: + +```console +[nix-shell]$ make install +[nix-shell]$ make installcheck -j $NIX_BUILD_CORES +[nix-shell]$ ./outputs/out/bin/nix --version +nix (Nix) 2.12 +``` + +To build a release version of Nix for the current operating system and CPU architecture: + +```console +$ nix-build +``` + +You can also build Nix for one of the [supported platforms](#platforms). ## Building Nix with flakes This section assumes you are using Nix with the [`flakes`] and [`nix-command`] experimental features enabled. -See the [Building Nix](#building-nix) section for equivalent instructions using stable Nix interfaces. -[`flakes`]: @docroot@/contributing/experimental-features.md#xp-feature-flakes -[`nix-command`]: @docroot@/contributing/experimental-features.md#xp-nix-command +[`flakes`]: @docroot@/development/experimental-features.md#xp-feature-flakes +[`nix-command`]: @docroot@/development/experimental-features.md#xp-nix-command To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: @@ -67,50 +110,6 @@ $ nix build You can also build Nix for one of the [supported platforms](#platforms). -## Building Nix - -To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: - -```console -$ nix-shell -``` - -To get a shell with one of the other [supported compilation environments](#compilation-environments): - -```console -$ nix-shell --attr devShells.x86_64-linux.native-clangStdenvPackages -``` - -> **Note** -> -> You can use `native-ccacheStdenvPackages` to drastically improve rebuild time. -> By default, [ccache](https://ccache.dev) keeps artifacts in `~/.cache/ccache/`. - -To build Nix itself in this shell: - -```console -[nix-shell]$ autoreconfPhase -[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out -[nix-shell]$ make -j $NIX_BUILD_CORES -``` - -To install it in `$(pwd)/outputs` and test it: - -```console -[nix-shell]$ make install -[nix-shell]$ make installcheck -j $NIX_BUILD_CORES -[nix-shell]$ ./outputs/out/bin/nix --version -nix (Nix) 2.12 -``` - -To build a release version of Nix for the current operating system and CPU architecture: - -```console -$ nix-build -``` - -You can also build Nix for one of the [supported platforms](#platforms). - ## Makefile variables You may need `profiledir=$out/etc/profile.d` and `sysconfdir=$out/etc` to run `make install`. @@ -122,7 +121,6 @@ Run `make` with [`-e` / `--environment-overrides`](https://www.gnu.org/software/ The docs can take a while to build, so you may want to disable this for local development. - `ENABLE_FUNCTIONAL_TESTS=yes` to enable building the functional tests. -- `ENABLE_UNIT_TESTS=yes` to enable building the unit tests. - `OPTIMIZE=1` to enable optimizations. - `libraries=libutil programs=` to only build a specific library. @@ -144,6 +142,7 @@ Nix can be built for various platforms, as specified in [`flake.nix`]: - `aarch64-darwin` - `armv6l-linux` - `armv7l-linux` +- `riscv64-linux` In order to build Nix for a different platform than the one you're currently on, you need a way for your current Nix installation to build code for that @@ -166,7 +165,10 @@ or for Nix with the [`flakes`] and [`nix-command`] experimental features enabled $ nix build .#packages.aarch64-linux.default ``` -Cross-compiled builds are available for ARMv6 (`armv6l-linux`) and ARMv7 (`armv7l-linux`). +Cross-compiled builds are available for: +- `armv6l-linux` +- `armv7l-linux` +- `riscv64-linux` Add more [system types](#system-type) to `crossSystems` in `flake.nix` to bootstrap Nix on unsupported platforms. ### Building for multiple platforms at once @@ -196,7 +198,7 @@ In order to facilitate this, Nix has some support for being built out of tree ## System type -Nix uses a string with he following format to identify the *system type* or *platform* it runs on: +Nix uses a string with the following format to identify the *system type* or *platform* it runs on: ``` -[-] @@ -258,10 +260,10 @@ See [supported compilation environments](#compilation-environments) and instruct To use the LSP with your editor, you first need to [set up `clangd`](https://clangd.llvm.org/installation#project-setup) by running: ```console -make clean && bear -- make -j$NIX_BUILD_CORES default check install +make compile_commands.json ``` -Configure your editor to use the `clangd` from the shell, either by running it inside the development shell, or by using [nix-direnv](https://github.com/nix-community/nix-direnv) and [the appropriate editor plugin](https://github.com/direnv/direnv/wiki#editor-integration). +Configure your editor to use the `clangd` from the `.#native-clangStdenvPackages` shell. You can do that either by running it inside the development shell, or by using [nix-direnv](https://github.com/nix-community/nix-direnv) and [the appropriate editor plugin](https://github.com/direnv/direnv/wiki#editor-integration). > **Note** > @@ -269,80 +271,25 @@ Configure your editor to use the `clangd` from the shell, either by running it i > Some other editors (e.g. Emacs, Vim) need a plugin to support LSP servers in general (e.g. [lsp-mode](https://github.com/emacs-lsp/lsp-mode) for Emacs and [vim-lsp](https://github.com/prabirshrestha/vim-lsp) for vim). > Editor-specific setup is typically opinionated, so we will not cover it here in more detail. -## Add a release note +## Formatting and pre-commit hooks -`doc/manual/rl-next` contains release notes entries for all unreleased changes. +You may run the formatters as a one-off using: -User-visible changes should come with a release note. - -### Add an entry - -Here's what a complete entry looks like. The file name is not incorporated in the document. - -``` ---- -synopsis: Basically a title -issues: 1234 -prs: 1238 ---- - -Here's one or more paragraphs that describe the change. - -- It's markdown -- Add references to the manual using @docroot@ +```console +make format ``` -Significant changes should add the following header, which moves them to the top. +If you'd like to run the formatters before every commit, install the hooks: ``` -significance: significant +pre-commit-hooks-install ``` - -See also the [format documentation](https://github.com/haskell/cabal/blob/master/CONTRIBUTING.md#changelog). +This installs [pre-commit](https://pre-commit.com) using [cachix/git-hooks.nix](https://github.com/cachix/git-hooks.nix). -### Build process +When making a commit, pay attention to the console output. +If it fails, run `git add --patch` to approve the suggestions _and commit again_. -Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`. - -## Branches - -- [`master`](https://github.com/NixOS/nix/commits/master) - - The main development branch. All changes are approved and merged here. - When developing a change, create a branch based on the latest `master`. - - Maintainers try to [keep it in a release-worthy state](#reverting). - -- [`maintenance-*.*`](https://github.com/NixOS/nix/branches/all?query=maintenance) - - These branches are the subject of backports only, and are - also [kept](#reverting) in a release-worthy state. - - See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) - -- [`latest-release`](https://github.com/NixOS/nix/tree/latest-release) - - The latest patch release of the latest minor version. - - See [`maintainers/release-process.md`](https://github.com/NixOS/nix/blob/master/maintainers/release-process.md) - -- [`backport-*-to-*`](https://github.com/NixOS/nix/branches/all?query=backport) - - Generally branches created by the backport action. - - See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) - -- [_other_](https://github.com/NixOS/nix/branches/all) - - Branches that do not conform to the above patterns should be feature branches. - -## Reverting - -If a change turns out to be merged by mistake, or contain a regression, it may be reverted. -A revert is not a rejection of the contribution, but merely part of an effective development process. -It makes sure that development keeps running smoothly, with minimal uncertainty, and less overhead. -If maintainers have to worry too much about avoiding reverts, they would not be able to merge as much. -By embracing reverts as a good part of the development process, everyone wins. - -However, taking a step back may be frustrating, so maintainers will be extra supportive on the next try. +To refresh pre-commit hook's config file, do the following: +1. Exit the development shell and start it again by running `nix develop`. +2. If you also use the pre-commit hook, also run `pre-commit-hooks-install` again. diff --git a/doc/manual/src/contributing/cli-guideline.md b/doc/manual/src/development/cli-guideline.md similarity index 88% rename from doc/manual/src/contributing/cli-guideline.md rename to doc/manual/src/development/cli-guideline.md index e90d6de8d..23df844ec 100644 --- a/doc/manual/src/contributing/cli-guideline.md +++ b/doc/manual/src/development/cli-guideline.md @@ -389,88 +389,6 @@ colors, no emojis and using ASCII instead of Unicode symbols). The same should happen when TTY is not detected on STDERR. We should not display progress / status section, but only print warnings and errors. -## Returning future proof JSON - -The schema of JSON output should allow for backwards compatible extension. This section explains how to achieve this. - -Two definitions are helpful here, because while JSON only defines one "key-value" -object type, we use it to cover two use cases: - - - **dictionary**: a map from names to value that all have the same type. In - C++ this would be a `std::map` with string keys. - - **record**: a fixed set of attributes each with their own type. In C++, this - would be represented by a `struct`. - -It is best not to mix these use cases, as that may lead to incompatibilities when the schema changes. For example, adding a record field to a dictionary breaks consumers that assume all JSON object fields to have the same meaning and type. - -This leads to the following guidelines: - - - The top-level (root) value must be a record. - - Otherwise, one can not change the structure of a command's output. - - - The value of a dictionary item must be a record. - - Otherwise, the item type can not be extended. - - - List items should be records. - - Otherwise, one can not change the structure of the list items. - - If the order of the items does not matter, and each item has a unique key that is a string, consider representing the list as a dictionary instead. If the order of the items needs to be preserved, return a list of records. - - - Streaming JSON should return records. - - An example of a streaming JSON format is [JSON lines](https://jsonlines.org/), where each line represents a JSON value. These JSON values can be considered top-level values or list items, and they must be records. - -### Examples - - -This is bad, because all keys must be assumed to be store types: - -```json -{ - "local": { ... }, - "remote": { ... }, - "http": { ... } -} -``` - -This is good, because the it is extensible at the root, and is somewhat self-documenting: - -```json -{ - "storeTypes": { "local": { ... }, ... }, - "pluginSupport": true -} -``` - -While the dictionary of store types seems like a very complete response at first, a use case may arise that warrants returning additional information. -For example, the presence of plugin support may be crucial information for a client to proceed when their desired store type is missing. - - - -The following representation is bad because it is not extensible: - -```json -{ "outputs": [ "out" "bin" ] } -``` - -However, simply converting everything to records is not enough, because the order of outputs must be preserved: - -```json -{ "outputs": { "bin": {}, "out": {} } } -``` - -The first item is the default output. Deriving this information from the outputs ordering is not great, but this is how Nix currently happens to work. -While it is possible for a JSON parser to preserve the order of fields, we can not rely on this capability to be present in all JSON libraries. - -This representation is extensible and preserves the ordering: - -```json -{ "outputs": [ { "outputName": "out" }, { "outputName": "bin" } ] } -``` - ## Dialog with the user CLIs don't always make it clear when an action has taken place. For every diff --git a/doc/manual/src/development/contributing.md b/doc/manual/src/development/contributing.md new file mode 100644 index 000000000..7de7489dc --- /dev/null +++ b/doc/manual/src/development/contributing.md @@ -0,0 +1,79 @@ +# Contributing + +## Add a release note + +`doc/manual/rl-next` contains release notes entries for all unreleased changes. + +User-visible changes should come with a release note. + +### Add an entry + +Here's what a complete entry looks like. The file name is not incorporated in the document. + +``` +--- +synopsis: Basically a title +issues: 1234 +prs: 1238 +--- + +Here's one or more paragraphs that describe the change. + +- It's markdown +- Add references to the manual using @docroot@ +``` + +Significant changes should add the following header, which moves them to the top. + +``` +significance: significant +``` + + +See also the [format documentation](https://github.com/haskell/cabal/blob/master/CONTRIBUTING.md#changelog). + +### Build process + +Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`. + +## Branches + +- [`master`](https://github.com/NixOS/nix/commits/master) + + The main development branch. All changes are approved and merged here. + When developing a change, create a branch based on the latest `master`. + + Maintainers try to [keep it in a release-worthy state](#reverting). + +- [`maintenance-*.*`](https://github.com/NixOS/nix/branches/all?query=maintenance) + + These branches are the subject of backports only, and are + also [kept](#reverting) in a release-worthy state. + + See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) + +- [`latest-release`](https://github.com/NixOS/nix/tree/latest-release) + + The latest patch release of the latest minor version. + + See [`maintainers/release-process.md`](https://github.com/NixOS/nix/blob/master/maintainers/release-process.md) + +- [`backport-*-to-*`](https://github.com/NixOS/nix/branches/all?query=backport) + + Generally branches created by the backport action. + + See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) + +- [_other_](https://github.com/NixOS/nix/branches/all) + + Branches that do not conform to the above patterns should be feature branches. + +## Reverting + +If a change turns out to be merged by mistake, or contain a regression, it may be reverted. +A revert is not a rejection of the contribution, but merely part of an effective development process. +It makes sure that development keeps running smoothly, with minimal uncertainty, and less overhead. +If maintainers have to worry too much about avoiding reverts, they would not be able to merge as much. +By embracing reverts as a good part of the development process, everyone wins. + +However, taking a step back may be frustrating, so maintainers will be extra supportive on the next try. diff --git a/doc/manual/src/contributing/cxx.md b/doc/manual/src/development/cxx.md similarity index 100% rename from doc/manual/src/contributing/cxx.md rename to doc/manual/src/development/cxx.md diff --git a/doc/manual/src/contributing/documentation.md b/doc/manual/src/development/documentation.md similarity index 89% rename from doc/manual/src/contributing/documentation.md rename to doc/manual/src/development/documentation.md index 1dddb207c..63f574ab7 100644 --- a/doc/manual/src/contributing/documentation.md +++ b/doc/manual/src/development/documentation.md @@ -24,14 +24,12 @@ nix build .#^doc and open `./result-doc/share/doc/nix/manual/index.html`. -To build the manual incrementally, [enter the development shell](./hacking.md) and run: +To build the manual incrementally, [enter the development shell](./building.md) and run: ```console -make manual-html -j $NIX_BUILD_CORES +make manual-html-open -j $NIX_BUILD_CORES ``` -and open `./outputs/out/share/doc/nix/manual/language/index.html`. - In order to reflect changes to the [Makefile for the manual], clear all generated files before re-building: [Makefile for the manual]: https://github.com/NixOS/nix/blob/master/doc/manual/local.mk @@ -149,7 +147,7 @@ Please observe these guidelines to ease reviews: ``` A [store object] contains a [file system object] and [references] to other store objects. - [store object]: @docroot@/glossary.md#gloss-store-object + [store object]: @docroot@/store/store-object.md [file system object]: @docroot@/architecture/file-system-object.md [references]: @docroot@/glossary.md#gloss-reference ``` @@ -198,13 +196,35 @@ You can also build and view it yourself: [Doxygen API documentation]: https://hydra.nixos.org/job/nix/master/internal-api-docs/latest/download-by-type/doc/internal-api-docs ```console -# nix build .#hydraJobs.internal-api-docs -# xdg-open ./result/share/doc/nix/internal-api/html/index.html +$ nix build .#hydraJobs.internal-api-docs +$ xdg-open ./result/share/doc/nix/internal-api/html/index.html +``` + +or inside `nix-shell` or `nix develop`: + +```console +$ mesonConfigurePhase +$ ninja src/internal-api-docs/html +$ xdg-open src/internal-api-docs/html/index.html +``` + +## C API documentation + +Note that the C API is not yet stable. +[C API documentation] is available online. +You can also build and view it yourself: + +[C API documentation]: https://hydra.nixos.org/job/nix/master/external-api-docs/latest/download-by-type/doc/external-api-docs + +```console +$ nix build .#hydraJobs.external-api-docs +$ xdg-open ./result/share/doc/nix/external-api/html/index.html ``` or inside `nix-shell` or `nix develop`: ``` -# make internal-api-html -# xdg-open ./outputs/doc/share/doc/nix/internal-api/html/index.html +$ mesonConfigurePhase +$ ninja src/external-api-docs/html +$ xdg-open src/external-api-docs/html/index.html ``` diff --git a/doc/manual/src/contributing/experimental-features.md b/doc/manual/src/development/experimental-features.md similarity index 100% rename from doc/manual/src/contributing/experimental-features.md rename to doc/manual/src/development/experimental-features.md diff --git a/doc/manual/src/contributing/index.md b/doc/manual/src/development/index.md similarity index 77% rename from doc/manual/src/contributing/index.md rename to doc/manual/src/development/index.md index 4d55c17a4..6403c3e66 100644 --- a/doc/manual/src/contributing/index.md +++ b/doc/manual/src/development/index.md @@ -5,4 +5,4 @@ Check the [contributing guide](https://github.com/NixOS/nix/blob/master/CONTRIBU This chapter is a collection of guides for making changes to the code and documentation. -If you're not sure where to start, try to [compile Nix from source](./hacking.md) and consider [making improvements to documentation](./documentation.md). +If you're not sure where to start, try to [compile Nix from source](./building.md) and consider [making improvements to documentation](./documentation.md). diff --git a/doc/manual/src/development/json-guideline.md b/doc/manual/src/development/json-guideline.md new file mode 100644 index 000000000..b4bc92af9 --- /dev/null +++ b/doc/manual/src/development/json-guideline.md @@ -0,0 +1,128 @@ +# JSON guideline + +Nix consumes and produces JSON in a variety of contexts. +These guidelines ensure consistent practices for all our JSON interfaces, for ease of use, and so that experience in one part carries over to another. + +## Extensibility + +The schema of JSON input and output should allow for backwards compatible extension. +This section explains how to achieve this. + +Two definitions are helpful here, because while JSON only defines one "key-value" object type, we use it to cover two use cases: + + - **dictionary**: a map from names to value that all have the same type. + In C++ this would be a `std::map` with string keys. + + - **record**: a fixed set of attributes each with their own type. + In C++, this would be represented by a `struct`. + +It is best not to mix these use cases, as that may lead to incompatibilities when the schema changes. +For example, adding a record field to a dictionary breaks consumers that assume all JSON object fields to have the same meaning and type, and dictionary items with a colliding name can not be represented anymore. + +This leads to the following guidelines: + + - The top-level (root) value must be a record. + + Otherwise, one can not change the structure of a command's output. + + - The value of a dictionary item must be a record. + + Otherwise, the item type can not be extended. + + - List items should be records. + + Otherwise, one can not change the structure of the list items. + + If the order of the items does not matter, and each item has a unique key that is a string, consider representing the list as a dictionary instead. + If the order of the items needs to be preserved, return a list of records. + + - Streaming JSON should return records. + + An example of a streaming JSON format is [JSON lines](https://jsonlines.org/), where each line represents a JSON value. + These JSON values can be considered top-level values or list items, and they must be records. + +### Examples + +This is bad, because all keys must be assumed to be store types: + +```json +{ + "local": { ... }, + "remote": { ... }, + "http": { ... } +} +``` + +This is good, because the it is extensible at the root, and is somewhat self-documenting: + +```json +{ + "storeTypes": { "local": { ... }, ... }, + "pluginSupport": true +} +``` + +While the dictionary of store types seems like a very complete response at first, a use case may arise that warrants returning additional information. +For example, the presence of plugin support may be crucial information for a client to proceed when their desired store type is missing. + + + +The following representation is bad because it is not extensible: + +```json +{ "outputs": [ "out" "bin" ] } +``` + +However, simply converting everything to records is not enough, because the order of outputs must be preserved: + +```json +{ "outputs": { "bin": {}, "out": {} } } +``` + +The first item is the default output. Deriving this information from the outputs ordering is not great, but this is how Nix currently happens to work. +While it is possible for a JSON parser to preserve the order of fields, we can not rely on this capability to be present in all JSON libraries. + +This representation is extensible and preserves the ordering: + +```json +{ "outputs": [ { "outputName": "out" }, { "outputName": "bin" } ] } +``` + +## Self-describing values + +As described in the previous section, it's crucial that schemas can be extended with with new fields without breaking compatibility. +However, that should *not* mean we use the presence/absence of fields to indicate optional information *within* a version of the schema. +Instead, always include the field, and use `null` to indicate the "nothing" case. + +### Examples + +Here are two JSON objects: + +```json +{ + "foo": {} +} +``` +```json +{ + "foo": {}, + "bar": {} +} +``` + +Since they differ in which fields they contain, they should *not* both be valid values of the same schema. +At most, they can match two different schemas where the second (with `foo` and `bar`) is considered a newer version of the first (with just `foo`). +Within each version, all fields are mandatory (always `foo`, and always `foo` and `bar`). +Only *between* each version, `bar` gets added as a new mandatory field. + +Here are another two JSON objects: + +```json +{ "foo": null } +``` +```json +{ "foo": { "bar": 1 } } +``` + +Since they both contain a `foo` field, they could be valid values of the same schema. +The schema would have `foo` has an optional field, which is either `null` or an object where `bar` is an integer. diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/development/testing.md similarity index 81% rename from doc/manual/src/contributing/testing.md rename to doc/manual/src/development/testing.md index 31c39c16c..3949164d5 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/development/testing.md @@ -59,15 +59,15 @@ The unit tests are defined using the [googletest] and [rapidcheck] frameworks. > … > ``` -The tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `tests/unit/${library_name_without-nix}`. -Given a interface (header) and implementation pair in the original library, say, `src/libexpr/value/context.{hh,cc}`, we write tests for it in `tests/unit/libexpr/tests/value/context.cc`, and (possibly) declare/define additional interfaces for testing purposes in `tests/unit/libexpr-support/tests/value/context.{hh,cc}`. +The tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `src/${library_name_without-nix}-test`. +Given an interface (header) and implementation pair in the original library, say, `src/libexpr/value/context.{hh,cc}`, we write tests for it in `src/nix-expr-tests/value/context.cc`, and (possibly) declare/define additional interfaces for testing purposes in `src/nix-expr-test-support/tests/value/context.{hh,cc}`. Data for unit tests is stored in a `data` subdir of the directory for each unit test executable. -For example, `libnixstore` code is in `src/libstore`, and its test data is in `tests/unit/libstore/data`. -The path to the `tests/unit/data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`. +For example, `libnixstore` code is in `src/libstore`, and its test data is in `src/nix-store-tests/data`. +The path to the `src/${library_name_without-nix}-test/data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`. Note that each executable only gets the data for its tests. -The unit test libraries are in `tests/unit/${library_name_without-nix}-lib`. +The unit test libraries are in `src/${library_name_without-nix}-test-support`. All headers are in a `tests` subdirectory so they are included with `#include "tests/"`. The use of all these separate directories for the unit tests might seem inconvenient, as for example the tests are not "right next to" the part of the code they are testing. @@ -76,8 +76,25 @@ there is no risk of any build-system wildcards for the library accidentally pick ### Running tests -You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. -Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option, or the `GTEST_FILTER` environment variable, e.g. `GTEST_FILTER='ErrorTraceTest.*' make check`. +You can run the whole testsuite with `meson test` from the Meson build directory, or the tests for a specific component with `meson test nix-store-tests`. +A environment variables that Google Test accepts are also worth knowing: + +1. [`GTEST_FILTER`](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) + + This is used for finer-grained filtering of which tests to run. + + +2. [`GTEST_BRIEF`](https://google.github.io/googletest/advanced.html#suppressing-test-passes) + + This is used to avoid logging passing tests. + +Putting the two together, one might run + +```bash +GTEST_BRIEF=1 GTEST_FILTER='ErrorTraceTest.*' meson test nix-expr-tests -v +``` + +for short but comprensive output. ### Characterisation testing { #characaterisation-testing-unit } @@ -86,7 +103,7 @@ See [functional characterisation testing](#characterisation-testing-functional) Like with the functional characterisation, `_NIX_TEST_ACCEPT=1` is also used. For example: ```shell-session -$ _NIX_TEST_ACCEPT=1 make libstore-tests_RUN +$ _NIX_TEST_ACCEPT=1 meson test nix-store-tests -v ... [ SKIPPED ] WorkerProtoTest.string_read [ SKIPPED ] WorkerProtoTest.string_write @@ -114,6 +131,8 @@ On other platforms they wouldn't be run at all. The functional tests reside under the `tests/functional` directory and are listed in `tests/functional/local.mk`. Each test is a bash script. +Functional tests are run during `installCheck` in the `nix` package build, as well as separately from the build, in VM tests. + ### Running the whole test suite The whole test suite can be run with: @@ -162,14 +181,14 @@ ran test tests/functional/${testName}.sh... [PASS] or without `make`: ```shell-session -$ ./mk/run-test.sh tests/functional/${testName}.sh tests/functional/init.sh +$ ./mk/run-test.sh tests/functional/${testName}.sh ran test tests/functional/${testName}.sh... [PASS] ``` To see the complete output, one can also run: ```shell-session -$ ./mk/debug-test.sh tests/functional/${testName}.sh tests/functional/init.sh +$ ./mk/debug-test.sh tests/functional/${testName}.sh +(${testName}.sh:1) foo output from foo +(${testName}.sh:2) bar @@ -204,7 +223,7 @@ edit it like so: Then, running the test with `./mk/debug-test.sh` will drop you into GDB once the script reaches that point: ```shell-session -$ ./mk/debug-test.sh tests/functional/${testName}.sh tests/functional/init.sh +$ ./mk/debug-test.sh tests/functional/${testName}.sh ... + gdb blash blub GNU gdb (GDB) 12.1 @@ -252,13 +271,30 @@ Regressions are caught, and improvements always show up in code review. To ensure that characterisation testing doesn't make it harder to intentionally change these interfaces, there always must be an easy way to regenerate the expected output, as we do with `_NIX_TEST_ACCEPT=1`. +### Running functional tests on NixOS + +We run the functional tests not just in the build, but also in VM tests. +This helps us ensure that Nix works correctly on NixOS, and environments that have similar characteristics that are hard to reproduce in a build environment. + +The recommended way to run these tests during development is: + +```shell +nix build .#hydraJobs.tests.functional_user.quickBuild +``` + +The `quickBuild` attribute configures the test to use a `nix` package that's built without integration tests, so that you can iterate on the tests without performing recompilations due to the changed sources for `installCheck`. + +Generally, this build is sufficient, but in nightly or CI we also test the attributes `functional_root` and `functional_trusted`, in which the test suite is run with different levels of authorization. + ## Integration tests The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute. These tests include everything that needs to interact with external services or run Nix in a non-trivial distributed setup. Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on ). -You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}` +You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}`. + +If you are testing a build of `nix` that you haven't compiled yet, you may iterate faster by appending the `quickBuild` attribute: `nix build .#hydraJobs.tests.{testName}.quickBuild`. ## Installer tests diff --git a/doc/manual/src/favicon.png b/doc/manual/src/favicon.png new file mode 100644 index 000000000..1ed2b5fe0 Binary files /dev/null and b/doc/manual/src/favicon.png differ diff --git a/doc/manual/src/favicon.svg b/doc/manual/src/favicon.svg new file mode 100644 index 000000000..1d2a6e835 --- /dev/null +++ b/doc/manual/src/favicon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index a71b2e2b3..877c4668b 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -1,5 +1,24 @@ # Glossary +- [content address]{#gloss-content-address} + + A + [*content address*](https://en.wikipedia.org/wiki/Content-addressable_storage) + is a secure way to reference immutable data. + The reference is calculated directly from the content of the data being referenced, which means the reference is + [*tamper proof*](https://en.wikipedia.org/wiki/Tamperproofing) + --- variations of the data should always calculate to distinct content addresses. + + For how Nix uses content addresses, see: + + - [Content-Addressing File System Objects](@docroot@/store/file-system-object/content-address.md) + - [Content-Addressing Store Objects](@docroot@/store/store-object/content-address.md) + - [content-addressed derivation](#gloss-content-addressed-derivation) + + Software Heritage's writing on [*Intrinsic and Extrinsic identifiers*](https://www.softwareheritage.org/2020/07/09/intrinsic-vs-extrinsic-identifiers) is also a good introduction to the value of content-addressing over other referencing schemes. + + Besides content addressing, the Nix store also uses [input addressing](#gloss-input-addressed-store-object). + - [derivation]{#gloss-derivation} A description of a build task. The result of a derivation is a @@ -52,10 +71,9 @@ [`__contentAddressed`](./language/advanced-attributes.md#adv-attr-__contentAddressed) attribute set to `true`. -- [fixed-output derivation]{#gloss-fixed-output-derivation} +- [fixed-output derivation]{#gloss-fixed-output-derivation} (FOD) - A derivation which includes the - [`outputHash`](./language/advanced-attributes.md#adv-attr-outputHash) attribute. + A [derivation] where a cryptographic hash of the [output] is determined in advance using the [`outputHash`](./language/advanced-attributes.md#adv-attr-outputHash) attribute, and where the [`builder`](@docroot@/language/derivations.md#attr-builder) executable has access to the network. - [store]{#gloss-store} @@ -86,7 +104,7 @@ [store path]: #gloss-store-path -- [file system object]{#gloss-store-object} +- [file system object]{#gloss-file-system-object} The Nix data model for representing simplified file system data. @@ -118,9 +136,12 @@ - [content-addressed store object]{#gloss-content-addressed-store-object} - A [store object] whose [store path] is determined by its contents. + A [store object] which is [content-addressed](#gloss-content-address), + i.e. whose [store path] is determined by its contents. This includes derivations, the outputs of [content-addressed derivations](#gloss-content-addressed-derivation), and the outputs of [fixed-output derivations](#gloss-fixed-output-derivation). + See [Content-Addressing Store Objects](@docroot@/store/store-object/content-address.md) for details. + - [substitute]{#gloss-substitute} A substitute is a command invocation stored in the [Nix database] that @@ -147,7 +168,7 @@ - [impure derivation]{#gloss-impure-derivation} - [An experimental feature](#@docroot@/contributing/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, + [An experimental feature](#@docroot@/development/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, so that they are always rebuilt, and their outputs not reused by subsequent calls to realise them. - [Nix database]{#gloss-nix-database} @@ -215,6 +236,20 @@ [output path]: #gloss-output-path +- [output closure]{#gloss-output-closure}\ + The [closure] of an [output path]. It only contains what is [reachable] from the output. + +- [deriving path]{#gloss-deriving-path} + + Deriving paths are a way to refer to [store objects][store object] that ar not yet [realised][realise]. + This is necessary because, in general and particularly for [content-addressed derivations][content-addressed derivation], the [output path] of an [output] is not known in advance. + There are two forms: + + - *constant*: just a [store path] + It can be made [valid][validity] by copying it into the store: from the evaluator, command line interface or another store. + + - *output*: a pair of a [store path] to a [derivation] and an [output] name. + - [deriver]{#gloss-deriver} The [store derivation] that produced an [output path]. @@ -252,13 +287,15 @@ See [installables](./command-ref/new-cli/nix.md#installables) for [`nix` commands](./command-ref/new-cli/nix.md) (experimental) for details. -- [NAR]{#gloss-nar} +- [Nix Archive (NAR)]{#gloss-nar} A *N*ix *AR*chive. This is a serialisation of a path in the Nix store. It can contain regular files, directories and symbolic links. NARs are generated and unpacked using `nix-store --dump` and `nix-store --restore`. + See [Nix Archive](store/file-system-object/content-address.html#serial-nix-archive) for details. + - [`∅`]{#gloss-emtpy-set} The empty set symbol. In the context of profile history, this denotes a package is not present in a particular version of the profile. @@ -275,7 +312,7 @@ - [package attribute set]{#package-attribute-set} - An [attribute set](@docroot@/language/values.md#attribute-set) containing the attribute `type = "derivation";` (derivation for historical reasons), as well as other attributes, such as + An [attribute set](@docroot@/language/types.md#attribute-set) containing the attribute `type = "derivation";` (derivation for historical reasons), as well as other attributes, such as - attributes that refer to the files of a [package], typically in the form of [derivation outputs](#output), - attributes that declare something about how the package is supposed to be installed or used, - other metadata or arbitrary attributes. @@ -288,16 +325,35 @@ See [String interpolation](./language/string-interpolation.md) for details. - [string]: ./language/values.md#type-string - [path]: ./language/values.md#type-path - [attribute name]: ./language/values.md#attribute-set + [string]: ./language/types.md#type-string + [path]: ./language/types.md#type-path + [attribute name]: ./language/types.md#attribute-set + +- [base directory]{#gloss-base-directory} + + The location from which relative paths are resolved. + + - For expressions in a file, the base directory is the directory containing that file. + This is analogous to the directory of a [base URL](https://datatracker.ietf.org/doc/html/rfc1808#section-3.3). + + + + - For expressions written in command line arguments with [`--expr`](@docroot@/command-ref/opt-common.html#opt-expr), the base directory is the current working directory. + + [base directory]: #gloss-base-directory - [experimental feature]{#gloss-experimental-feature} Not yet stabilized functionality guarded by named experimental feature flags. These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting. - See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/contributing/experimental-features.md). + See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/development/experimental-features.md). [Nix language]: ./language/index.md diff --git a/doc/manual/src/installation/env-variables.md b/doc/manual/src/installation/env-variables.md index db98f52ff..035090421 100644 --- a/doc/manual/src/installation/env-variables.md +++ b/doc/manual/src/installation/env-variables.md @@ -53,7 +53,8 @@ ssl-cert-file = /etc/ssl/my-certificate-bundle.crt The Nix installer has special handling for these proxy-related environment variables: `http_proxy`, `https_proxy`, `ftp_proxy`, -`no_proxy`, `HTTP_PROXY`, `HTTPS_PROXY`, `FTP_PROXY`, `NO_PROXY`. +`all_proxy`, `no_proxy`, `HTTP_PROXY`, `HTTPS_PROXY`, `FTP_PROXY`, +`ALL_PROXY`, `NO_PROXY`. If any of these variables are set when running the Nix installer, then the installer will create an override file at diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index 0dc989159..6a168ff3d 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -50,7 +50,7 @@ Supported systems: To explicitly instruct the installer to perform a multi-user installation on your system: ```console -$ curl -L https://nixos.org/nix/install | sh -s -- --daemon +$ bash <(curl -L https://nixos.org/nix/install) --daemon ``` You can run this under your usual user account or `root`. @@ -61,7 +61,7 @@ The script will invoke `sudo` as needed. To explicitly select a single-user installation on your system: ```console -$ curl -L https://nixos.org/nix/install | sh -s -- --no-daemon +$ bash <(curl -L https://nixos.org/nix/install) --no-daemon ``` In a single-user installation, `/nix` is owned by the invoking user. @@ -77,7 +77,7 @@ $ su root # Installing from a binary tarball You can also download a binary tarball that contains Nix and all its dependencies: -- Choose a [version](https://releases.nixos.org/?prefix=nix/) and [system type](../contributing/hacking.md#platforms) +- Choose a [version](https://releases.nixos.org/?prefix=nix/) and [system type](../development/building.md#platforms) - Download and unpack the tarball - Run the installer diff --git a/doc/manual/src/installation/uninstall.md b/doc/manual/src/installation/uninstall.md index 9ead5e53c..590327fea 100644 --- a/doc/manual/src/installation/uninstall.md +++ b/doc/manual/src/installation/uninstall.md @@ -1,16 +1,8 @@ # Uninstalling Nix -## Single User - -If you have a [single-user installation](./installing-binary.md#single-user-installation) of Nix, uninstall it by running: - -```console -$ rm -rf /nix -``` - ## Multi User -Removing a [multi-user installation](./installing-binary.md#multi-user-installation) of Nix is more involved, and depends on the operating system. +Removing a [multi-user installation](./installing-binary.md#multi-user-installation) depends on the operating system. ### Linux @@ -51,7 +43,15 @@ which you may remove. ### macOS -1. Edit `/etc/zshrc`, `/etc/bashrc`, and `/etc/bash.bashrc` to remove the lines sourcing `nix-daemon.sh`, which should look like this: +1. If system-wide shell initialisation files haven't been altered since installing Nix, use the backups made by the installer: + + ```console + sudo mv /etc/zshrc.backup-before-nix /etc/zshrc + sudo mv /etc/bashrc.backup-before-nix /etc/bashrc + sudo mv /etc/bash.bashrc.backup-before-nix /etc/bash.bashrc + ``` + + Otherwise, edit `/etc/zshrc`, `/etc/bashrc`, and `/etc/bash.bashrc` to remove the lines sourcing `nix-daemon.sh`, which should look like this: ```bash # Nix @@ -61,18 +61,6 @@ which you may remove. # End Nix ``` - If these files haven't been altered since installing Nix you can simply put - the backups back in place: - - ```console - sudo mv /etc/zshrc.backup-before-nix /etc/zshrc - sudo mv /etc/bashrc.backup-before-nix /etc/bashrc - sudo mv /etc/bash.bashrc.backup-before-nix /etc/bash.bashrc - ``` - - This will stop shells from sourcing the file and bringing everything you - installed using Nix in scope. - 2. Stop and remove the Nix daemon services: ```console @@ -82,8 +70,7 @@ which you may remove. sudo rm /Library/LaunchDaemons/org.nixos.darwin-store.plist ``` - This stops the Nix daemon and prevents it from being started next time you - boot the system. + This stops the Nix daemon and prevents it from being started next time you boot the system. 3. Remove the `nixbld` group and the `_nixbuildN` users: @@ -94,25 +81,42 @@ which you may remove. This will remove all the build users that no longer serve a purpose. -4. Edit fstab using `sudo vifs` to remove the line mounting the Nix Store - volume on `/nix`, which looks like - `UUID= /nix apfs rw,noauto,nobrowse,suid,owners` or - `LABEL=Nix\040Store /nix apfs rw,nobrowse`. This will prevent automatic - mounting of the Nix Store volume. +4. Edit fstab using `sudo vifs` to remove the line mounting the Nix Store volume on `/nix`, which looks like -5. Edit `/etc/synthetic.conf` to remove the `nix` line. If this is the only - line in the file you can remove it entirely, `sudo rm /etc/synthetic.conf`. - This will prevent the creation of the empty `/nix` directory to provide a - mountpoint for the Nix Store volume. + ``` + UUID= /nix apfs rw,noauto,nobrowse,suid,owners + ``` + or -6. Remove the files Nix added to your system: + ``` + LABEL=Nix\040Store /nix apfs rw,nobrowse + ``` + + by setting the cursor on the respective line using the error keys, and pressing `dd`, and then `:wq` to save the file. + + This will prevent automatic mounting of the Nix Store volume. + +5. Edit `/etc/synthetic.conf` to remove the `nix` line. + If this is the only line in the file you can remove it entirely: + + ```bash + if [ -f /etc/synthetic.conf ]; then + if [ "$(cat /etc/synthetic.conf)" = "nix" ]; then + sudo rm /etc/synthetic.conf + else + sudo vi /etc/synthetic.conf + fi + fi + ``` + + This will prevent the creation of the empty `/nix` directory. + +6. Remove the files Nix added to your system, except for the store: ```console sudo rm -rf /etc/nix /var/root/.nix-profile /var/root/.nix-defexpr /var/root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels ``` - This gets rid of any data Nix may have created except for the store which is - removed next. 7. Remove the Nix Store volume: @@ -120,29 +124,32 @@ which you may remove. sudo diskutil apfs deleteVolume /nix ``` - This will remove the Nix Store volume and everything that was added to the - store. + This will remove the Nix Store volume and everything that was added to the store. - If the output indicates that the command couldn't remove the volume, you should - make sure you don't have an _unmounted_ Nix Store volume. Look for a - "Nix Store" volume in the output of the following command: + If the output indicates that the command couldn't remove the volume, you should make sure you don't have an _unmounted_ Nix Store volume. + Look for a "Nix Store" volume in the output of the following command: ```console diskutil list ``` - If you _do_ see a "Nix Store" volume, delete it by re-running the diskutil - deleteVolume command, but replace `/nix` with the store volume's `diskXsY` - identifier. + If you _do_ find a "Nix Store" volume, delete it by running `diskutil deleteVolume` with the store volume's `diskXsY` identifier. > **Note** > -> After you complete the steps here, you will still have an empty `/nix` -> directory. This is an expected sign of a successful uninstall. The empty -> `/nix` directory will disappear the next time you reboot. +> After you complete the steps here, you will still have an empty `/nix` directory. +> This is an expected sign of a successful uninstall. +> The empty `/nix` directory will disappear the next time you reboot. > -> You do not have to reboot to finish uninstalling Nix. The uninstall is -> complete. macOS (Catalina+) directly controls root directories and its -> read-only root will prevent you from manually deleting the empty `/nix` -> mountpoint. +> You do not have to reboot to finish uninstalling Nix. +> The uninstall is complete. +> macOS (Catalina+) directly controls root directories, and its read-only root will prevent you from manually deleting the empty `/nix` mountpoint. +## Single User + +To remove a [single-user installation](./installing-binary.md#single-user-installation) of Nix, run: + +```console +$ rm -rf /nix ~/.nix-channels ~/.nix-defexpr ~/.nix-profile +``` +You might also want to manually remove references to Nix from your `~/.profile`. diff --git a/doc/manual/src/installation/upgrading.md b/doc/manual/src/installation/upgrading.md index 38edcdbc5..a433f1d30 100644 --- a/doc/manual/src/installation/upgrading.md +++ b/doc/manual/src/installation/upgrading.md @@ -28,7 +28,7 @@ $ sudo su ## macOS multi-user ```console -$ sudo nix-env --install --file '' --attr nix -I nixpkgs=channel:nixpkgs-unstable +$ sudo nix-env --install --file '' --attr nix cacert -I nixpkgs=channel:nixpkgs-unstable $ sudo launchctl remove org.nixos.nix-daemon $ sudo launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist ``` diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 7306fc182..51b83fc8a 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -113,19 +113,18 @@ Derivations can declare some infrequently used optional attributes. > `nix-build`. If the [`configurable-impure-env` experimental - feature](@docroot@/contributing/experimental-features.md#xp-feature-configurable-impure-env) + feature](@docroot@/development/experimental-features.md#xp-feature-configurable-impure-env) is enabled, these environment variables can also be controlled through the [`impure-env`](@docroot@/command-ref/conf-file.md#conf-impure-env) configuration setting. - [`outputHash`]{#adv-attr-outputHash}; [`outputHashAlgo`]{#adv-attr-outputHashAlgo}; [`outputHashMode`]{#adv-attr-outputHashMode}\ - These attributes declare that the derivation is a so-called - *fixed-output derivation*, which means that a cryptographic hash of - the output is already known in advance. When the build of a - fixed-output derivation finishes, Nix computes the cryptographic - hash of the output and compares it to the hash declared with these - attributes. If there is a mismatch, the build fails. + These attributes declare that the derivation is a so-called *fixed-output derivation* (FOD), which means that a cryptographic hash of the output is already known in advance. + + As opposed to regular derivations, the [`builder`] executable of a fixed-output derivation has access to the network. + Nix computes a cryptographic hash of its output and compares that to the hash declared with these attributes. + If there is a mismatch, the derivation fails. The rationale for fixed-output derivations is derivations such as those produced by the `fetchurl` function. This function downloads a @@ -188,38 +187,49 @@ Derivations can declare some infrequently used optional attributes. } ``` - The `outputHashAlgo` attribute specifies the hash algorithm used to - compute the hash. It can currently be `"sha1"`, `"sha256"` or - `"sha512"`. + The `outputHash` attribute must be a string containing the hash in either hexadecimal or "nix32" encoding, or following the format for integrity metadata as defined by [SRI](https://www.w3.org/TR/SRI/). + The "nix32" encoding is an adaptation of base-32 encoding. + The [`convertHash`](@docroot@/language/builtins.md#builtins-convertHash) function shows how to convert between different encodings, and the [`nix-hash` command](../command-ref/nix-hash.md) has information about obtaining the hash for some contents, as well as converting to and from encodings. + + The `outputHashAlgo` attribute specifies the hash algorithm used to compute the hash. + It can currently be `"sha1"`, `"sha256"`, `"sha512"`, or `null`. + `outputHashAlgo` can only be `null` when `outputHash` follows the SRI format. The `outputHashMode` attribute determines how the hash is computed. - It must be one of the following two values: + It must be one of the following values: - - `"flat"`\ - The output must be a non-executable regular file. If it isn’t, - the build fails. The hash is simply computed over the contents - of that file (so it’s equal to what Unix commands like - `sha256sum` or `sha1sum` produce). + - [`"flat"`](@docroot@/store/store-object/content-address.md#method-flat) This is the default. - - `"recursive"`\ - The hash is computed over the NAR archive dump of the output - (i.e., the result of [`nix-store --dump`](@docroot@/command-ref/nix-store/dump.md)). In - this case, the output can be anything, including a directory - tree. + - [`"recursive"` or `"nar"`](@docroot@/store/store-object/content-address.md#method-nix-archive) - The `outputHash` attribute, finally, must be a string containing - the hash in either hexadecimal or base-32 notation. (See the - [`nix-hash` command](../command-ref/nix-hash.md) for information - about converting to and from base-32 notation.) + > **Compatibility** + > + > `"recursive"` is the traditional way of indicating this, + > and is supported since 2005 (virtually the entire history of Nix). + > `"nar"` is more clear, and consistent with other parts of Nix (such as the CLI), + > however support for it is only added in Nix version 2.21. + + - [`"text"`](@docroot@/store/store-object/content-address.md#method-text) + + > **Warning** + > + > The use of this method for derivation outputs is part of the [`dynamic-derivations`][xp-feature-dynamic-derivations] experimental feature. + + - [`"git"`](@docroot@/store/store-object/content-address.md#method-git) + + > **Warning** + > + > This method is part of the [`git-hashing`][xp-feature-git-hashing] experimental feature. - [`__contentAddressed`]{#adv-attr-__contentAddressed} + > **Warning** - > This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md). + > This attribute is part of an [experimental feature](@docroot@/development/experimental-features.md). > > To use this attribute, you must enable the - > [`ca-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-ca-derivations) experimental feature. + > [`ca-derivations`][xp-feature-ca-derivations] experimental feature. > For example, in [nix.conf](../command-ref/conf-file.md) you could add: > > ``` @@ -268,7 +278,9 @@ Derivations can declare some infrequently used optional attributes. > **Note** > - > If set to `false`, the [`builder`](./derivations.md#attr-builder) should be able to run on the system type specified in the [`system` attribute](./derivations.md#attr-system), since the derivation cannot be substituted. + > If set to `false`, the [`builder`] should be able to run on the system type specified in the [`system` attribute](./derivations.md#attr-system), since the derivation cannot be substituted. + + [`builder`]: ./derivations.md#attr-builder - [`__structuredAttrs`]{#adv-attr-structuredAttrs}\ If the special attribute `__structuredAttrs` is set to `true`, the other derivation @@ -290,6 +302,12 @@ Derivations can declare some infrequently used optional attributes. (associative) arrays. For example, the attribute `hardening.format = true` ends up as the Bash associative array element `${hardening[format]}`. + > **Warning** + > + > If set to `true`, other advanced attributes such as [`allowedReferences`](#adv-attr-allowedReferences), [`allowedReferences`](#adv-attr-allowedReferences), [`allowedRequisites`](#adv-attr-allowedRequisites), + [`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites), maxSize, and maxClosureSize. + will have no effect. + - [`outputChecks`]{#adv-attr-outputChecks}\ When using [structured attributes](#adv-attr-structuredAttrs), the `outputChecks` attribute allows defining checks per-output. @@ -299,7 +317,7 @@ Derivations can declare some infrequently used optional attributes. [`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites), the following attributes are available: - - `maxSize` defines the maximum size of the resulting [store object](../glossary.md#gloss-store-object). + - `maxSize` defines the maximum size of the resulting [store object](@docroot@/store/store-object.md). - `maxClosureSize` defines the maximum size of the output's closure. - `ignoreSelfRefs` controls whether self-references should be considered when checking for allowed references/requisites. @@ -351,3 +369,7 @@ Derivations can declare some infrequently used optional attributes. ``` ensures that the derivation can only be built on a machine with the `kvm` feature. + +[xp-feature-ca-derivations]: @docroot@/development/experimental-features.md#xp-feature-ca-derivations +[xp-feature-dynamic-derivations]: @docroot@/development/experimental-features.md#xp-feature-dynamic-derivations +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/doc/manual/src/language/builtin-constants-prefix.md b/doc/manual/src/language/builtin-constants-prefix.md deleted file mode 100644 index 50f43006d..000000000 --- a/doc/manual/src/language/builtin-constants-prefix.md +++ /dev/null @@ -1,5 +0,0 @@ -# Built-in Constants - -These constants are built into the Nix language evaluator: - - diff --git a/doc/manual/src/language/builtin-constants-suffix.md b/doc/manual/src/language/builtin-constants-suffix.md deleted file mode 100644 index a74db2857..000000000 --- a/doc/manual/src/language/builtin-constants-suffix.md +++ /dev/null @@ -1 +0,0 @@ - diff --git a/doc/manual/src/language/builtins-prefix.md b/doc/manual/src/language/builtins-prefix.md index 7b2321466..fb983bb7f 100644 --- a/doc/manual/src/language/builtins-prefix.md +++ b/doc/manual/src/language/builtins-prefix.md @@ -1,9 +1,11 @@ -# Built-in Functions +# Built-ins -This section lists the functions built into the Nix language evaluator. -All built-in functions are available through the global [`builtins`](./builtin-constants.md#builtins-builtins) constant. +This section lists the values and functions built into the Nix language evaluator. +All built-ins are available through the global [`builtins`](#builtins-builtins) constant. -For convenience, some built-ins can be accessed directly: +Some built-ins are also exposed directly in the global scope: + + - [`derivation`](#builtins-derivation) - [`import`](#builtins-import) diff --git a/doc/manual/src/language/constructs.md b/doc/manual/src/language/constructs.md deleted file mode 100644 index a82ec5960..000000000 --- a/doc/manual/src/language/constructs.md +++ /dev/null @@ -1,408 +0,0 @@ -# Language Constructs - -## Recursive sets - -Recursive sets are like normal [attribute sets](./values.md#attribute-set), but the attributes can refer to each other. - -> *rec-attrset* = `rec {` [ *name* `=` *expr* `;` `]`... `}` - -Example: - -```nix -rec { - x = y; - y = 123; -}.x -``` - -This evaluates to `123`. - -Note that without `rec` the binding `x = y;` would -refer to the variable `y` in the surrounding scope, if one exists, and -would be invalid if no such variable exists. That is, in a normal -(non-recursive) set, attributes are not added to the lexical scope; in a -recursive set, they are. - -Recursive sets of course introduce the danger of infinite recursion. For -example, the expression - -```nix -rec { - x = y; - y = x; -}.x -``` - -will crash with an `infinite recursion encountered` error message. - -## Let-expressions - -A let-expression allows you to define local variables for an expression. - -> *let-in* = `let` [ *identifier* = *expr* ]... `in` *expr* - -Example: - -```nix -let - x = "foo"; - y = "bar"; -in x + y -``` - -This evaluates to `"foobar"`. - -## Inheriting attributes - -When defining an [attribute set](./values.md#attribute-set) or in a [let-expression](#let-expressions) it is often convenient to copy variables from the surrounding lexical scope (e.g., when you want to propagate attributes). -This can be shortened using the `inherit` keyword. - -Example: - -```nix -let x = 123; in -{ - inherit x; - y = 456; -} -``` - -is equivalent to - -```nix -let x = 123; in -{ - x = x; - y = 456; -} -``` - -and both evaluate to `{ x = 123; y = 456; }`. - -> **Note** -> -> This works because `x` is added to the lexical scope by the `let` construct. - -It is also possible to inherit attributes from another attribute set. - -Example: - -In this fragment from `all-packages.nix`, - -```nix -graphviz = (import ../tools/graphics/graphviz) { - inherit fetchurl stdenv libpng libjpeg expat x11 yacc; - inherit (xorg) libXaw; -}; - -xorg = { - libX11 = ...; - libXaw = ...; - ... -} - -libpng = ...; -libjpg = ...; -... -``` - -the set used in the function call to the function defined in -`../tools/graphics/graphviz` inherits a number of variables from the -surrounding scope (`fetchurl` ... `yacc`), but also inherits `libXaw` -(the X Athena Widgets) from the `xorg` set. - -Summarizing the fragment - -```nix -... -inherit x y z; -inherit (src-set) a b c; -... -``` - -is equivalent to - -```nix -... -x = x; y = y; z = z; -a = src-set.a; b = src-set.b; c = src-set.c; -... -``` - -when used while defining local variables in a let-expression or while -defining a set. - -In a `let` expression, `inherit` can be used to selectively bring specific attributes of a set into scope. For example - - -```nix -let - x = { a = 1; b = 2; }; - inherit (builtins) attrNames; -in -{ - names = attrNames x; -} -``` - -is equivalent to - -```nix -let - x = { a = 1; b = 2; }; -in -{ - names = builtins.attrNames x; -} -``` - -both evaluate to `{ names = [ "a" "b" ]; }`. - -## Functions - -Functions have the following form: - -```nix -pattern: body -``` - -The pattern specifies what the argument of the function must look like, -and binds variables in the body to (parts of) the argument. There are -three kinds of patterns: - - - If a pattern is a single identifier, then the function matches any - argument. Example: - - ```nix - let negate = x: !x; - concat = x: y: x + y; - in if negate true then concat "foo" "bar" else "" - ``` - - Note that `concat` is a function that takes one argument and returns - a function that takes another argument. This allows partial - parameterisation (i.e., only filling some of the arguments of a - function); e.g., - - ```nix - map (concat "foo") [ "bar" "bla" "abc" ] - ``` - - evaluates to `[ "foobar" "foobla" "fooabc" ]`. - - - A *set pattern* of the form `{ name1, name2, …, nameN }` matches a - set containing the listed attributes, and binds the values of those - attributes to variables in the function body. For example, the - function - - ```nix - { x, y, z }: z + y + x - ``` - - can only be called with a set containing exactly the attributes `x`, - `y` and `z`. No other attributes are allowed. If you want to allow - additional arguments, you can use an ellipsis (`...`): - - ```nix - { x, y, z, ... }: z + y + x - ``` - - This works on any set that contains at least the three named - attributes. - - It is possible to provide *default values* for attributes, in - which case they are allowed to be missing. A default value is - specified by writing `name ? e`, where *e* is an arbitrary - expression. For example, - - ```nix - { x, y ? "foo", z ? "bar" }: z + y + x - ``` - - specifies a function that only requires an attribute named `x`, but - optionally accepts `y` and `z`. - - - An `@`-pattern provides a means of referring to the whole value - being matched: - - ```nix - args@{ x, y, z, ... }: z + y + x + args.a - ``` - - but can also be written as: - - ```nix - { x, y, z, ... } @ args: z + y + x + args.a - ``` - - Here `args` is bound to the argument *as passed*, which is further - matched against the pattern `{ x, y, z, ... }`. - The `@`-pattern makes mainly sense with an ellipsis(`...`) as - you can access attribute names as `a`, using `args.a`, which was - given as an additional attribute to the function. - - > **Warning** - > - > `args@` binds the name `args` to the attribute set that is passed to the function. - > In particular, `args` does *not* include any default values specified with `?` in the function's set pattern. - > - > For instance - > - > ```nix - > let - > f = args@{ a ? 23, ... }: [ a args ]; - > in - > f {} - > ``` - > - > is equivalent to - > - > ```nix - > let - > f = args @ { ... }: [ (args.a or 23) args ]; - > in - > f {} - > ``` - > - > and both expressions will evaluate to: - > - > ```nix - > [ 23 {} ] - > ``` - -Note that functions do not have names. If you want to give them a name, -you can bind them to an attribute, e.g., - -```nix -let concat = { x, y }: x + y; -in concat { x = "foo"; y = "bar"; } -``` - -## Conditionals - -Conditionals look like this: - -```nix -if e1 then e2 else e3 -``` - -where *e1* is an expression that should evaluate to a Boolean value -(`true` or `false`). - -## Assertions - -Assertions are generally used to check that certain requirements on or -between features and dependencies hold. They look like this: - -```nix -assert e1; e2 -``` - -where *e1* is an expression that should evaluate to a Boolean value. If -it evaluates to `true`, *e2* is returned; otherwise expression -evaluation is aborted and a backtrace is printed. - -Here is a Nix expression for the Subversion package that shows how -assertions can be used:. - -```nix -{ localServer ? false -, httpServer ? false -, sslSupport ? false -, pythonBindings ? false -, javaSwigBindings ? false -, javahlBindings ? false -, stdenv, fetchurl -, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null -}: - -assert localServer -> db4 != null; â‘ -assert httpServer -> httpd != null && httpd.expat == expat; â‘¡ -assert sslSupport -> openssl != null && (httpServer -> httpd.openssl == openssl); â‘¢ -assert pythonBindings -> swig != null && swig.pythonSupport; -assert javaSwigBindings -> swig != null && swig.javaSupport; -assert javahlBindings -> j2sdk != null; - -stdenv.mkDerivation { - name = "subversion-1.1.1"; - ... - openssl = if sslSupport then openssl else null; â‘£ - ... -} -``` - -The points of interest are: - -1. This assertion states that if Subversion is to have support for - local repositories, then Berkeley DB is needed. So if the Subversion - function is called with the `localServer` argument set to `true` but - the `db4` argument set to `null`, then the evaluation fails. - - Note that `->` is the [logical - implication](https://en.wikipedia.org/wiki/Truth_table#Logical_implication) - Boolean operation. - -2. This is a more subtle condition: if Subversion is built with Apache - (`httpServer`) support, then the Expat library (an XML library) used - by Subversion should be same as the one used by Apache. This is - because in this configuration Subversion code ends up being linked - with Apache code, and if the Expat libraries do not match, a build- - or runtime link error or incompatibility might occur. - -3. This assertion says that in order for Subversion to have SSL support - (so that it can access `https` URLs), an OpenSSL library must be - passed. Additionally, it says that *if* Apache support is enabled, - then Apache's OpenSSL should match Subversion's. (Note that if - Apache support is not enabled, we don't care about Apache's - OpenSSL.) - -4. The conditional here is not really related to assertions, but is - worth pointing out: it ensures that if SSL support is disabled, then - the Subversion derivation is not dependent on OpenSSL, even if a - non-`null` value was passed. This prevents an unnecessary rebuild of - Subversion if OpenSSL changes. - -## With-expressions - -A *with-expression*, - -```nix -with e1; e2 -``` - -introduces the set *e1* into the lexical scope of the expression *e2*. -For instance, - -```nix -let as = { x = "foo"; y = "bar"; }; -in with as; x + y -``` - -evaluates to `"foobar"` since the `with` adds the `x` and `y` attributes -of `as` to the lexical scope in the expression `x + y`. The most common -use of `with` is in conjunction with the `import` function. E.g., - -```nix -with (import ./definitions.nix); ... -``` - -makes all attributes defined in the file `definitions.nix` available as -if they were defined locally in a `let`-expression. - -The bindings introduced by `with` do not shadow bindings introduced by -other means, e.g. - -```nix -let a = 3; in with { a = 1; }; let a = 4; in with { a = 2; }; ... -``` - -establishes the same scope as - -```nix -let a = 1; in let a = 2; in let a = 3; in let a = 4; in ... -``` - -## Comments - -Comments can be single-line, started with a `#` character, or -inline/multi-line, enclosed within `/* ... */`. diff --git a/doc/manual/src/language/constructs/lookup-path.md b/doc/manual/src/language/constructs/lookup-path.md index e87d2922b..11b9fe88c 100644 --- a/doc/manual/src/language/constructs/lookup-path.md +++ b/doc/manual/src/language/constructs/lookup-path.md @@ -4,9 +4,9 @@ > > *lookup-path* = `<` *identifier* [ `/` *identifier* ]... `>` -A lookup path is an identifier with an optional path suffix that resolves to a [path value](@docroot@/language/values.md#type-path) if the identifier matches a search path entry. +A lookup path is an identifier with an optional path suffix that resolves to a [path value](@docroot@/language/types.md#type-path) if the identifier matches a search path entry. -The value of a lookup path is determined by [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath). +The value of a lookup path is determined by [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath). See [`builtins.findFile`](@docroot@/language/builtins.md#builtins-findFile) for details on lookup path resolution. diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index 75f824a34..8e3f0f791 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -12,12 +12,12 @@ It outputs an attribute set, and produces a [store derivation] as a side effect ### Required -- [`name`]{#attr-name} ([String](@docroot@/language/values.md#type-string)) +- [`name`]{#attr-name} ([String](@docroot@/language/types.md#type-string)) A symbolic name for the derivation. It is added to the [store path] of the corresponding [store derivation] as well as to its [output paths](@docroot@/glossary.md#gloss-output-path). - [store path]: @docroot@/glossary.md#gloss-store-path + [store path]: @docroot@/store/store-path.md > **Example** > @@ -31,7 +31,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect > The store derivation's path will be `/nix/store/-hello.drv`. > The [output](#attr-outputs) paths will be of the form `/nix/store/-hello[-]` -- [`system`]{#attr-system} ([String](@docroot@/language/values.md#type-string)) +- [`system`]{#attr-system} ([String](@docroot@/language/types.md#type-string)) The system type on which the [`builder`](#attr-builder) executable is meant to be run. @@ -64,9 +64,9 @@ It outputs an attribute set, and produces a [store derivation] as a side effect > } > ``` > - > [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) has the value of the [`system` configuration option], and defaults to the system type of the current Nix installation. + > [`builtins.currentSystem`](@docroot@/language/builtins.md#builtins-currentSystem) has the value of the [`system` configuration option], and defaults to the system type of the current Nix installation. -- [`builder`]{#attr-builder} ([Path](@docroot@/language/values.md#type-path) | [String](@docroot@/language/values.md#type-string)) +- [`builder`]{#attr-builder} ([Path](@docroot@/language/types.md#type-path) | [String](@docroot@/language/types.md#type-string)) Path to an executable that will perform the build. @@ -113,7 +113,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect ### Optional -- [`args`]{#attr-args} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) +- [`args`]{#attr-args} ([List](@docroot@/language/types.md#list) of [String](@docroot@/language/types.md#type-string)) Default: `[ ]` @@ -132,7 +132,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect > }; > ``` -- [`outputs`]{#attr-outputs} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) +- [`outputs`]{#attr-outputs} ([List](@docroot@/language/types.md#list) of [String](@docroot@/language/types.md#type-string)) Default: `[ "out" ]` @@ -141,7 +141,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect By default, a derivation produces a single output called `out`. However, derivations can produce multiple outputs. - This allows the associated [store objects](@docroot@/glossary.md#gloss-store-object) and their [closures](@docroot@/glossary.md#gloss-closure) to be copied or garbage-collected separately. + This allows the associated [store objects](@docroot@/store/store-object.md) and their [closures](@docroot@/glossary.md#gloss-closure) to be copied or garbage-collected separately. > **Example** > diff --git a/doc/manual/src/language/identifiers.md b/doc/manual/src/language/identifiers.md new file mode 100644 index 000000000..861ee3e20 --- /dev/null +++ b/doc/manual/src/language/identifiers.md @@ -0,0 +1,51 @@ +# Identifiers + +An *identifier* is an [ASCII](https://en.wikipedia.org/wiki/ASCII) character sequence that: +- Starts with a letter (`a-z`, `A-Z`) or underscore (`_`) +- Can contain any number of: + - Letters (`a-z`, `A-Z`) + - Digits (`0-9`) + - Underscores (`_`) + - Apostrophes (`'`) + - Hyphens (`-`) +- Is not one of the [keywords](#keywords) + +> **Syntax** +> +> *identifier* ~ `[A-Za-z_][A-Za-z0-9_'-]*` + +# Names + +A *name* can be written as an [identifier](#identifier) or a [string literal](./syntax.md#string-literal). + +> **Syntax** +> +> *name* → *identifier* | *string* + +Names are used in [attribute sets](./syntax.md#attrs-literal), [`let` bindings](./syntax.md#let-expressions), and [`inherit`](./syntax.md#inheriting-attributes). +Two names are the same if they represent the same sequence of characters, regardless of whether they are written as identifiers or strings. + +# Keywords + +These keywords are reserved and cannot be used as [identifiers](#identifiers): + +- [`assert`](./syntax.md#assertions) +- [`else`][if] +- [`if`][if] +- [`in`][let] +- [`inherit`](./syntax.md#inheriting-attributes) +- [`let`][let] +- [`or`](./operators.md#attribute-selection) (see note) +- [`rec`](./syntax.md#recursive-sets) +- [`then`][if] +- [`with`](./syntax.md#with-expressions) + +[if]: ./syntax.md#conditionals +[let]: ./syntax.md#let-expressions + +> **Note** +> +> The Nix language evaluator currently allows `or` to be used as a name in some contexts, for backwards compatibility reasons. +> Users are advised not to rely on this. +> +> There are long-standing issues with how `or` is parsed as a name, which can't be resolved without making a breaking change to the language. diff --git a/doc/manual/src/language/import-from-derivation.md b/doc/manual/src/language/import-from-derivation.md index fb12ba51a..e901f5bcf 100644 --- a/doc/manual/src/language/import-from-derivation.md +++ b/doc/manual/src/language/import-from-derivation.md @@ -2,9 +2,9 @@ The value of a Nix expression can depend on the contents of a [store object]. -[store object]: @docroot@/glossary.md#gloss-store-object +[store object]: @docroot@/store/store-object.md -Passing an expression `expr` that evaluates to a [store path](@docroot@/glossary.md#gloss-store-path) to any built-in function which reads from the filesystem constitutes Import From Derivation (IFD): +Passing an expression `expr` that evaluates to a [store path](@docroot@/store/store-path.md) to any built-in function which reads from the filesystem constitutes Import From Derivation (IFD): - [`import`](./builtins.md#builtins-import)` expr` - [`builtins.readFile`](./builtins.md#builtins-readFile)` expr` diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md index a26e43a05..2bfdbb8a0 100644 --- a/doc/manual/src/language/index.md +++ b/doc/manual/src/language/index.md @@ -1,7 +1,13 @@ # Nix Language The Nix language is designed for conveniently creating and composing *derivations* – precise descriptions of how contents of existing files are used to derive new files. -It is: + +> **Tip** +> +> These pages are written as a reference. +> If you are learning Nix, nix.dev has a good [introduction to the Nix language](https://nix.dev/tutorials/nix-language). + +The language is: - *domain-specific* @@ -47,7 +53,7 @@ This is an incomplete overview of language features, by example. - *Basic values* + *Basic values ([primitives](@docroot@/language/types.md#primitives))* @@ -65,7 +71,7 @@ This is an incomplete overview of language features, by example. - A string + A [string](@docroot@/language/types.md#type-string) @@ -88,6 +94,18 @@ This is an incomplete overview of language features, by example. + + + + `# Explanation` + + + + + A [comment](@docroot@/language/syntax.md#comments). + + + @@ -100,7 +118,7 @@ This is an incomplete overview of language features, by example. - String interpolation (expands to `"hello world"`, `"1 2 3"`, `"/nix/store/-bash-/bin/sh"`) + [String interpolation](@docroot@/language/string-interpolation.md) (expands to `"hello world"`, `"1 2 3"`, `"/nix/store/-bash-/bin/sh"`) @@ -112,7 +130,7 @@ This is an incomplete overview of language features, by example. - Booleans + [Booleans](@docroot@/language/types.md#type-boolean) @@ -124,7 +142,7 @@ This is an incomplete overview of language features, by example. - Null value + [Null](@docroot@/language/types.md#type-null) value @@ -136,7 +154,7 @@ This is an incomplete overview of language features, by example. - An integer + An [integer](@docroot@/language/types.md#type-int) @@ -148,7 +166,7 @@ This is an incomplete overview of language features, by example. - A floating point number + A [floating point number](@docroot@/language/types.md#type-float) @@ -160,7 +178,7 @@ This is an incomplete overview of language features, by example. - An absolute path + An absolute [path](@docroot@/language/types.md#type-path) @@ -172,7 +190,7 @@ This is an incomplete overview of language features, by example. - A path relative to the file containing this Nix expression + A [path](@docroot@/language/types.md#type-path) relative to the file containing this Nix expression @@ -184,7 +202,7 @@ This is an incomplete overview of language features, by example. - A home path. Evaluates to the `"/.config"`. + A home [path](@docroot@/language/types.md#type-path). Evaluates to the `"/.config"`. @@ -196,7 +214,7 @@ This is an incomplete overview of language features, by example. - Search path for Nix files. Value determined by [`$NIX_PATH` environment variable](../command-ref/env-common.md#env-NIX_PATH). + A [lookup path](@docroot@/language/constructs/lookup-path.md) for Nix files. Value determined by [`$NIX_PATH` environment variable](../command-ref/env-common.md#env-NIX_PATH). @@ -220,7 +238,7 @@ This is an incomplete overview of language features, by example. - A set with attributes named `x` and `y` + An [attribute set](@docroot@/language/types.md#attribute-set) with attributes named `x` and `y` @@ -244,7 +262,7 @@ This is an incomplete overview of language features, by example. - A recursive set, equivalent to `{ x = "foo"; y = "foobar"; }` + A [recursive set](@docroot@/language/syntax.md#recursive-sets), equivalent to `{ x = "foo"; y = "foobar"; }`. @@ -260,7 +278,7 @@ This is an incomplete overview of language features, by example. - Lists with three elements. + [Lists](@docroot@/language/types.md#list) with three elements. @@ -344,7 +362,7 @@ This is an incomplete overview of language features, by example. - Attribute selection (evaluates to `1`) + [Attribute selection](@docroot@/language/types.md#attribute-set) (evaluates to `1`) @@ -356,7 +374,7 @@ This is an incomplete overview of language features, by example. - Attribute selection with default (evaluates to `3`) + [Attribute selection](@docroot@/language/types.md#attribute-set) with default (evaluates to `3`) @@ -392,7 +410,7 @@ This is an incomplete overview of language features, by example. - Conditional expression + [Conditional expression](@docroot@/language/syntax.md#conditionals). @@ -404,7 +422,7 @@ This is an incomplete overview of language features, by example. - Assertion check (evaluates to `"yes!"`). + [Assertion](@docroot@/language/syntax.md#assertions) check (evaluates to `"yes!"`). @@ -416,7 +434,7 @@ This is an incomplete overview of language features, by example. - Variable definition + Variable definition. See [`let`-expressions](@docroot@/language/syntax.md#let-expressions). @@ -428,14 +446,44 @@ This is an incomplete overview of language features, by example. - Add all attributes from the given set to the scope (evaluates to `1`) + Add all attributes from the given set to the scope (evaluates to `1`). + + See [`with`-expressions](@docroot@/language/syntax.md#with-expressions) for details and shadowing caveats. - *Functions (lambdas)* + `inherit pkgs src;` + + + + + Adds the variables to the current scope (attribute set or `let` binding). + Desugars to `pkgs = pkgs; src = src;`. + See [Inheriting attributes](@docroot@/language/syntax.md#inheriting-attributes). + + + + + + + `inherit (pkgs) lib stdenv;` + + + + + Adds the attributes, from the attribute set in parentheses, to the current scope (attribute set or `let` binding). + Desugars to `lib = pkgs.lib; stdenv = pkgs.stdenv;`. + See [Inheriting attributes](@docroot@/language/syntax.md#inheriting-attributes). + + + + + + + *[Functions](@docroot@/language/syntax.md#functions) (lambdas)* @@ -452,7 +500,7 @@ This is an incomplete overview of language features, by example. - A function that expects an integer and returns it increased by 1 + A [function](@docroot@/language/syntax.md#functions) that expects an integer and returns it increased by 1. @@ -464,7 +512,7 @@ This is an incomplete overview of language features, by example. - Curried function, equivalent to `x: (y: x + y)`. Can be used like a function that takes two arguments and returns their sum. + Curried [function](@docroot@/language/syntax.md#functions), equivalent to `x: (y: x + y)`. Can be used like a function that takes two arguments and returns their sum. @@ -476,7 +524,7 @@ This is an incomplete overview of language features, by example. - A function call (evaluates to 101) + A [function](@docroot@/language/syntax.md#functions) call (evaluates to 101) @@ -488,7 +536,7 @@ This is an incomplete overview of language features, by example. - A function bound to a variable and subsequently called by name (evaluates to 103) + A [function](@docroot@/language/syntax.md#functions) bound to a variable and subsequently called by name (evaluates to 103) @@ -500,7 +548,7 @@ This is an incomplete overview of language features, by example. - A function that expects a set with required attributes `x` and `y` and concatenates them + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attributes `x` and `y` and concatenates them @@ -512,7 +560,7 @@ This is an incomplete overview of language features, by example. - A function that expects a set with required attribute `x` and optional `y`, using `"bar"` as default value for `y` + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attribute `x` and optional `y`, using `"bar"` as default value for `y` @@ -524,7 +572,7 @@ This is an incomplete overview of language features, by example. - A function that expects a set with required attributes `x` and `y` and ignores any other attributes + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attributes `x` and `y` and ignores any other attributes @@ -538,7 +586,7 @@ This is an incomplete overview of language features, by example. - A function that expects a set with required attributes `x` and `y`, and binds the whole set to `args` + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attributes `x` and `y`, and binds the whole set to `args` @@ -562,7 +610,8 @@ This is an incomplete overview of language features, by example. - Load and return Nix expression in given file + Load and return Nix expression in given file. + See [import](@docroot@/language/builtins.md#builtins-import). @@ -574,7 +623,8 @@ This is an incomplete overview of language features, by example. - Apply a function to every element of a list (evaluates to `[ 2 4 6 ]`) + Apply a function to every element of a list (evaluates to `[ 2 4 6 ]`). + See [`map`](@docroot@/language/builtins.md#builtins-map). diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index 6fd66864b..e1c020781 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -26,12 +26,16 @@ | Logical conjunction (`AND`) | *bool* `&&` *bool* | left | 12 | | Logical disjunction (`OR`) | *bool* \|\| *bool* | left | 13 | | [Logical implication] | *bool* `->` *bool* | right | 14 | +| [Pipe operator] (experimental) | *expr* `\|>` *func* | left | 15 | +| [Pipe operator] (experimental) | *func* `<\|` *expr* | right | 15 | -[string]: ./values.md#type-string -[path]: ./values.md#type-path -[number]: ./values.md#type-number -[list]: ./values.md#list -[attribute set]: ./values.md#attribute-set +[string]: ./types.md#type-string +[path]: ./types.md#type-path +[number]: ./types.md#type-float +[list]: ./types.md#list +[attribute set]: ./types.md#attribute-set + + ## Attribute selection @@ -42,12 +46,6 @@ Select the attribute denoted by attribute path *attrpath* from [attribute set] *attrset*. If the attribute doesn’t exist, return the *expr* after `or` if provided, otherwise abort evaluation. -An attribute path is a dot-separated list of [attribute names](./values.md#attribute-set). - -> **Syntax** -> -> *attrpath* = *name* [ `.` *name* ]... - [Attribute selection]: #attribute-selection ## Has attribute @@ -61,7 +59,7 @@ The result is a [Boolean] value. See also: [`builtins.hasAttr`](@docroot@/language/builtins.md#builtins-hasAttr) -[Boolean]: ./values.md#type-boolean +[Boolean]: ./types.md#type-boolean [Has attribute]: #has-attribute @@ -128,8 +126,8 @@ The result is a string. > The file or directory at *path* must exist and is copied to the [store]. > The path appears in the result as the corresponding [store path]. -[store path]: ../glossary.md#gloss-store-path -[store]: ../glossary.md#gloss-store +[store path]: @docroot@/store/store-path.md +[store]: @docroot@/glossary.md#gloss-store [String and path concatenation]: #string-and-path-concatenation @@ -141,7 +139,7 @@ The result is a string. Update [attribute set] *attrset1* with names and values from *attrset2*. -The returned attribute set will have of all the attributes in *attrset1* and *attrset2*. +The returned attribute set will have all of the attributes in *attrset1* and *attrset2*. If an attribute name is present in both, the attribute value from the latter is taken. [Update]: #update @@ -172,7 +170,7 @@ All comparison operators are implemented in terms of `<`, and the following equi - Numbers are type-compatible, see [arithmetic] operators. - Floating point numbers only differ up to a limited precision. -[function]: ./constructs.md#functions +[function]: ./syntax.md#functions [Equality]: #equality @@ -182,3 +180,34 @@ Equivalent to `!`*b1* `||` *b2*. [Logical implication]: #logical-implication +## Pipe operators + +- *a* `|>` *b* is equivalent to *b* *a* +- *a* `<|` *b* is equivalent to *a* *b* + +> **Example** +> +> ``` +> nix-repl> 1 |> builtins.add 2 |> builtins.mul 3 +> 9 +> +> nix-repl> builtins.add 1 <| builtins.mul 2 <| 3 +> 7 +> ``` + +> **Warning** +> +> This syntax is part of an +> [experimental feature](@docroot@/development/experimental-features.md) +> and may change in future releases. +> +> To use this syntax, make sure the +> [`pipe-operators` experimental feature](@docroot@/development/experimental-features.md#xp-feature-pipe-operators) +> is enabled. +> For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): +> +> ``` +> extra-experimental-features = pipe-operators +> ``` + +[Pipe operator]: #pipe-operators diff --git a/doc/manual/src/language/scope.md b/doc/manual/src/language/scope.md new file mode 100644 index 000000000..9373324e2 --- /dev/null +++ b/doc/manual/src/language/scope.md @@ -0,0 +1,28 @@ +# Scoping rules + +A *scope* in the Nix language is a dictionary keyed by [name](./identifiers.md#names), mapping each name to an expression and a *definition type*. +The definition type is either *explicit* or *implicit*. +Each entry in this dictionary is a *definition*. + +Explicit definitions are created by the following expressions: +- [let-expressions](syntax.md#let-expressions) +- [recursive attribute set literals](syntax.md#recursive-sets) (`rec`) +- [function literals](syntax.md#functions) + +Implicit definitions are only created by [with-expressions](./syntax.md#with-expressions). + +Every expression is *enclosed* by a scope. +The outermost expression is enclosed by the [built-in, global scope](./builtins.md), which contains only explicit definitions. +The expressions listed above *extend* their enclosing scope by adding new definitions, or replacing existing ones with the same name. +An explicit definition can replace a definition of any type; an implicit definition can only replace another implicit definition. + +Each of the above expressions defines which of its subexpressions are enclosed by the extended scope. +In all other cases, the same scope that encloses an expression is the enclosing scope for its subexpressions. + +The Nix language is [statically scoped](https://en.wikipedia.org/wiki/Scope_(computer_science)#Lexical_scope); +the value of a variable is determined only by the variable's enclosing scope, and not by the dynamic context in which the variable is evaluated. + +> **Note** +> +> Expressions entered into the [Nix REPL](@docroot@/command-ref/new-cli/nix3-repl.md) are enclosed by a scope that can be extended by command line arguments or previous REPL commands. +> These ways of extending scope are not, strictly speaking, part of the Nix language. diff --git a/doc/manual/src/language/string-context.md b/doc/manual/src/language/string-context.md new file mode 100644 index 000000000..6a3482cfd --- /dev/null +++ b/doc/manual/src/language/string-context.md @@ -0,0 +1,134 @@ +# String context + +> **Note** +> +> This is an advanced topic. +> The Nix language is designed to be used without the programmer consciously dealing with string contexts or even knowing what they are. + +A string in the Nix language is not just a sequence of characters like strings in other languages. +It is actually a pair of a sequence of characters and a *string context*. +The string context is an (unordered) set of *string context elements*. + +The purpose of string contexts is to collect non-string values attached to strings via +[string concatenation](./operators.md#string-concatenation), +[string interpolation](./string-interpolation.md), +and similar operations. +The idea is that a user can combine together values to create a build instructions for derivations without manually keeping track of where they come from. +Then the Nix language implicitly does that bookkeeping to efficiently obtain the closure of derivation inputs. + +> **Note** +> +> String contexts are *not* explicitly manipulated in idiomatic Nix language code. + +String context elements come in different forms: + +- [deriving path]{#string-context-element-derived-path} + + A string context element of this type is a [deriving path](@docroot@/glossary.md#gloss-deriving-path). + They can be either of type [constant](#string-context-constant) or [output](#string-context-output), which correspond to the types of deriving paths. + + - [Constant string context elements]{#string-context-constant} + + > **Example** + > + > [`builtins.storePath`] creates a string with a single constant string context element: + > + > ```nix + > builtins.getContext (builtins.storePath "/nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10") + > ``` + > evaluates to + > ```nix + > { + > "/nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10" = { + > path = true; + > }; + > } + > ``` + + [deriving path]: @docroot@/glossary.md#gloss-deriving-path + [store path]: @docroot@/glossary.md#gloss-store-path + [`builtins.storePath`]: ./builtins.md#builtins-storePath + + - [Output string context elements]{#string-context-output} + + > **Example** + > + > The behavior of string contexts are best demonstrated with a built-in function that is still experimental: [`builtins.outputOf`]. + > This example will *not* work with stable Nix! + > + > ```nix + > builtins.getContext + > (builtins.outputOf + > (builtins.storePath "/nix/store/fvchh9cvcr7kdla6n860hshchsba305w-hello-2.12.drv") + > "out") + > ``` + > evaluates to + > ```nix + > { + > "/nix/store/fvchh9cvcr7kdla6n860hshchsba305w-hello-2.12.drv" = { + > outputs = [ "out" ]; + > }; + > } + > ``` + + [`builtins.outputOf`]: ./builtins.md#builtins-outputOf + +- [*derivation deep*]{#string-context-element-derivation-deep} + + *derivation deep* is an advanced feature intended to be used with the + [`exportReferencesGraph` derivation attribute](./advanced-attributes.html#adv-attr-exportReferencesGraph). + A *derivation deep* string context element is a derivation path, and refers to both its outputs and the entire build closure of that derivation: + all its outputs, all the other derivations the given derivation depends on, and all the outputs of those. + + > **Example** + > + > The best way to illustrate *derivation deep* string contexts is with [`builtins.addDrvOutputDependencies`]. + > Take a regular constant string context element pointing to a derivation, and transform it into a "Derivation deep" string context element. + > + > ```nix + > builtins.getContext + > (builtins.addDrvOutputDependencies + > (builtins.storePath "/nix/store/fvchh9cvcr7kdla6n860hshchsba305w-hello-2.12.drv")) + > ``` + > evaluates to + > ```nix + > { + > "/nix/store/fvchh9cvcr7kdla6n860hshchsba305w-hello-2.12.drv" = { + > allOutputs = true; + > }; + > } + > ``` + + [`builtins.addDrvOutputDependencies`]: ./builtins.md#builtins-addDrvOutputDependencies + [`builtins.unsafeDiscardOutputDependency`]: ./builtins.md#builtins-unsafeDiscardOutputDependency + +## Inspecting string contexts + +Most basically, [`builtins.hasContext`] will tell whether a string has a non-empty context. + +When more granular information is needed, [`builtins.getContext`] can be used. +It creates an [attribute set] representing the string context, which can be inspected as usual. + +[`builtins.hasContext`]: ./builtins.md#builtins-hasContext +[`builtins.getContext`]: ./builtins.md#builtins-getContext +[attribute set]: ./types.md#attribute-set + +## Clearing string contexts + +[`buitins.unsafeDiscardStringContext`](./builtins.md#builtins-unsafeDiscardStringContext) will make a copy of a string, but with an empty string context. +The returned string can be used in more ways, e.g. by operators that require the string context to be empty. +The requirement to explicitly discard the string context in such use cases helps ensure that string context elements are not lost by mistake. +The "unsafe" marker is only there to remind that Nix normally guarantees that dependencies are tracked, whereas the returned string has lost them. + +## Constructing string contexts + +[`builtins.appendContext`] will create a copy of a string, but with additional string context elements. +The context is specified explicitly by an [attribute set] in the format that [`builtins.hasContext`] produces. +A string with arbitrary contexts can be made like this: + +1. Create a string with the desired string context elements. + (The contents of the string do not matter.) +2. Dump its context with [`builtins.getContext`]. +3. Combine it with a base string and repeated [`builtins.appendContext`] calls. + +[`builtins.appendContext`]: ./builtins.md#builtins-appendContext diff --git a/doc/manual/src/language/string-interpolation.md b/doc/manual/src/language/string-interpolation.md index 7d81c2020..1778bdfa0 100644 --- a/doc/manual/src/language/string-interpolation.md +++ b/doc/manual/src/language/string-interpolation.md @@ -4,9 +4,9 @@ String interpolation is a language feature where a [string], [path], or [attribu Such a construct is called *interpolated string*, and the expression inside is an [interpolated expression](#interpolated-expression). -[string]: ./values.md#type-string -[path]: ./values.md#type-path -[attribute set]: ./values.md#attribute-set +[string]: ./types.md#type-string +[path]: ./types.md#type-path +[attribute set]: ./types.md#attribute-set ## Examples @@ -20,7 +20,7 @@ Rather than writing (where `freetype` is a [derivation]), you can instead write -[derivation]: ../glossary.md#gloss-derivation +[derivation]: @docroot@/glossary.md#gloss-derivation ```nix "--with-freetype2-library=${freetype}/lib" @@ -43,6 +43,47 @@ configureFlags = " Note that Nix expressions and strings can be arbitrarily nested; in this case the outer string contains various interpolated expressions that themselves contain strings (e.g., `"-thread"`), some of which in turn contain interpolated expressions (e.g., `${mesa}`). +To write a literal `${` in an regular string, escape it with a backslash (`\`). + +> **Example** +> +> ```nix +> "echo \${PATH}" +> ``` +> +> "echo ${PATH}" + +To write a literal `${` in an indented string, escape it with two single quotes (`''`). + +> **Example** +> +> ```nix +> '' +> echo ''${PATH} +> '' +> ``` +> +> "echo ${PATH}\n" + +`$${` can be written literally in any string. + +> **Example** +> +> In Make, `$` in file names or recipes is represented as `$$`, see [GNU `make`: Basics of Variable Reference](https://www.gnu.org/software/make/manual/html_node/Reference.html#Basics-of-Variable-References). +> This can be expressed directly in the Nix language strings: +> +> ```nix +> '' +> MAKEVAR = Hello +> all: +> @export BASHVAR=world; echo $(MAKEVAR) $${BASHVAR} +> '' +> ``` +> +> "MAKEVAR = Hello\nall:\n\t@export BASHVAR=world; echo $(MAKEVAR) $\${BASHVAR}\n" + +See the [documentation on strings][string] for details. + ### Path Rather than writing @@ -107,9 +148,9 @@ An expression that is interpolated must evaluate to one of the following: A string interpolates to itself. -A path in an interpolated expression is first copied into the Nix store, and the resulting string is the [store path] of the newly created [store object](../glossary.md#gloss-store-object). +A path in an interpolated expression is first copied into the Nix store, and the resulting string is the [store path] of the newly created [store object](@docroot@/store/store-object.md). -[store path]: ../glossary.md#gloss-store-path +[store path]: @docroot@/store/store-path.md > **Example** > diff --git a/doc/manual/src/language/syntax.md b/doc/manual/src/language/syntax.md new file mode 100644 index 000000000..6108bacd6 --- /dev/null +++ b/doc/manual/src/language/syntax.md @@ -0,0 +1,866 @@ +# Language Constructs + +This section covers syntax and semantics of the Nix language. + +## Basic Literals + +### String {#string-literal} + + *Strings* can be written in three ways. + + The most common way is to enclose the string between double quotes, e.g., `"foo bar"`. + Strings can span multiple lines. + The results of other expressions can be included into a string by enclosing them in `${ }`, a feature known as [string interpolation]. + + [string interpolation]: ./string-interpolation.md + + The following must be escaped to represent them within a string, by prefixing with a backslash (`\`): + + - Double quote (`"`) + + > **Example** + > + > ```nix + > "\"" + > ``` + > + > "\"" + + - Backslash (`\`) + + > **Example** + > + > ```nix + > "\\" + > ``` + > + > "\\" + + - Dollar sign followed by an opening curly bracket (`${`) – "dollar-curly" + + > **Example** + > + > ```nix + > "\${" + > ``` + > + > "\${" + + The newline, carriage return, and tab characters can be written as `\n`, `\r` and `\t`, respectively. + + A "double-dollar-curly" (`$${`) can be written literally. + + > **Example** + > + > ```nix + > "$${" + > ``` + > + > "$\${" + + String values are output on the terminal with Nix-specific escaping. + Strings written to files will contain the characters encoded by the escaping. + + The second way to write string literals is as an *indented string*, which is enclosed between pairs of *double single-quotes* (`''`), like so: + + ```nix + '' + This is the first line. + This is the second line. + This is the third line. + '' + ``` + + This kind of string literal intelligently strips indentation from + the start of each line. To be precise, it strips from each line a + number of spaces equal to the minimal indentation of the string as a + whole (disregarding the indentation of empty lines). For instance, + the first and second line are indented two spaces, while the third + line is indented four spaces. Thus, two spaces are stripped from + each line, so the resulting string is + + ```nix + "This is the first line.\nThis is the second line.\n This is the third line.\n" + ``` + + > **Note** + > + > Whitespace and newline following the opening `''` is ignored if there is no non-whitespace text on the initial line. + + > **Warning** + > + > Prefixed tab characters are not stripped. + > + > > **Example** + > > + > > The following indented string is prefixed with tabs: + > > + > > '' + > > all: + > > @echo hello + > > '' + > > + > > "\tall:\n\t\t@echo hello\n" + + Indented strings support [string interpolation]. + + The following must be escaped to represent them in an indented string: + + - `$` is escaped by prefixing it with two single quotes (`''`) + + > **Example** + > + > ```nix + > '' + > ''$ + > '' + > ``` + > + > "$\n" + + - `''` is escaped by prefixing it with one single quote (`'`) + + > **Example** + > + > ```nix + > '' + > ''' + > '' + > ``` + > + > "''\n" + + These special characters are escaped as follows: + - Linefeed (`\n`): `''\n` + - Carriage return (`\r`): `''\r` + - Tab (`\t`): `''\t` + + `''\` escapes any other character. + + A "double-dollar-curly" (`$${`) can be written literally. + + > **Example** + > + > ```nix + > '' + > $${ + > '' + > ``` + > + > "$\${\n" + + Indented strings are primarily useful in that they allow multi-line + string literals to follow the indentation of the enclosing Nix + expression, and that less escaping is typically necessary for + strings representing languages such as shell scripts and + configuration files because `''` is much less common than `"`. + Example: + + ```nix + stdenv.mkDerivation { + ... + postInstall = + '' + mkdir $out/bin $out/etc + cp foo $out/bin + echo "Hello World" > $out/etc/foo.conf + ${if enableBar then "cp bar $out/bin" else ""} + ''; + ... + } + ``` + + Finally, as a convenience, *URIs* as defined in appendix B of + [RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as + is*, without quotes. For instance, the string + `"http://example.org/foo.tar.bz2"` can also be written as + `http://example.org/foo.tar.bz2`. + +### Number {#number-literal} + + + + Numbers, which can be *integers* (like `123`) or *floating point* + (like `123.43` or `.27e13`). + + See [arithmetic] and [comparison] operators for semantics. + + [arithmetic]: ./operators.md#arithmetic + [comparison]: ./operators.md#comparison + +### Path {#path-literal} + + *Paths* can be expressed by path literals such as `./builder.sh`. + + A path literal must contain at least one slash to be recognised as such. + For instance, `builder.sh` is not a path: + it's parsed as an expression that selects the attribute `sh` from the variable `builder`. + + Path literals are resolved relative to their [base directory](@docroot@/glossary.md#gloss-base-directory). + Path literals may also refer to absolute paths by starting with a slash. + + > **Note** + > + > Absolute paths make expressions less portable. + > In the case where a function translates a path literal into an absolute path string for a configuration file, it is recommended to write a string literal instead. + > This avoids some confusion about whether files at that location will be used during evaluation. + > It also avoids unintentional situations where some function might try to copy everything at the location into the store. + + If the first component of a path is a `~`, it is interpreted such that the rest of the path were relative to the user's home directory. + For example, `~/foo` would be equivalent to `/home/edolstra/foo` for a user whose home directory is `/home/edolstra`. + Path literals that start with `~` are not allowed in [pure](@docroot@/command-ref/conf-file.md#conf-pure-eval) evaluation. + + Path literals can also include [string interpolation], besides being [interpolated into other expressions]. + + [interpolated into other expressions]: ./string-interpolation.md#interpolated-expressions + + At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path. + + `a.${foo}/b.${bar}` is a syntactically valid number division operation. + `./a.${foo}/b.${bar}` is a path. + + [Lookup path](./constructs/lookup-path.md) literals such as `` also resolve to path values. + +## List {#list-literal} + +Lists are formed by enclosing a whitespace-separated list of values +between square brackets. For example, + +```nix +[ 123 ./foo.nix "abc" (f { x = y; }) ] +``` + +defines a list of four elements, the last being the result of a call to +the function `f`. Note that function calls have to be enclosed in +parentheses. If they had been omitted, e.g., + +```nix +[ 123 ./foo.nix "abc" f { x = y; } ] +``` + +the result would be a list of five elements, the fourth one being a +function and the fifth being a set. + +Note that lists are only lazy in values, and they are strict in length. + +Elements in a list can be accessed using [`builtins.elemAt`](./builtins.md#builtins-elemAt). + +## Attribute Set {#attrs-literal} + +An attribute set is a collection of name-value-pairs called *attributes*. + +Attribute sets are written enclosed in curly brackets (`{ }`). +Attribute names and attribute values are separated by an equal sign (`=`). +Each value can be an arbitrary expression, terminated by a semicolon (`;`) + +An attribute name is a string without context, and is denoted by a [name] (an [identifier](./identifiers.md#identifiers) or [string literal](#string-literal)). + +[name]: ./identifiers.md#names + +> **Syntax** +> +> *attrset* → `{` { *name* `=` *expr* `;` } `}` + +Attributes can appear in any order. +An attribute name may only occur once in each attribute set. + +> **Example** +> +> This defines an attribute set with attributes named: +> - `x` with the value `123`, an integer +> - `text` with the value `"Hello"`, a string +> - `y` where the value is the result of applying the function `f` to the attribute set `{ bla = 456; }` +> +> ```nix +> { +> x = 123; +> text = "Hello"; +> y = f { bla = 456; }; +> } +> ``` + +Attributes in nested attribute sets can be written using *attribute paths*. + +> **Syntax** +> +> *attrset* → `{` { *attrpath* `=` *expr* `;` } `}` + +An attribute path is a dot-separated list of [names][name]. + +> **Syntax** +> +> *attrpath* = *name* { `.` *name* } + + + +> **Example** +> +> ```nix +> { a.b.c = 1; a.b.d = 2; } +> ``` +> +> { +> a = { +> b = { +> c = 1; +> d = 2; +> }; +> }; +> } + +Attribute names can also be set implicitly by using the [`inherit` keyword](#inheriting-attributes). + +> **Example** +> +> ```nix +> { inherit (builtins) true; } +> ``` +> +> { true = true; } + +Attributes can be accessed with the [`.` operator](./operators.md#attribute-selection). + +Example: + +```nix +{ a = "Foo"; b = "Bar"; }.a +``` + +This evaluates to `"Foo"`. + +It is possible to provide a default value in an attribute selection using the `or` keyword. + +Example: + +```nix +{ a = "Foo"; b = "Bar"; }.c or "Xyzzy" +``` + +```nix +{ a = "Foo"; b = "Bar"; }.c.d.e.f.g or "Xyzzy" +``` + +will both evaluate to `"Xyzzy"` because there is no `c` attribute in the set. + +You can use arbitrary double-quoted strings as attribute names: + +```nix +{ "$!@#?" = 123; }."$!@#?" +``` + +```nix +let bar = "bar"; in +{ "foo ${bar}" = 123; }."foo ${bar}" +``` + +Both will evaluate to `123`. + +Attribute names support [string interpolation]: + +```nix +let bar = "foo"; in +{ foo = 123; }.${bar} +``` + +```nix +let bar = "foo"; in +{ ${bar} = 123; }.foo +``` + +Both will evaluate to `123`. + +In the special case where an attribute name inside of a set declaration +evaluates to `null` (which is normally an error, as `null` cannot be coerced to +a string), that attribute is simply not added to the set: + +```nix +{ ${if foo then "bar" else null} = true; } +``` + +This will evaluate to `{}` if `foo` evaluates to `false`. + +A set that has a `__functor` attribute whose value is callable (i.e. is +itself a function or a set with a `__functor` attribute whose value is +callable) can be applied as if it were a function, with the set itself +passed in first , e.g., + +```nix +let add = { __functor = self: x: x + self.x; }; + inc = add // { x = 1; }; +in inc 1 +``` + +evaluates to `2`. This can be used to attach metadata to a function +without the caller needing to treat it specially, or to implement a form +of object-oriented programming, for example. + +## Recursive sets + +Recursive sets are like normal [attribute sets](./types.md#attribute-set), but the attributes can refer to each other. + +> *rec-attrset* = `rec {` [ *name* `=` *expr* `;` `]`... `}` + +Example: + +```nix +rec { + x = y; + y = 123; +}.x +``` + +This evaluates to `123`. + +Note that without `rec` the binding `x = y;` would +refer to the variable `y` in the surrounding scope, if one exists, and +would be invalid if no such variable exists. That is, in a normal +(non-recursive) set, attributes are not added to the lexical scope; in a +recursive set, they are. + +Recursive sets of course introduce the danger of infinite recursion. For +example, the expression + +```nix +rec { + x = y; + y = x; +}.x +``` + +will crash with an `infinite recursion encountered` error message. + +## Let-expressions + +A let-expression allows you to define local variables for an expression. + +> *let-in* = `let` [ *identifier* = *expr* ]... `in` *expr* + +Example: + +```nix +let + x = "foo"; + y = "bar"; +in x + y +``` + +This evaluates to `"foobar"`. + +## Inheriting attributes + +When defining an [attribute set](./types.md#attribute-set) or in a [let-expression](#let-expressions) it is often convenient to copy variables from the surrounding lexical scope (e.g., when you want to propagate attributes). +This can be shortened using the `inherit` keyword. + +Example: + +```nix +let x = 123; in +{ + inherit x; + y = 456; +} +``` + +is equivalent to + +```nix +let x = 123; in +{ + x = x; + y = 456; +} +``` + +and both evaluate to `{ x = 123; y = 456; }`. + +> **Note** +> +> This works because `x` is added to the lexical scope by the `let` construct. + +It is also possible to inherit attributes from another attribute set. + +Example: + +In this fragment from `all-packages.nix`, + +```nix +graphviz = (import ../tools/graphics/graphviz) { + inherit fetchurl stdenv libpng libjpeg expat x11 yacc; + inherit (xorg) libXaw; +}; + +xorg = { + libX11 = ...; + libXaw = ...; + ... +} + +libpng = ...; +libjpg = ...; +... +``` + +the set used in the function call to the function defined in +`../tools/graphics/graphviz` inherits a number of variables from the +surrounding scope (`fetchurl` ... `yacc`), but also inherits `libXaw` +(the X Athena Widgets) from the `xorg` set. + +Summarizing the fragment + +```nix +... +inherit x y z; +inherit (src-set) a b c; +... +``` + +is equivalent to + +```nix +... +x = x; y = y; z = z; +a = src-set.a; b = src-set.b; c = src-set.c; +... +``` + +when used while defining local variables in a let-expression or while +defining a set. + +In a `let` expression, `inherit` can be used to selectively bring specific attributes of a set into scope. For example + + +```nix +let + x = { a = 1; b = 2; }; + inherit (builtins) attrNames; +in +{ + names = attrNames x; +} +``` + +is equivalent to + +```nix +let + x = { a = 1; b = 2; }; +in +{ + names = builtins.attrNames x; +} +``` + +both evaluate to `{ names = [ "a" "b" ]; }`. + +## Functions + +Functions have the following form: + +```nix +pattern: body +``` + +The pattern specifies what the argument of the function must look like, +and binds variables in the body to (parts of) the argument. There are +three kinds of patterns: + + - If a pattern is a single identifier, then the function matches any + argument. Example: + + ```nix + let negate = x: !x; + concat = x: y: x + y; + in if negate true then concat "foo" "bar" else "" + ``` + + Note that `concat` is a function that takes one argument and returns + a function that takes another argument. This allows partial + parameterisation (i.e., only filling some of the arguments of a + function); e.g., + + ```nix + map (concat "foo") [ "bar" "bla" "abc" ] + ``` + + evaluates to `[ "foobar" "foobla" "fooabc" ]`. + + - A *set pattern* of the form `{ name1, name2, …, nameN }` matches a + set containing the listed attributes, and binds the values of those + attributes to variables in the function body. For example, the + function + + ```nix + { x, y, z }: z + y + x + ``` + + can only be called with a set containing exactly the attributes `x`, + `y` and `z`. No other attributes are allowed. If you want to allow + additional arguments, you can use an ellipsis (`...`): + + ```nix + { x, y, z, ... }: z + y + x + ``` + + This works on any set that contains at least the three named + attributes. + + It is possible to provide *default values* for attributes, in + which case they are allowed to be missing. A default value is + specified by writing `name ? e`, where *e* is an arbitrary + expression. For example, + + ```nix + { x, y ? "foo", z ? "bar" }: z + y + x + ``` + + specifies a function that only requires an attribute named `x`, but + optionally accepts `y` and `z`. + + - An `@`-pattern provides a means of referring to the whole value + being matched: + + ```nix + args@{ x, y, z, ... }: z + y + x + args.a + ``` + + but can also be written as: + + ```nix + { x, y, z, ... } @ args: z + y + x + args.a + ``` + + Here `args` is bound to the argument *as passed*, which is further + matched against the pattern `{ x, y, z, ... }`. + The `@`-pattern makes mainly sense with an ellipsis(`...`) as + you can access attribute names as `a`, using `args.a`, which was + given as an additional attribute to the function. + + > **Warning** + > + > `args@` binds the name `args` to the attribute set that is passed to the function. + > In particular, `args` does *not* include any default values specified with `?` in the function's set pattern. + > + > For instance + > + > ```nix + > let + > f = args@{ a ? 23, ... }: [ a args ]; + > in + > f {} + > ``` + > + > is equivalent to + > + > ```nix + > let + > f = args @ { ... }: [ (args.a or 23) args ]; + > in + > f {} + > ``` + > + > and both expressions will evaluate to: + > + > ```nix + > [ 23 {} ] + > ``` + +Note that functions do not have names. If you want to give them a name, +you can bind them to an attribute, e.g., + +```nix +let concat = { x, y }: x + y; +in concat { x = "foo"; y = "bar"; } +``` + +## Conditionals + +Conditionals look like this: + +```nix +if e1 then e2 else e3 +``` + +where *e1* is an expression that should evaluate to a Boolean value +(`true` or `false`). + +## Assertions + +Assertions are generally used to check that certain requirements on or +between features and dependencies hold. They look like this: + +```nix +assert e1; e2 +``` + +where *e1* is an expression that should evaluate to a Boolean value. If +it evaluates to `true`, *e2* is returned; otherwise expression +evaluation is aborted and a backtrace is printed. + +Here is a Nix expression for the Subversion package that shows how +assertions can be used:. + +```nix +{ localServer ? false +, httpServer ? false +, sslSupport ? false +, pythonBindings ? false +, javaSwigBindings ? false +, javahlBindings ? false +, stdenv, fetchurl +, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null +}: + +assert localServer -> db4 != null; â‘ +assert httpServer -> httpd != null && httpd.expat == expat; â‘¡ +assert sslSupport -> openssl != null && (httpServer -> httpd.openssl == openssl); â‘¢ +assert pythonBindings -> swig != null && swig.pythonSupport; +assert javaSwigBindings -> swig != null && swig.javaSupport; +assert javahlBindings -> j2sdk != null; + +stdenv.mkDerivation { + name = "subversion-1.1.1"; + ... + openssl = if sslSupport then openssl else null; â‘£ + ... +} +``` + +The points of interest are: + +1. This assertion states that if Subversion is to have support for + local repositories, then Berkeley DB is needed. So if the Subversion + function is called with the `localServer` argument set to `true` but + the `db4` argument set to `null`, then the evaluation fails. + + Note that `->` is the [logical + implication](https://en.wikipedia.org/wiki/Truth_table#Logical_implication) + Boolean operation. + +2. This is a more subtle condition: if Subversion is built with Apache + (`httpServer`) support, then the Expat library (an XML library) used + by Subversion should be same as the one used by Apache. This is + because in this configuration Subversion code ends up being linked + with Apache code, and if the Expat libraries do not match, a build- + or runtime link error or incompatibility might occur. + +3. This assertion says that in order for Subversion to have SSL support + (so that it can access `https` URLs), an OpenSSL library must be + passed. Additionally, it says that *if* Apache support is enabled, + then Apache's OpenSSL should match Subversion's. (Note that if + Apache support is not enabled, we don't care about Apache's + OpenSSL.) + +4. The conditional here is not really related to assertions, but is + worth pointing out: it ensures that if SSL support is disabled, then + the Subversion derivation is not dependent on OpenSSL, even if a + non-`null` value was passed. This prevents an unnecessary rebuild of + Subversion if OpenSSL changes. + +## With-expressions + +A *with-expression*, + +```nix +with e1; e2 +``` + +introduces the set *e1* into the lexical scope of the expression *e2*. +For instance, + +```nix +let as = { x = "foo"; y = "bar"; }; +in with as; x + y +``` + +evaluates to `"foobar"` since the `with` adds the `x` and `y` attributes +of `as` to the lexical scope in the expression `x + y`. The most common +use of `with` is in conjunction with the `import` function. E.g., + +```nix +with (import ./definitions.nix); ... +``` + +makes all attributes defined in the file `definitions.nix` available as +if they were defined locally in a `let`-expression. + +The bindings introduced by `with` do not shadow bindings introduced by +other means, e.g. + +```nix +let a = 3; in with { a = 1; }; let a = 4; in with { a = 2; }; ... +``` + +establishes the same scope as + +```nix +let a = 1; in let a = 2; in let a = 3; in let a = 4; in ... +``` + +Variables coming from outer `with` expressions *are* shadowed: + +```nix +with { a = "outer"; }; +with { a = "inner"; }; +a +``` + +Does evaluate to `"inner"`. + +## Comments + +- Inline comments start with `#` and run until the end of the line. + + > **Example** + > + > ```nix + > # A number + > 2 # Equals 1 + 1 + > ``` + > + > ```console + > 2 + > ``` + +- Block comments start with `/*` and run until the next occurrence of `*/`. + + > **Example** + > + > ```nix + > /* + > Block comments + > can span multiple lines. + > */ "hello" + > ``` + > + > ```console + > "hello" + > ``` + + This means that block comments cannot be nested. + + > **Example** + > + > ```nix + > /* /* nope */ */ 1 + > ``` + > + > ```console + > error: syntax error, unexpected '*' + > + > at «string»:1:15: + > + > 1| /* /* nope */ * + > | ^ + > ``` + + Consider escaping nested comments and unescaping them in post-processing. + + > **Example** + > + > ```nix + > /* /* nested *\/ */ 1 + > ``` + > + > ```console + > 1 + > ``` diff --git a/doc/manual/src/language/types.md b/doc/manual/src/language/types.md new file mode 100644 index 000000000..229756e6b --- /dev/null +++ b/doc/manual/src/language/types.md @@ -0,0 +1,120 @@ +# Data Types + +Every value in the Nix language has one of the following types: + +* [Integer](#type-int) +* [Float](#type-float) +* [Boolean](#type-bool) +* [String](#type-string) +* [Path](#type-path) +* [Null](#type-null) +* [Attribute set](#type-attrs) +* [List](#type-list) +* [Function](#type-function) +* [External](#type-external) + +## Primitives + +### Integer {#type-int} + +An _integer_ in the Nix language is a signed 64-bit integer. + +Non-negative integers can be expressed as [integer literals](syntax.md#number-literal). +Negative integers are created with the [arithmetic negation operator](./operators.md#arithmetic). +The function [`builtins.isInt`](builtins.md#builtins-isInt) can be used to determine if a value is an integer. + +### Float {#type-float} + +A _float_ in the Nix language is a 64-bit [IEEE 754](https://en.wikipedia.org/wiki/IEEE_754) floating-point number. + +Most non-negative floats can be expressed as [float literals](syntax.md#number-literal). +Negative floats are created with the [arithmetic negation operator](./operators.md#arithmetic). +The function [`builtins.isFloat`](builtins.md#builtins-isFloat) can be used to determine if a value is a float. + +### Boolean {#type-bool} + +A _boolean_ in the Nix language is one of _true_ or _false_. + + + +These values are available as attributes of [`builtins`](builtins.md#builtins-builtins) as [`builtins.true`](builtins.md#builtins-true) and [`builtins.false`](builtins.md#builtins-false). +The function [`builtins.isBool`](builtins.md#builtins-isBool) can be used to determine if a value is a boolean. + +### String {#type-string} + +A _string_ in the Nix language is an immutable, finite-length sequence of bytes, along with a [string context](string-context.md). +Nix does not assume or support working natively with character encodings. + +String values without string context can be expressed as [string literals](syntax.md#string-literal). +The function [`builtins.isString`](builtins.md#builtins-isString) can be used to determine if a value is a string. + +### Path {#type-path} + +A _path_ in the Nix language is an immutable, finite-length sequence of bytes starting with `/`, representing a POSIX-style, canonical file system path. +Path values are distinct from string values, even if they contain the same sequence of bytes. +Operations that produce paths will simplify the result as the standard C function [`realpath`] would, except that there is no symbolic link resolution. + +[`realpath`]: https://pubs.opengroup.org/onlinepubs/9699919799/functions/realpath.html + +Paths are suitable for referring to local files, and are often preferable over strings. +- Path values do not contain trailing or duplicate slashes, `.`, or `..`. +- Relative path literals are automatically resolved relative to their [base directory]. +- Tooling can recognize path literals and provide additional features, such as autocompletion, refactoring automation and jump-to-file. + +[base directory]: @docroot@/glossary.md#gloss-base-directory + +A file is not required to exist at a given path in order for that path value to be valid, but a path that is converted to a string with [string interpolation] or [string-and-path concatenation] must resolve to a readable file or directory which will be copied into the Nix store. +For instance, evaluating `"${./foo.txt}"` will cause `foo.txt` from the same directory to be copied into the Nix store and result in the string `"/nix/store/-foo.txt"`. +Operations such as [`import`] can also expect a path to resolve to a readable file or directory. + +[string interpolation]: string-interpolation.md#interpolated-expression +[string-and-path concatenation]: operators.md#string-and-path-concatenation +[`import`]: builtins.md#builtins-import + +> **Note** +> +> The Nix language assumes that all input files will remain _unchanged_ while evaluating a Nix expression. +> For example, assume you used a file path in an interpolated string during a `nix repl` session. +> Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new [store path], since Nix might not re-read the file contents. +> Use `:r` to reset the repl as needed. + +[store path]: @docroot@/store/store-path.md + +Path values can be expressed as [path literals](syntax.md#path-literal). +The function [`builtins.isPath`](builtins.md#builtins-isPath) can be used to determine if a value is a path. + +### Null {#type-null} + +There is a single value of type _null_ in the Nix language. + + + +This value is available as an attribute on the [`builtins`](builtins.md#builtins-builtins) attribute set as [`builtins.null`](builtins.md#builtins-null). + +## Compound values + +### Attribute set {#type-attrs} + + + +An attribute set can be constructed with an [attribute set literal](syntax.md#attrs-literal). +The function [`builtins.isAttrs`](builtins.md#builtins-isAttrs) can be used to determine if a value is an attribute set. + +### List {#type-list} + + + +A list can be constructed with a [list literal](syntax.md#list-literal). +The function [`builtins.isList`](builtins.md#builtins-isList) can be used to determine if a value is a list. + +## Function {#type-function} + + + +A function can be constructed with a [function expression](syntax.md#functions). +The function [`builtins.isFunction`](builtins.md#builtins-isFunction) can be used to determine if a value is a function. + +## External {#type-external} + +An _external_ value is an opaque value created by a Nix [plugin](../command-ref/conf-file.md#conf-plugin-files). +Such a value can be substituted in Nix expressions but only created and used by plugin code. diff --git a/doc/manual/src/language/values.md b/doc/manual/src/language/values.md deleted file mode 100644 index 74ffc7070..000000000 --- a/doc/manual/src/language/values.md +++ /dev/null @@ -1,269 +0,0 @@ -# Data Types - -## Primitives - -- String - - *Strings* can be written in three ways. - - The most common way is to enclose the string between double quotes, - e.g., `"foo bar"`. Strings can span multiple lines. The special - characters `"` and `\` and the character sequence `${` must be - escaped by prefixing them with a backslash (`\`). Newlines, carriage - returns and tabs can be written as `\n`, `\r` and `\t`, - respectively. - - You can include the results of other expressions into a string by enclosing them in `${ }`, a feature known as [string interpolation]. - - [string interpolation]: ./string-interpolation.md - - The second way to write string literals is as an *indented string*, - which is enclosed between pairs of *double single-quotes*, like so: - - ```nix - '' - This is the first line. - This is the second line. - This is the third line. - '' - ``` - - This kind of string literal intelligently strips indentation from - the start of each line. To be precise, it strips from each line a - number of spaces equal to the minimal indentation of the string as a - whole (disregarding the indentation of empty lines). For instance, - the first and second line are indented two spaces, while the third - line is indented four spaces. Thus, two spaces are stripped from - each line, so the resulting string is - - ```nix - "This is the first line.\nThis is the second line.\n This is the third line.\n" - ``` - - Note that the whitespace and newline following the opening `''` is - ignored if there is no non-whitespace text on the initial line. - - Indented strings support [string interpolation]. - - Since `${` and `''` have special meaning in indented strings, you - need a way to quote them. `$` can be escaped by prefixing it with - `''` (that is, two single quotes), i.e., `''$`. `''` can be escaped - by prefixing it with `'`, i.e., `'''`. `$` removes any special - meaning from the following `$`. Linefeed, carriage-return and tab - characters can be written as `''\n`, `''\r`, `''\t`, and `''\` - escapes any other character. - - Indented strings are primarily useful in that they allow multi-line - string literals to follow the indentation of the enclosing Nix - expression, and that less escaping is typically necessary for - strings representing languages such as shell scripts and - configuration files because `''` is much less common than `"`. - Example: - - ```nix - stdenv.mkDerivation { - ... - postInstall = - '' - mkdir $out/bin $out/etc - cp foo $out/bin - echo "Hello World" > $out/etc/foo.conf - ${if enableBar then "cp bar $out/bin" else ""} - ''; - ... - } - ``` - - Finally, as a convenience, *URIs* as defined in appendix B of - [RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as - is*, without quotes. For instance, the string - `"http://example.org/foo.tar.bz2"` can also be written as - `http://example.org/foo.tar.bz2`. - -- Number - - Numbers, which can be *integers* (like `123`) or *floating point* - (like `123.43` or `.27e13`). - - See [arithmetic] and [comparison] operators for semantics. - - [arithmetic]: ./operators.md#arithmetic - [comparison]: ./operators.md#comparison - -- Path - - *Paths*, e.g., `/bin/sh` or `./builder.sh`. A path must contain at - least one slash to be recognised as such. For instance, `builder.sh` - is not a path: it's parsed as an expression that selects the - attribute `sh` from the variable `builder`. If the file name is - relative, i.e., if it does not begin with a slash, it is made - absolute at parse time relative to the directory of the Nix - expression that contained it. For instance, if a Nix expression in - `/foo/bar/bla.nix` refers to `../xyzzy/fnord.nix`, the absolute path - is `/foo/xyzzy/fnord.nix`. - - If the first component of a path is a `~`, it is interpreted as if - the rest of the path were relative to the user's home directory. - e.g. `~/foo` would be equivalent to `/home/edolstra/foo` for a user - whose home directory is `/home/edolstra`. - - For instance, evaluating `"${./foo.txt}"` will cause `foo.txt` in the current directory to be copied into the Nix store and result in the string `"/nix/store/-foo.txt"`. - - Note that the Nix language assumes that all input files will remain _unchanged_ while evaluating a Nix expression. - For example, assume you used a file path in an interpolated string during a `nix repl` session. - Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new [store path], since Nix might not re-read the file contents. - - [store path]: ../glossary.md#gloss-store-path - - Paths can include [string interpolation] and can themselves be [interpolated in other expressions]. - - [interpolated in other expressions]: ./string-interpolation.md#interpolated-expressions - - At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path. - - `a.${foo}/b.${bar}` is a syntactically valid division operation. - `./a.${foo}/b.${bar}` is a path. - - [Lookup paths](./constructs/lookup-path.md) such as `` resolve to path values. - -- Boolean - - *Booleans* with values `true` and `false`. - -- Null - - The null value, denoted as `null`. - -## List - -Lists are formed by enclosing a whitespace-separated list of values -between square brackets. For example, - -```nix -[ 123 ./foo.nix "abc" (f { x = y; }) ] -``` - -defines a list of four elements, the last being the result of a call to -the function `f`. Note that function calls have to be enclosed in -parentheses. If they had been omitted, e.g., - -```nix -[ 123 ./foo.nix "abc" f { x = y; } ] -``` - -the result would be a list of five elements, the fourth one being a -function and the fifth being a set. - -Note that lists are only lazy in values, and they are strict in length. - -Elements in a list can be accessed using [`builtins.elemAt`](./builtins.md#builtins-elemAt). - -## Attribute Set - -An attribute set is a collection of name-value-pairs (called *attributes*) enclosed in curly brackets (`{ }`). - -An attribute name can be an identifier or a [string](#string). -An identifier must start with a letter (`a-z`, `A-Z`) or underscore (`_`), and can otherwise contain letters (`a-z`, `A-Z`), numbers (`0-9`), underscores (`_`), apostrophes (`'`), or dashes (`-`). - -> **Syntax** -> -> *name* = *identifier* | *string* \ -> *identifier* ~ `[a-zA-Z_][a-zA-Z0-9_'-]*` - -Names and values are separated by an equal sign (`=`). -Each value is an arbitrary expression terminated by a semicolon (`;`). - -> **Syntax** -> -> *attrset* = `{` [ *name* `=` *expr* `;` ]... `}` - -Attributes can appear in any order. -An attribute name may only occur once. - -Example: - -```nix -{ - x = 123; - text = "Hello"; - y = f { bla = 456; }; -} -``` - -This defines a set with attributes named `x`, `text`, `y`. - -Attributes can be accessed with the [`.` operator](./operators.md#attribute-selection). - -Example: - -```nix -{ a = "Foo"; b = "Bar"; }.a -``` - -This evaluates to `"Foo"`. - -It is possible to provide a default value in an attribute selection using the `or` keyword. - -Example: - -```nix -{ a = "Foo"; b = "Bar"; }.c or "Xyzzy" -``` - -```nix -{ a = "Foo"; b = "Bar"; }.c.d.e.f.g or "Xyzzy" -``` - -will both evaluate to `"Xyzzy"` because there is no `c` attribute in the set. - -You can use arbitrary double-quoted strings as attribute names: - -```nix -{ "$!@#?" = 123; }."$!@#?" -``` - -```nix -let bar = "bar"; in -{ "foo ${bar}" = 123; }."foo ${bar}" -``` - -Both will evaluate to `123`. - -Attribute names support [string interpolation]: - -```nix -let bar = "foo"; in -{ foo = 123; }.${bar} -``` - -```nix -let bar = "foo"; in -{ ${bar} = 123; }.foo -``` - -Both will evaluate to `123`. - -In the special case where an attribute name inside of a set declaration -evaluates to `null` (which is normally an error, as `null` cannot be coerced to -a string), that attribute is simply not added to the set: - -```nix -{ ${if foo then "bar" else null} = true; } -``` - -This will evaluate to `{}` if `foo` evaluates to `false`. - -A set that has a `__functor` attribute whose value is callable (i.e. is -itself a function or a set with a `__functor` attribute whose value is -callable) can be applied as if it were a function, with the set itself -passed in first , e.g., - -```nix -let add = { __functor = self: x: x + self.x; }; - inc = add // { x = 1; }; -in inc 1 -``` - -evaluates to `2`. This can be used to attach metadata to a function -without the caller needing to treat it specially, or to implement a form -of object-oriented programming, for example. diff --git a/doc/manual/src/language/variables.md b/doc/manual/src/language/variables.md new file mode 100644 index 000000000..af6aff8a2 --- /dev/null +++ b/doc/manual/src/language/variables.md @@ -0,0 +1,10 @@ +# Variables + +A *variable* is an [identifier](identifiers.md) used as an expression. + +> **Syntax** +> +> *expression* → *identifier* + +A variable must have the same name as a definition in the [scope](./scope.md) that encloses it. +The value of a variable is the value of the corresponding expression in the enclosing scope. diff --git a/doc/manual/src/package-management/copy-closure.md b/doc/manual/src/package-management/copy-closure.md deleted file mode 100644 index 14326298b..000000000 --- a/doc/manual/src/package-management/copy-closure.md +++ /dev/null @@ -1,34 +0,0 @@ -# Copying Closures via SSH - -The command `nix-copy-closure` copies a Nix store path along with all -its dependencies to or from another machine via the SSH protocol. It -doesn’t copy store paths that are already present on the target machine. -For example, the following command copies Firefox with all its -dependencies: - - $ nix-copy-closure --to alice@itchy.example.org $(type -p firefox) - -See the [manpage for `nix-copy-closure`](../command-ref/nix-copy-closure.md) for details. - -With `nix-store ---export` and `nix-store --import` you can write the closure of a store -path (that is, the path and all its dependencies) to a file, and then -unpack that file into another Nix store. For example, - - $ nix-store --export $(nix-store --query --requisites $(type -p firefox)) > firefox.closure - -writes the closure of Firefox to a file. You can then copy this file to -another machine and install the closure: - - $ nix-store --import < firefox.closure - -Any store paths in the closure that are already present in the target -store are ignored. It is also possible to pipe the export into another -command, e.g. to copy and install a closure directly to/on another -machine: - - $ nix-store --export $(nix-store --query --requisites $(type -p firefox)) | bzip2 | \ - ssh alice@itchy.example.org "bunzip2 | nix-store --import" - -However, `nix-copy-closure` is generally more efficient because it only -copies paths that are not already present in the target Nix store. diff --git a/doc/manual/src/protocols/derivation-aterm.md b/doc/manual/src/protocols/derivation-aterm.md index e58b602a3..1ba757ae0 100644 --- a/doc/manual/src/protocols/derivation-aterm.md +++ b/doc/manual/src/protocols/derivation-aterm.md @@ -14,6 +14,6 @@ Derivations are serialised in one of the following formats: DrvWithVersion(, ...) ``` - The only `version-string`s that are in use today are for [experimental features](@docroot@/contributing/experimental-features.md): + The only `version-string`s that are in use today are for [experimental features](@docroot@/development/experimental-features.md): - - `"xp-dyn-drv"` for the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. + - `"xp-dyn-drv"` for the [`dynamic-derivations`](@docroot@/development/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. diff --git a/doc/manual/src/protocols/json/derivation.md b/doc/manual/src/protocols/json/derivation.md index 649d543cc..2f85340d6 100644 --- a/doc/manual/src/protocols/json/derivation.md +++ b/doc/manual/src/protocols/json/derivation.md @@ -3,7 +3,7 @@ > **Warning** > > This JSON format is currently -> [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) +> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) > and subject to change. The JSON serialization of a @@ -18,10 +18,30 @@ is a JSON object with the following fields: Information about the output paths of the derivation. This is a JSON object with one member per output, where the key is the output name and the value is a JSON object with these fields: - * `path`: The output path. + * `path`: + The output path, if it is known in advanced. + Otherwise, `null`. + + + * `method`: + For an output which will be [content addresed], a string representing the [method](@docroot@/store/store-object/content-address.md) of content addressing that is chosen. + Valid method strings are: + + - [`flat`](@docroot@/store/store-object/content-address.md#method-flat) + - [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive) + - [`text`](@docroot@/store/store-object/content-address.md#method-text) + - [`git`](@docroot@/store/store-object/content-address.md#method-git) + + Otherwise, `null`. * `hashAlgo`: - For fixed-output derivations, the hashing algorithm (e.g. `sha256`), optionally prefixed by `r:` if `hash` denotes a NAR hash rather than a flat file hash. + For an output which will be [content addresed], the name of the hash algorithm used. + Valid algorithm strings are: + + - `md5` + - `sha1` + - `sha256` + - `sha512` * `hash`: For fixed-output derivations, the expected content hash in base-16. @@ -32,7 +52,8 @@ is a JSON object with the following fields: > "outputs": { > "out": { > "path": "/nix/store/2543j7c6jn75blc3drf4g5vhb1rhdq29-source", - > "hashAlgo": "r:sha256", + > "method": "nar", + > "hashAlgo": "sha256", > "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62" > } > } diff --git a/doc/manual/src/protocols/json/store-object-info.md b/doc/manual/src/protocols/json/store-object-info.md index ba4ab098f..6b4f48437 100644 --- a/doc/manual/src/protocols/json/store-object-info.md +++ b/doc/manual/src/protocols/json/store-object-info.md @@ -3,7 +3,7 @@ > **Warning** > > This JSON format is currently -> [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) +> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) > and subject to change. Info about a [store object]. @@ -24,41 +24,45 @@ Info about a [store object]. An array of [store paths][store path], possibly including this one. -* `ca` (optional): +* `ca`: - Content address of this store object's file system object, used to compute its store path. + If the store object is [content-addressed], + this is the content address of this store object's file system object, used to compute its store path. + Otherwise (i.e. if it is [input-addressed]), this is `null`. -[store path]: @docroot@/glossary.md#gloss-store-path +[store path]: @docroot@/store/store-path.md [file system object]: @docroot@/store/file-system-object.md -[Nix Archive]: @docroot@/glossary.md#gloss-nar +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive ## Impure fields These are not intrinsic properties of the store object. In other words, the same store object residing in different store could have different values for these properties. -* `deriver` (optional): +* `deriver`: - The path to the [derivation] from which this store object is produced. + If known, the path to the [derivation] from which this store object was produced. + Otherwise `null`. [derivation]: @docroot@/glossary.md#gloss-store-derivation * `registrationTime` (optional): - When this derivation was added to the store. + If known, when this derivation was added to the store. + Otherwise `null`. -* `ultimate` (optional): +* `ultimate`: Whether this store object is trusted because we built it ourselves, rather than substituted a build product from elsewhere. -* `signatures` (optional): +* `signatures`: Signatures claiming that this store object is what it claims to be. Not relevant for [content-addressed] store objects, but useful for [input-addressed] store objects. - [content-addressed]: @docroot@/glossary.md#gloss-content-addressed-store-object - [input-addressed]: @docroot@/glossary.md#gloss-input-addressed-store-object +[content-addressed]: @docroot@/store/store-object/content-address.md +[input-addressed]: @docroot@/glossary.md#gloss-input-addressed-store-object ### `.narinfo` extra fields @@ -83,7 +87,7 @@ This information is not intrinsic to the store object, but about how it is store ## Computed closure fields -These fields are not stored at all, but computed by traverising the other other fields across all the store objects in a [closure]. +These fields are not stored at all, but computed by traversing the other fields across all the store objects in a [closure]. * `closureSize`: diff --git a/doc/manual/src/protocols/nix-archive.md b/doc/manual/src/protocols/nix-archive.md new file mode 100644 index 000000000..640b527f1 --- /dev/null +++ b/doc/manual/src/protocols/nix-archive.md @@ -0,0 +1,43 @@ +# Nix Archive (NAR) format + +This is the complete specification of the [Nix Archive] format. +The Nix Archive format closely follows the abstract specification of a [file system object] tree, +because it is designed to serialize exactly that data structure. + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#nix-archive +[file system object]: @docroot@/store/file-system-object.md + +The format of this specification is close to [Extended Backus–Naur form](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form), with the exception of the `str(..)` function / parameterized rule, which length-prefixes and pads strings. +This makes the resulting binary format easier to parse. + +Regular users do *not* need to know this information. +But for those interested in exactly how Nix works, e.g. if they are reimplementing it, this information can be useful. + +```ebnf +nar = str("nix-archive-1"), nar-obj; + +nar-obj = str("("), nar-obj-inner, str(")"); + +nar-obj-inner + = str("type"), str("regular") regular + | str("type"), str("symlink") symlink + | str("type"), str("directory") directory + ; + +regular = [ str("executable"), str("") ], str("contents"), str(contents); + +symlink = str("target"), str(target); + +(* side condition: directory entries must be ordered by their names *) +directory = { directory-entry }; + +directory-entry = str("entry"), str("("), str("name"), str(name), str("node"), nar-obj, str(")"); +``` + +The `str` function / parameterized rule is defined as follows: + +- `str(s)` = `int(|s|), pad(s);` + +- `int(n)` = the 64-bit little endian representation of the number `n` + +- `pad(s)` = the byte sequence `s`, padded with 0s to a multiple of 8 byte diff --git a/doc/manual/src/protocols/store-path.md b/doc/manual/src/protocols/store-path.md index 565c4fa75..52352d358 100644 --- a/doc/manual/src/protocols/store-path.md +++ b/doc/manual/src/protocols/store-path.md @@ -1,12 +1,14 @@ # Complete Store Path Calculation -This is the complete specification for how store paths are calculated. +This is the complete specification for how [store path]s are calculated. The format of this specification is close to [Extended Backus–Naur form](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form), but must deviate for a few things such as hash functions which we treat as bidirectional for specification purposes. Regular users do *not* need to know this information --- store paths can be treated as black boxes computed from the properties of the store objects they refer to. But for those interested in exactly how Nix works, e.g. if they are reimplementing it, this information can be useful. +[store path](@docroot@/store/store-path.md) + ## Store path proper ```ebnf @@ -34,18 +36,23 @@ where - `type` = one of: - ```ebnf - | "text" ( ":" store-path )* + | "text" { ":" store-path } ``` - for encoded derivations written to the store. + This is for the + ["Text"](@docroot@/store/store-object/content-address.md#method-text) + method of content addressing store objects. The optional trailing store paths are the references of the store object. - ```ebnf - | "source" ( ":" store-path )* + | "source" { ":" store-path } [ ":self" ] ``` - For paths copied to the store and hashed via a [Nix Archive (NAR)] and [SHA-256][sha-256]. - Just like in the text case, we can have the store objects referenced by their paths. + This is for the + ["Nix Archive"](@docroot@/store/store-object/content-address.md#method-nix-archive) + method of content addressing store objects, + if the hash algorithm is [SHA-256]. + Just like in the "Text" case, we can have the store objects referenced by their paths. Additionally, we can have an optional `:self` label to denote self reference. - ```ebnf @@ -53,8 +60,12 @@ where ``` For either the outputs built from derivations, - paths copied to the store hashed that area single file hashed directly, or the via a hash algorithm other than [SHA-256][sha-256]. - (in that case "source" is used; this is only necessary for compatibility). + or content-addressed store objects that are not using one of the two above cases. + To be explicit about the latter, that is currently these methods: + + - ["Flat"](@docroot@/store/store-object/content-address.md#method-flat) + - ["Git"](@docroot@/store/store-object/content-address.md#method-git) + - ["Nix Archive"](@docroot@/store/store-object/content-address.md#method-nix-archive) if the hash algorithm is not [SHA-256]. `id` is the name of the output (usually, "out"). For content-addressed store objects, `id`, is always "out". @@ -113,8 +124,8 @@ where Note that `id` = `"out"`, regardless of the name part of the store path. Also note that NAR + SHA-256 must not use this case, and instead must use the `type` = `"source:" ...` case. -[Nix Archive (NAR)]: @docroot@/glossary.md#gloss-NAR -[sha-256]: https://en.m.wikipedia.org/wiki/SHA-256 +[Nix Archive (NAR)]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive +[SHA-256]: https://en.m.wikipedia.org/wiki/SHA-256 ### Historical Note diff --git a/doc/manual/src/protocols/tarball-fetcher.md b/doc/manual/src/protocols/tarball-fetcher.md index 274fa6d63..5cff05d66 100644 --- a/doc/manual/src/protocols/tarball-fetcher.md +++ b/doc/manual/src/protocols/tarball-fetcher.md @@ -22,7 +22,7 @@ Link: ; rel="immutable" *flakeref* must be a tarball flakeref. It can contain the tarball flake attributes `narHash`, `rev`, `revCount` and `lastModified`. If `narHash` is included, its -value must be the NAR hash of the unpacked tarball (as computed via +value must be the [NAR hash][Nix Archive] of the unpacked tarball (as computed via `nix hash path`). Nix checks the contents of the returned tarball against the `narHash` attribute. The `rev` and `revCount` attributes are useful when the tarball flake is a mirror of a fetcher type that @@ -40,3 +40,31 @@ Link: ///archive/.tar.gz +``` + +> **Example** +> +> +> ```nix +> # flake.nix +> { +> inputs = { +> foo.url = "https://gitea.example.org/some-person/some-flake/archive/main.tar.gz"; +> bar.url = "https://gitea.example.org/some-other-person/other-flake/archive/442793d9ec0584f6a6e82fa253850c8085bb150a.tar.gz"; +> qux = { +> url = "https://forgejo.example.org/another-person/some-non-flake-repo/archive/development.tar.gz"; +> flake = false; +> }; +> }; +> outputs = { foo, bar, qux }: { /* ... */ }; +> } +``` + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive diff --git a/doc/manual/src/quick-start.md b/doc/manual/src/quick-start.md index 75853ced7..9eb7a3265 100644 --- a/doc/manual/src/quick-start.md +++ b/doc/manual/src/quick-start.md @@ -34,7 +34,7 @@ For more in-depth information you are kindly referred to subsequent chapters. lolcat: command not found ``` -1. Search for more packages on to try them out. +1. Search for more packages on [search.nixos.org](https://search.nixos.org/) to try them out. 1. Free up storage space: diff --git a/doc/manual/src/release-notes/index.md b/doc/manual/src/release-notes/index.md index cc805e631..d4e6292a6 100644 --- a/doc/manual/src/release-notes/index.md +++ b/doc/manual/src/release-notes/index.md @@ -1,12 +1,13 @@ # Nix Release Notes +The Nix release cycle is calendar-based as follows: + Nix has a release cycle of roughly 6 weeks. Notable changes and additions are announced in the release notes for each version. -Bugfixes can be backported on request to previous Nix releases. -We typically backport only as far back as the Nix version used in the latest NixOS release, which is announced in the [NixOS release notes](https://nixos.org/manual/nixos/stable/release-notes.html#ch-release-notes). - -Backports never skip releases. -If a feature is backported to version `x.y`, it must also be available in version `x.(y+1)`. -This ensures that upgrading from an older version with backports is still safe and no backported functionality will go missing. +The supported Nix versions are: +- The latest release +- The version used in the stable NixOS release, which is announced in the [NixOS release notes](https://nixos.org/manual/nixos/stable/release-notes.html#ch-release-notes). +Bugfixes and security issues are backported to every supported version. +Patch releases are published as needed. diff --git a/doc/manual/src/release-notes/rl-2.15.md b/doc/manual/src/release-notes/rl-2.15.md index 133121999..e7e52631b 100644 --- a/doc/manual/src/release-notes/rl-2.15.md +++ b/doc/manual/src/release-notes/rl-2.15.md @@ -11,7 +11,7 @@ As the choice of hash formats is no longer binary, the `--base16` flag is also added to explicitly specify the Base16 format, which is still the default. -* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](../glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents. +* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](@docroot@/glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents. The new `^` syntax for store paths introduced in Nix 2.13 allows explicitly referencing output paths of a derivation. Using this is better and more clear than relying on the now-removed `.drv` special handling. diff --git a/doc/manual/src/release-notes/rl-2.18.md b/doc/manual/src/release-notes/rl-2.18.md index 4bbc52b50..eb26fc9e7 100644 --- a/doc/manual/src/release-notes/rl-2.18.md +++ b/doc/manual/src/release-notes/rl-2.18.md @@ -13,7 +13,7 @@ - The `discard-references` feature has been stabilized. This means that the - [unsafeDiscardReferences](@docroot@/contributing/experimental-features.md#xp-feature-discard-references) + [unsafeDiscardReferences](@docroot@/development/experimental-features.md#xp-feature-discard-references) attribute is no longer guarded by an experimental flag and can be used freely. @@ -21,7 +21,7 @@ This only affects `nix-build --json` when "building" non-derivation things like fetched sources, which is a no-op. - A new builtin [`outputOf`](@docroot@/language/builtins.md#builtins-outputOf) has been added. - It is part of the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. + It is part of the [`dynamic-derivations`](@docroot@/development/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. - Flake follow paths at depths greater than 2 are now handled correctly, preventing "follows a non-existent input" errors. diff --git a/doc/manual/src/release-notes/rl-2.19.md b/doc/manual/src/release-notes/rl-2.19.md index ba6eb9c64..e2e2f85cc 100644 --- a/doc/manual/src/release-notes/rl-2.19.md +++ b/doc/manual/src/release-notes/rl-2.19.md @@ -17,8 +17,8 @@ - `nix-shell` shebang lines now support single-quoted arguments. -- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). - This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). +- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/development/experimental-features.md#xp-fetch-tree). + This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/development/experimental-features.md#xp-fetch-tree). - The interface for creating and updating lock files has been overhauled: @@ -33,7 +33,7 @@ - The flake-specific flags `--recreate-lock-file` and `--update-input` have been removed from all commands operating on installables. They are superceded by `nix flake update`. -- Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). +- Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches). - [`nix path-info --json`](@docroot@/command-ref/new-cli/nix3-path-info.md) (experimental) now returns a JSON map rather than JSON list. diff --git a/doc/manual/src/release-notes/rl-2.20.md b/doc/manual/src/release-notes/rl-2.20.md index 8ede168a4..eb724f600 100644 --- a/doc/manual/src/release-notes/rl-2.20.md +++ b/doc/manual/src/release-notes/rl-2.20.md @@ -200,3 +200,9 @@ while performing various operations (including `nix develop`, `nix flake update`, and so on). With several fixes to Nix's signal handlers, Nix commands will now exit quickly after Ctrl-C is pressed. + +- `nix copy` to a `ssh-ng` store now needs `--substitute-on-destination` (a.k.a. `-s`) + in order to substitute paths on the remote store instead of copying them. + The behavior is consistent with `nix copy` to a different kind of remote store. + Previously this behavior was controlled by the + `builders-use-substitutes` setting and `--substitute-on-destination` was ignored. diff --git a/doc/manual/src/release-notes/rl-2.21.md b/doc/manual/src/release-notes/rl-2.21.md new file mode 100644 index 000000000..75114f117 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.21.md @@ -0,0 +1,302 @@ +# Release 2.21.0 (2024-03-11) + +- Fix a fixed-output derivation sandbox escape (CVE-2024-27297) + + Cooperating Nix derivations could send file descriptors to files in the Nix + store to each other via Unix domain sockets in the abstract namespace. This + allowed one derivation to modify the output of the other derivation, after Nix + has registered the path as "valid" and immutable in the Nix database. + In particular, this allowed the output of fixed-output derivations to be + modified from their expected content. + + This isn't the case any more. + +- CLI options `--arg-from-file` and `--arg-from-stdin` [#10122](https://github.com/NixOS/nix/pull/10122) + + The new CLI option `--arg-from-file` *name* *path* passes the contents + of file *path* as a string value via the function argument *name* to a + Nix expression. Similarly, the new option `--arg-from-stdin` *name* + reads the contents of the string from standard input. + +- Concise error printing in `nix repl` [#9928](https://github.com/NixOS/nix/pull/9928) + + Previously, if an element of a list or attribute set threw an error while + evaluating, `nix repl` would print the entire error (including source location + information) inline. This output was clumsy and difficult to parse: + + ``` + nix-repl> { err = builtins.throw "uh oh!"; } + { err = «error: + … while calling the 'throw' builtin + at «string»:1:9: + 1| { err = builtins.throw "uh oh!"; } + | ^ + + error: uh oh!»; } + ``` + + Now, only the error message is displayed, making the output much more readable. + ``` + nix-repl> { err = builtins.throw "uh oh!"; } + { err = «error: uh oh!»; } + ``` + + However, if the whole expression being evaluated throws an error, source + locations and (if applicable) a stack trace are printed, just like you'd expect: + + ``` + nix-repl> builtins.throw "uh oh!" + error: + … while calling the 'throw' builtin + at «string»:1:1: + 1| builtins.throw "uh oh!" + | ^ + + error: uh oh! + ``` + +- `--debugger` can now access bindings from `let` expressions [#8827](https://github.com/NixOS/nix/issues/8827) [#9918](https://github.com/NixOS/nix/pull/9918) + + Breakpoints and errors in the bindings of a `let` expression can now access + those bindings in the debugger. Previously, only the body of `let` expressions + could access those bindings. + +- Enter the `--debugger` when `builtins.trace` is called if `debugger-on-trace` is set [#9914](https://github.com/NixOS/nix/pull/9914) + + If the `debugger-on-trace` option is set and `--debugger` is given, + `builtins.trace` calls will behave similarly to `builtins.break` and will enter + the debug REPL. This is useful for determining where warnings are being emitted + from. + +- Debugger prints source position information [#9913](https://github.com/NixOS/nix/pull/9913) + + The `--debugger` now prints source location information, instead of the + pointers of source location information. Before: + + ``` + nix-repl> :bt + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + 0x600001522598 + ``` + + After: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + /nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27 + + 131| + 132| bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs; + | ^ + 133| in + ``` + +- The `--debugger` will start more reliably in `let` expressions and function calls [#6649](https://github.com/NixOS/nix/issues/6649) [#9917](https://github.com/NixOS/nix/pull/9917) + + Previously, if you attempted to evaluate this file with the debugger: + + ```nix + let + a = builtins.trace "before inner break" ( + builtins.break "hello" + ); + b = builtins.trace "before outer break" ( + builtins.break a + ); + in + b + ``` + + Nix would correctly enter the debugger at `builtins.break a`, but if you asked + it to `:continue`, it would skip over the `builtins.break "hello"` expression + entirely. + + Now, Nix will correctly enter the debugger at both breakpoints. + +- Nested debuggers are no longer supported [#9920](https://github.com/NixOS/nix/pull/9920) + + Previously, evaluating an expression that throws an error in the debugger would + enter a second, nested debugger: + + ``` + nix-repl> builtins.throw "what" + error: what + + + Starting REPL to allow you to inspect the current state of the evaluator. + + Welcome to Nix 2.18.1. Type :? for help. + + nix-repl> + ``` + + Now, it just prints the error message like `nix repl`: + + ``` + nix-repl> builtins.throw "what" + error: + … while calling the 'throw' builtin + at «string»:1:1: + 1| builtins.throw "what" + | ^ + + error: what + ``` + +- Consistent order of function arguments in printed expressions [#9874](https://github.com/NixOS/nix/pull/9874) + + Function arguments are now printed in lexicographic order rather than the internal, creation-time based symbol order. + +- Fix duplicate attribute error positions for `inherit` [#9874](https://github.com/NixOS/nix/pull/9874) + + When an `inherit` caused a duplicate attribute error the position of the error was not reported correctly, placing the error with the inherit itself or at the start of the bindings block instead of the offending attribute name. + +- `inherit (x) ...` evaluates `x` only once [#9847](https://github.com/NixOS/nix/pull/9847) + + `inherit (x) a b ...` now evaluates the expression `x` only once for all inherited attributes rather than once for each inherited attribute. + This does not usually have a measurable impact, but side-effects (such as `builtins.trace`) would be duplicated and expensive expressions (such as derivations) could cause a measurable slowdown. + +- Store paths are allowed to start with `.` [#912](https://github.com/NixOS/nix/issues/912) [#9091](https://github.com/NixOS/nix/pull/9091) [#9095](https://github.com/NixOS/nix/pull/9095) [#9120](https://github.com/NixOS/nix/pull/9120) [#9121](https://github.com/NixOS/nix/pull/9121) [#9122](https://github.com/NixOS/nix/pull/9122) [#9130](https://github.com/NixOS/nix/pull/9130) [#9219](https://github.com/NixOS/nix/pull/9219) [#9224](https://github.com/NixOS/nix/pull/9224) [#9867](https://github.com/NixOS/nix/pull/9867) + + Leading periods were allowed by accident in Nix 2.4. The Nix team has considered this to be a bug, but this behavior has since been relied on by users, leading to unnecessary difficulties. + From now on, leading periods are supported. The names `.` and `..` are disallowed, as well as those starting with `.-` or `..-`. + + Nix versions that denied leading periods are documented [in the issue](https://github.com/NixOS/nix/issues/912#issuecomment-1919583286). + +- `nix repl` pretty-prints values [#9931](https://github.com/NixOS/nix/pull/9931) + + `nix repl` will now pretty-print values: + + ``` + { + attrs = { + a = { + b = { + c = { }; + }; + }; + }; + list = [ 1 ]; + list' = [ + 1 + 2 + 3 + ]; + } + ``` + +- Introduction of `--regex` and `--all` in `nix profile remove` and `nix profile upgrade` [#10166](https://github.com/NixOS/nix/pull/10166) + + Previously the command-line arguments for `nix profile remove` and `nix profile upgrade` matched the package entries using regular expression. + For instance: + + ``` + nix profile remove '.*vim.*' + ``` + + This would remove all packages that contain `vim` in their name. + + In most cases, only singular package names were used to remove and upgrade packages. Mixing this with regular expressions sometimes lead to unintended behavior. For instance, `python3.1` could match `python311`. + + To avoid unintended behavior, the arguments are now only matching exact names. + + Matching using regular expressions is still possible by using the new `--regex` flag: + + ``` + nix profile remove --regex '.*vim.*' + ``` + + One of the most useful cases for using regular expressions was to upgrade all packages. This was previously accomplished by: + + ``` + nix profile upgrade '.*' + ``` + + With the introduction of the `--all` flag, this now becomes more straightforward: + + ``` + nix profile upgrade --all + ``` + +- Visual clutter in `--debugger` is reduced [#9919](https://github.com/NixOS/nix/pull/9919) + + Before: + ``` + info: breakpoint reached + + + Starting REPL to allow you to inspect the current state of the evaluator. + + Welcome to Nix 2.20.0pre20231222_dirty. Type :? for help. + + nix-repl> :continue + error: uh oh + + + Starting REPL to allow you to inspect the current state of the evaluator. + + Welcome to Nix 2.20.0pre20231222_dirty. Type :? for help. + + nix-repl> + ``` + + After: + + ``` + info: breakpoint reached + + Nix 2.20.0pre20231222_dirty debugger + Type :? for help. + nix-repl> :continue + error: uh oh + + nix-repl> + ``` + +- Cycle detection in `nix repl` is simpler and more reliable [#8672](https://github.com/NixOS/nix/issues/8672) [#9926](https://github.com/NixOS/nix/pull/9926) + + The cycle detection in `nix repl`, `nix eval`, `builtins.trace`, and everywhere + else values are printed is now simpler and matches the cycle detection in + `nix-instantiate --eval` output. + + Before: + + ``` + nix eval --expr 'let self = { inherit self; }; in self' + { self = { self = «repeated»; }; } + ``` + + After: + + ``` + { self = «repeated»; } + ``` + +- In the debugger, `while evaluating the attribute` errors now include position information [#9915](https://github.com/NixOS/nix/pull/9915) + + Before: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + 0x600001522598 + ``` + + After: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + /nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27 + + 131| + 132| bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs; + | ^ + 133| in + ``` + +- Stack size is increased on macOS [#9860](https://github.com/NixOS/nix/pull/9860) + + Previously, Nix would set the stack size to 64MiB on Linux, but would leave the + stack size set to the default (approximately 8KiB) on macOS. Now, the stack + size is correctly set to 64MiB on macOS as well, which should reduce stack + overflow segfaults in deeply-recursive Nix expressions. + diff --git a/doc/manual/src/release-notes/rl-2.22.md b/doc/manual/src/release-notes/rl-2.22.md new file mode 100644 index 000000000..c78d3d692 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.22.md @@ -0,0 +1,21 @@ +# Release 2.22.0 (2024-04-23) + +### Significant changes + +- Remove experimental repl-flake [#10103](https://github.com/NixOS/nix/issues/10103) [#10299](https://github.com/NixOS/nix/pull/10299) + + The `repl-flake` experimental feature has been removed. The `nix repl` command now works like the rest of the new CLI in that `nix repl {path}` now tries to load a flake at `{path}` (or fails if the `flakes` experimental feature isn't enabled). + +### Other changes + +- `nix eval` prints derivations as `.drv` paths [#10200](https://github.com/NixOS/nix/pull/10200) + + `nix eval` will now print derivations as their `.drv` paths, rather than as + attribute sets. This makes commands like `nix eval nixpkgs#bash` terminate + instead of infinitely looping into recursive self-referential attributes: + + ```ShellSession + $ nix eval nixpkgs#bash + «derivation /nix/store/m32cbgbd598f4w299g0hwyv7gbw6rqcg-bash-5.2p26.drv» + ``` + diff --git a/doc/manual/src/release-notes/rl-2.23.md b/doc/manual/src/release-notes/rl-2.23.md new file mode 100644 index 000000000..ac842fdc0 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.23.md @@ -0,0 +1,102 @@ +# Release 2.23.0 (2024-06-03) + +- New builtin: `builtins.warn` [#306026](https://github.com/NixOS/nix/issues/306026) [#10592](https://github.com/NixOS/nix/pull/10592) + + `builtins.warn` behaves like `builtins.trace "warning: ${msg}"`, has an accurate log level, and is controlled by the options + [`debugger-on-trace`](@docroot@/command-ref/conf-file.md#conf-debugger-on-trace), + [`debugger-on-warn`](@docroot@/command-ref/conf-file.md#conf-debugger-on-warn) and + [`abort-on-warn`](@docroot@/command-ref/conf-file.md#conf-abort-on-warn). + +- Make `nix build --keep-going` consistent with `nix-build --keep-going` + + This means that if e.g. multiple fixed-output derivations fail to + build, all hash mismatches are displayed. + +- Modify `nix derivation {add,show}` JSON format [#9866](https://github.com/NixOS/nix/issues/9866) [#10722](https://github.com/NixOS/nix/pull/10722) + + The JSON format for derivations has been slightly revised to better conform to our [JSON guidelines](@docroot@/development/cli-guideline.md#returning-future-proof-json). + In particular, the hash algorithm and content addressing method of content-addresed derivation outputs are now separated into two fields `hashAlgo` and `method`, + rather than one field with an arcane `:`-separated format. + + This JSON format is only used by the experimental `nix derivation` family of commands, at this time. + Future revisions are expected as the JSON format is still not entirely in compliance even after these changes. + +- Warn on unknown settings anywhere in the command line [#10701](https://github.com/NixOS/nix/pull/10701) + + All `nix` commands will now properly warn when an unknown option is specified anywhere in the command line. + + Before: + + ```console + $ nix-instantiate --option foobar baz --expr '{}' + warning: unknown setting 'foobar' + $ nix-instantiate '{}' --option foobar baz --expr + $ nix eval --expr '{}' --option foobar baz + { } + ``` + + After: + + ```console + $ nix-instantiate --option foobar baz --expr '{}' + warning: unknown setting 'foobar' + $ nix-instantiate '{}' --option foobar baz --expr + warning: unknown setting 'foobar' + $ nix eval --expr '{}' --option foobar baz + warning: unknown setting 'foobar' + { } + ``` + +- `nix env shell` is the new `nix shell`, and `nix shell` remains an accepted alias [#10504](https://github.com/NixOS/nix/issues/10504) [#10807](https://github.com/NixOS/nix/pull/10807) + + This is part of an effort to bring more structure to the CLI subcommands. + + `nix env` will be about the process environment. + Future commands may include `nix env run` and `nix env print-env`. + + It is also somewhat analogous to the [planned](https://github.com/NixOS/nix/issues/10504) `nix dev shell` (currently `nix develop`), which is less about environment variables, and more about running a development shell, which is a more powerful command, but also requires more setup. + +- Flake operations that expect derivations now print the failing value and its type [#10778](https://github.com/NixOS/nix/pull/10778) + + In errors like `flake output attribute 'nixosConfigurations.yuki.config' is not a derivation or path`, the message now includes the failing value and type. + + Before: + + ``` + error: flake output attribute 'nixosConfigurations.yuki.config' is not a derivation or path + ```` + + After: + + ``` + error: expected flake output attribute 'nixosConfigurations.yuki.config' to be a derivation or path but found a set: { appstream = «thunk»; assertions = «thunk»; boot = { bcache = «thunk»; binfmt = «thunk»; binfmtMiscRegistrations = «thunk»; blacklistedKernelModules = «thunk»; bootMount = «thunk»; bootspec = «thunk»; cleanTmpDir = «thunk»; consoleLogLevel = «thunk»; «43 attributes elided» }; «48 attributes elided» } + ``` + +- `fetchTree` now fetches Git repositories shallowly by default [#10028](https://github.com/NixOS/nix/pull/10028) + + `builtins.fetchTree` now clones Git repositories shallowly by default, which reduces network traffic and disk usage significantly in many cases. + + Previously, the default behavior was to clone the full history of a specific tag or branch (e.g. `ref`) and only afterwards extract the files of one specific revision. + + From now on, the `ref` and `allRefs` arguments will be ignored, except if shallow cloning is disabled by setting `shallow = false`. + + The defaults for `builtins.fetchGit` remain unchanged. Here, shallow cloning has to be enabled manually by passing `shallow = true`. + +- Store object info JSON format now uses `null` rather than omitting fields [#9995](https://github.com/NixOS/nix/pull/9995) + + The [store object info JSON format](@docroot@/protocols/json/store-object-info.md), used for e.g. `nix path-info`, no longer omits fields to indicate absent information, but instead includes the fields with a `null` value. + For example, `"ca": null` is used to to indicate a store object that isn't content-addressed rather than omitting the `ca` field entirely. + This makes records of this sort more self-describing, and easier to consume programmatically. + + We will follow this design principle going forward; + the [JSON guidelines](@docroot@/development/json-guideline.md) in the contributing section have been updated accordingly. + +- Large path warnings [#10661](https://github.com/NixOS/nix/pull/10661) + + Nix can now warn when evaluation of a Nix expression causes a large + path to be copied to the Nix store. The threshold for this warning can + be configured using [the `warn-large-path-threshold` + setting](@docroot@/command-ref/conf-file.md#warn-large-path-threshold), + e.g. `--warn-large-path-threshold 100M` will warn about paths larger + than 100 MiB. + diff --git a/doc/manual/src/release-notes/rl-2.24.md b/doc/manual/src/release-notes/rl-2.24.md new file mode 100644 index 000000000..5bcc1d79c --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.24.md @@ -0,0 +1,324 @@ +# Release 2.24.0 (2024-07-31) + +### Significant changes + +- Harden user sandboxing + + The build directory has been hardened against interference with the outside world by nesting it inside another directory owned by (and only readable by) the daemon user. + + This is a low severity security fix, [CVE-2024-38531](https://www.cve.org/CVERecord?id=CVE-2024-38531). + + Credit: [**@alois31**](https://github.com/alois31), [**Linus Heckemann (@lheckemann)**](https://github.com/lheckemann) + Co-authors: [**@edolstra**](https://github.com/edolstra) + +- `nix-shell ` looks for `shell.nix` [#496](https://github.com/NixOS/nix/issues/496) [#2279](https://github.com/NixOS/nix/issues/2279) [#4529](https://github.com/NixOS/nix/issues/4529) [#5431](https://github.com/NixOS/nix/issues/5431) [#11053](https://github.com/NixOS/nix/issues/11053) [#11057](https://github.com/NixOS/nix/pull/11057) + + `nix-shell $x` now looks for `$x/shell.nix` when `$x` resolves to a directory. + + Although this might be seen as a breaking change, its primarily interactive usage makes it a minor issue. + This adjustment addresses a commonly reported problem. + + This also applies to `nix-shell` shebang scripts. Consider the following example: + + ```shell + #!/usr/bin/env nix-shell + #!nix-shell -i bash + ``` + + This will now load `shell.nix` from the script's directory, if it exists; `default.nix` otherwise. + + The old behavior can be opted into by setting the option [`nix-shell-always-looks-for-shell-nix`](@docroot@/command-ref/conf-file.md#conf-nix-shell-always-looks-for-shell-nix) to `false`. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + +- `nix-repl`'s `:doc` shows documentation comments [#3904](https://github.com/NixOS/nix/issues/3904) [#10771](https://github.com/NixOS/nix/issues/10771) [#1652](https://github.com/NixOS/nix/pull/1652) [#9054](https://github.com/NixOS/nix/pull/9054) [#11072](https://github.com/NixOS/nix/pull/11072) + + `nix repl` has a `:doc` command that previously only rendered documentation for internally defined functions. + This feature has been extended to also render function documentation comments, in accordance with [RFC 145]. + + Example: + + ``` + nix-repl> :doc lib.toFunction + Function toFunction + … defined at /home/user/h/nixpkgs/lib/trivial.nix:1072:5 + + Turns any non-callable values into constant functions. Returns + callable values as is. + + Inputs + + v + + : Any value + + Examples + + :::{.example} + + ## lib.trivial.toFunction usage example + + | nix-repl> lib.toFunction 1 2 + | 1 + | + | nix-repl> lib.toFunction (x: x + 1) 2 + | 3 + + ::: + ``` + + Known limitations: + - It does not render documentation for "formals", such as `{ /** the value to return */ x, ... }: x`. + - Some extensions to markdown are not yet supported, as you can see in the example above. + + We'd like to acknowledge [Yingchi Long (@inclyc)](https://github.com/inclyc) for proposing a proof of concept for this functionality in [#9054](https://github.com/NixOS/nix/pull/9054), as well as [@sternenseemann](https://github.com/sternenseemann) and [Johannes Kirschbauer (@hsjobeki)](https://github.com/hsjobeki) for their contributions, proposals, and their work on [RFC 145]. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + + [RFC 145]: https://github.com/NixOS/rfcs/pull/145 + +### Other changes + +- Solve `cached failure of attribute X` [#9165](https://github.com/NixOS/nix/issues/9165) [#10513](https://github.com/NixOS/nix/issues/10513) [#10564](https://github.com/NixOS/nix/pull/10564) + + This eliminates all "cached failure of attribute X" messages by forcing evaluation of the original value when needed to show the exception to the user. This enhancement improves error reporting by providing the underlying message and stack trace. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Run the flake regressions test suite [#10603](https://github.com/NixOS/nix/pull/10603) + + This update introduces a GitHub action to run a subset of the [flake regressions test suite](https://github.com/NixOS/flake-regressions), which includes 259 flakes with their expected evaluation results. Currently, the action runs the first 25 flakes due to the full test suite's extensive runtime. A manually triggered action may be implemented later to run the entire test suite. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Support unit prefixes in configuration settings [#10668](https://github.com/NixOS/nix/pull/10668) + + Configuration settings in Nix now support unit prefixes, allowing for more intuitive and readable configurations. For example, you can now specify [`--min-free 1G`](@docroot@/command-ref/opt-common.md#opt-min-free) to set the minimum free space to 1 gigabyte. + + This enhancement was extracted from [#7851](https://github.com/NixOS/nix/pull/7851) and is also useful for PR [#10661](https://github.com/NixOS/nix/pull/10661). + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- `nix build`: show all FOD errors with `--keep-going` [#10734](https://github.com/NixOS/nix/pull/10734) + + The [`nix build`](@docroot@/command-ref/new-cli/nix3-build.md) command has been updated to improve the behavior of the [`--keep-going`] flag. Now, when `--keep-going` is used, all hash-mismatch errors of failing fixed-output derivations (FODs) are displayed, similar to the behavior for other build failures. This enhancement ensures that all relevant build errors are shown, making it easier for users to update multiple derivations at once or to diagnose and fix issues. + + Author: [**Jörg Thalheim (@Mic92)**](https://github.com/Mic92), [**Maximilian Bosch (@Ma27)**](https://github.com/Ma27) + + [`--keep-going`](@docroot@/command-ref/opt-common.md#opt-keep-going) + +- Build with Meson [#2503](https://github.com/NixOS/nix/issues/2503) [#10378](https://github.com/NixOS/nix/pull/10378) [#10855](https://github.com/NixOS/nix/pull/10855) [#10904](https://github.com/NixOS/nix/pull/10904) [#10908](https://github.com/NixOS/nix/pull/10908) [#10914](https://github.com/NixOS/nix/pull/10914) [#10933](https://github.com/NixOS/nix/pull/10933) [#10936](https://github.com/NixOS/nix/pull/10936) [#10954](https://github.com/NixOS/nix/pull/10954) [#10955](https://github.com/NixOS/nix/pull/10955) [#10963](https://github.com/NixOS/nix/pull/10963) [#10967](https://github.com/NixOS/nix/pull/10967) [#10973](https://github.com/NixOS/nix/pull/10973) [#11034](https://github.com/NixOS/nix/pull/11034) [#11054](https://github.com/NixOS/nix/pull/11054) [#11055](https://github.com/NixOS/nix/pull/11055) [#11060](https://github.com/NixOS/nix/pull/11060) [#11064](https://github.com/NixOS/nix/pull/11064) [#11155](https://github.com/NixOS/nix/pull/11155) + + These changes aim to replace the use of autotools and `make` with Meson for building various components of Nix. Additionally, each library is built in its own derivation, leveraging Meson's "subprojects" feature to allow a single development shell for building all libraries while also supporting separate builds. This approach aims to improve productivity and build modularity, compared to both make and a monolithic Meson-based derivation. + + Special thanks to everyone who has contributed to the Meson port, particularly [**@p01arst0rm**](https://github.com/p01arst0rm) and [**@Qyriad**](https://github.com/Qyriad). + + Authors: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314), [**Tom Bereknyei**](https://github.com/tomberek), [**Théophane Hufschmitt (@thufschmitt)**](https://github.com/thufschmitt), [**Valentin Gagarin (@fricklerhandwerk)**](https://github.com/fricklerhandwerk), [**Robert Hensing (@roberth)**](https://github.com/roberth) + Co-authors: [**@p01arst0rm**](https://github.com/p01arst0rm), [**@Qyriad**](https://github.com/Qyriad) + +- Evaluation cache: fix cache regressions [#10570](https://github.com/NixOS/nix/issues/10570) [#11086](https://github.com/NixOS/nix/pull/11086) + + This update addresses two bugs in the evaluation cache system: + + 1. Regression in #10570: The evaluation cache was not being persisted in `nix develop`. + 2. Nix could sometimes try to commit the evaluation cache SQLite transaction without there being an active transaction, resulting in non-error errors being printed. + + Author: [**Lexi Mattick (@kognise)**](https://github.com/kognise) + +- Introduce `libnixflake` [#9063](https://github.com/NixOS/nix/pull/9063) + + A new library, `libnixflake`, has been introduced to better separate the Flakes layer within Nix. This change refactors the codebase to encapsulate Flakes-specific functionality within its own library. + + See the commits in the pull request for detailed changes, with the only significant code modifications happening in the initial commit. + + This change was alluded to in [RFC 134](https://github.com/nixos/rfcs/blob/master/rfcs/0134-nix-store-layer.md) and is a step towards a more modular and maintainable codebase. + + Author: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314) + +- CLI options `--arg-from-file` and `--arg-from-stdin` [#9913](https://github.com/NixOS/nix/pull/9913) + +- The `--debugger` now prints source location information, instead of the + pointers of source location information. Before: + + ``` + nix-repl> :bt + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + 0x600001522598 + ``` + + After: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + /nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27 + + 131| + 132| bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs; + | ^ + 133| in + ``` + +- Stop vendoring `toml11` + + We don't apply any patches to it, and vendoring it locks users into + bugs (it hasn't been updated since its introduction in late 2021). + + Author: [**Winter (@winterqt)**](https://github.com/winterqt) + +- Rename hash format `base32` to `nix32` [#8678](https://github.com/NixOS/nix/pull/8678) + + Hash format `base32` was renamed to `nix32` since it used a special nix-specific character set for + [Base32](https://en.wikipedia.org/wiki/Base32). + + **Deprecation**: Use `nix32` instead of `base32` as `toHashFormat` + + For the builtin `convertHash`, the `toHashFormat` parameter now accepts the same hash formats as the `--to`/`--from` + parameters of the `nix hash conert` command: `"base16"`, `"nix32"`, `"base64"`, and `"sri"`. The former `"base32"` value + remains as a deprecated alias for `"nix32"`. Please convert your code from: + + ```nix + builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base32";} + ``` + + to + + ```nix + builtins.convertHash { inherit hash hashAlgo; toHashFormat = "nix32";} + ``` + +- Add `pipe-operators` experimental feature [#11131](https://github.com/NixOS/nix/pull/11131) + + This is a draft implementation of [RFC 0148](https://github.com/NixOS/rfcs/pull/148). + + The `pipe-operators` experimental feature adds [`<|` and `|>` operators][pipe operators] to the Nix language. + *a* `|>` *b* is equivalent to the function application *b* *a*, and + *a* `<|` *b* is equivalent to the function application *a* *b*. + + For example: + + ``` + nix-repl> 1 |> builtins.add 2 |> builtins.mul 3 + 9 + + nix-repl> builtins.add 1 <| builtins.mul 2 <| 3 + 7 + ``` + + `<|` and `|>` are right and left associative, respectively, and have lower precedence than any other operator. + These properties may change in future releases. + + See [the RFC](https://github.com/NixOS/rfcs/pull/148) for more examples and rationale. + + [pipe operators]: @docroot@/language/operators.md#pipe-operators + +- `nix-shell` shebang uses relative path [#4232](https://github.com/NixOS/nix/issues/4232) [#5088](https://github.com/NixOS/nix/pull/5088) [#11058](https://github.com/NixOS/nix/pull/11058) + + + Relative [path](@docroot@/language/types.md#type-path) literals in `nix-shell` shebang scripts' options are now resolved relative to the [script's location](@docroot@/glossary.md?highlight=base%20directory#gloss-base-directory). + Previously they were resolved relative to the current working directory. + + For example, consider the following script in `~/myproject/say-hi`: + + ```shell + #!/usr/bin/env nix-shell + #!nix-shell --expr 'import ./shell.nix' + #!nix-shell --arg toolset './greeting-tools.nix' + #!nix-shell -i bash + hello + ``` + + Older versions of `nix-shell` would resolve `shell.nix` relative to the current working directory, such as the user's home directory in this example: + + ```console + [hostname:~]$ ./myproject/say-hi + error: + … while calling the 'import' builtin + at «string»:1:2: + 1| (import ./shell.nix) + | ^ + + error: path '/home/user/shell.nix' does not exist + ``` + + Since this release, `nix-shell` resolves `shell.nix` relative to the script's location, and `~/myproject/shell.nix` is used. + + ```console + $ ./myproject/say-hi + Hello, world! + ``` + + **Opt-out** + + This is technically a breaking change, so we have added an option so you can adapt independently of your Nix update. + The old behavior can be opted into by setting the option [`nix-shell-shebang-arguments-relative-to-script`](@docroot@/command-ref/conf-file.md#conf-nix-shell-shebang-arguments-relative-to-script) to `false`. + This option will be removed in a future release. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + +- Improve handling of tarballs that don't consist of a single top-level directory [#11195](https://github.com/NixOS/nix/pull/11195) + + In previous Nix releases, the tarball fetcher (used by `builtins.fetchTarball`) erroneously merged top-level directories into a single directory, and silently discarded top-level files that are not directories. This is no longer the case. The new behaviour is that *only* if the tarball consists of a single directory, the top-level path component of the files in the tarball is removed (similar to `tar`'s `--strip-components=1`). + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Improve handling of tarballs that don't consist of a single top-level directory [#11195](https://github.com/NixOS/nix/pull/11195) + + In previous Nix releases, the tarball fetcher (used by `builtins.fetchTarball`) erroneously merged top-level directories into a single directory, and silently discarded top-level files that are not directories. This is no longer the case. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Setting to warn about large paths [#10778](https://github.com/NixOS/nix/pull/10778) + + Nix can now warn when evaluation of a Nix expression causes a large + path to be copied to the Nix store. The threshold for this warning can + be configured using the `warn-large-path-threshold` setting, + e.g. `--warn-large-path-threshold 100M`. + + +# Contributors + +This release was made possible by the following 43 contributors: + +- Andreas Rammhold [**(@andir)**](https://github.com/andir) +- Andrew Marshall [**(@amarshall)**](https://github.com/amarshall) +- Brian McKenna [**(@puffnfresh)**](https://github.com/puffnfresh) +- Cameron [**(@SkamDart)**](https://github.com/SkamDart) +- Cole Helbling [**(@cole-h)**](https://github.com/cole-h) +- Corbin Simpson [**(@MostAwesomeDude)**](https://github.com/MostAwesomeDude) +- Eelco Dolstra [**(@edolstra)**](https://github.com/edolstra) +- Emily [**(@emilazy)**](https://github.com/emilazy) +- Enno Richter [**(@elohmeier)**](https://github.com/elohmeier) +- Farid Zakaria [**(@fzakaria)**](https://github.com/fzakaria) +- HaeNoe [**(@haenoe)**](https://github.com/haenoe) +- Hamir Mahal [**(@hamirmahal)**](https://github.com/hamirmahal) +- Harmen [**(@alicebob)**](https://github.com/alicebob) +- Ivan Trubach [**(@tie)**](https://github.com/tie) +- Jared Baur [**(@jmbaur)**](https://github.com/jmbaur) +- John Ericson [**(@Ericson2314)**](https://github.com/Ericson2314) +- Jonathan De Troye [**(@detroyejr)**](https://github.com/detroyejr) +- Jörg Thalheim [**(@Mic92)**](https://github.com/Mic92) +- Klemens Nanni [**(@klemensn)**](https://github.com/klemensn) +- Las Safin [**(@L-as)**](https://github.com/L-as) +- Lexi Mattick [**(@kognise)**](https://github.com/kognise) +- Matthew Bauer [**(@matthewbauer)**](https://github.com/matthewbauer) +- Max “Goldstein†Siling [**(@GoldsteinE)**](https://github.com/GoldsteinE) +- Mingye Wang [**(@Artoria2e5)**](https://github.com/Artoria2e5) +- Philip Taron [**(@philiptaron)**](https://github.com/philiptaron) +- Pierre Bourdon [**(@delroth)**](https://github.com/delroth) +- Pino Toscano [**(@pinotree)**](https://github.com/pinotree) +- RTUnreal [**(@RTUnreal)**](https://github.com/RTUnreal) +- Robert Hensing [**(@roberth)**](https://github.com/roberth) +- Romain Neil [**(@romain-neil)**](https://github.com/romain-neil) +- Ryan Hendrickson [**(@rhendric)**](https://github.com/rhendric) +- Sergei Trofimovich [**(@trofi)**](https://github.com/trofi) +- Shogo Takata [**(@pineapplehunter)**](https://github.com/pineapplehunter) +- Siddhant Kumar [**(@siddhantk232)**](https://github.com/siddhantk232) +- Silvan Mosberger [**(@infinisil)**](https://github.com/infinisil) +- Théophane Hufschmitt [**(@thufschmitt)**](https://github.com/thufschmitt) +- Valentin Gagarin [**(@fricklerhandwerk)**](https://github.com/fricklerhandwerk) +- Winter [**(@winterqt)**](https://github.com/winterqt) +- jade [**(@lf-)**](https://github.com/lf-) +- kirillrdy [**(@kirillrdy)**](https://github.com/kirillrdy) +- pennae [**(@pennae)**](https://github.com/pennae) +- poweredbypie [**(@poweredbypie)**](https://github.com/poweredbypie) +- tomberek [**(@tomberek)**](https://github.com/tomberek) diff --git a/doc/manual/src/release-notes/rl-2.4.md b/doc/manual/src/release-notes/rl-2.4.md index 8b566fc7b..1201e53b6 100644 --- a/doc/manual/src/release-notes/rl-2.4.md +++ b/doc/manual/src/release-notes/rl-2.4.md @@ -23,7 +23,7 @@ more than 2800 commits from 195 contributors since release 2.3. * The **`nix` command** has seen a lot of work and is now almost at feature parity with the old command-line interface (the `nix-*` commands). It aims to be [more modern, consistent and pleasant to - use](../contributing/cli-guideline.md) than the old CLI. It is still + use](../development/cli-guideline.md) than the old CLI. It is still marked as experimental but its interface should not change much anymore in future releases. diff --git a/doc/manual/src/store/file-system-object/content-address.md b/doc/manual/src/store/file-system-object/content-address.md new file mode 100644 index 000000000..410d7fb7c --- /dev/null +++ b/doc/manual/src/store/file-system-object/content-address.md @@ -0,0 +1,85 @@ +# Content-Addressing File System Objects + +For many operations, Nix needs to calculate [a content addresses](@docroot@/glossary.md#gloss-content-address) of [a file system object][file system object]. +Usually this is needed as part of +[content addressing store objects](../store-object/content-address.md), +since store objects always have a root file system object. +But some command-line utilities also just work on "raw" file system objects, not part of any store object. + +Every content addressing scheme Nix uses ultimately involves feeding data into a [hash function](https://en.wikipedia.org/wiki/Hash_function), and getting back an opaque fixed-size digest which is deemed a content address. +The various *methods* of content addressing thus differ in how abstract data (in this case, a file system object and its descendents) are fed into the hash function. + +## Serialising File System Objects { #serial } + +The simplest method is to serialise the entire file system object tree into a single binary string, and then hash that binary string, yielding the content address. +In this section we describe the currently-supported methods of serialising file system objects. + +### Flat { #serial-flat } + +A single file object can just be hashed by its contents. +This is not enough information to encode the fact that the file system object is a file, +but if we *already* know that the FSO is a single non-executable file by other means, it is sufficient. + +Because the hashed data is just the raw file, as is, this choice is good for compatibility with other systems. +For example, Unix commands like `sha256sum` or `sha1sum` will produce hashes for single files that match this. + +### Nix Archive (NAR) { #serial-nix-archive } + +For the other cases of [file system objects][file system object], especially directories with arbitrary descendents, we need a more complex serialisation format. +Examples of such serialisations are the ZIP and TAR file formats. +However, for our purposes these formats have two problems: + +- They do not have a canonical serialisation, meaning that given an FSO, there can +be many different serialisations. + For instance, TAR files can have variable amounts of padding between archive members; + and some archive formats leave the order of directory entries undefined. + This would be bad because we use serialisation to compute cryptographic hashes over file system objects, and for those hashes to be useful as a content address or for integrity checking, uniqueness is crucial. + Otherwise, correct hashes would report false mismatches, and the store would fail to find the content. + +- They store more information than we have in our notion of FSOs, such as time stamps. + This can cause FSOs that Nix should consider equal to hash to different values on different machines, just because the dates differ. + +- As a practical consideration, the TAR format is the only truly universal format in the Unix environment. + It has many problems, such as an inability to deal with long file names and files larger than 2^33 bytes. + Current implementations such as GNU Tar work around these limitations in various ways. + +For these reasons, Nix has its very own archive format—the Nix Archive (NAR) format, +which is carefully designed to avoid the problems described above. + +The exact specification of the Nix Archive format is in `protocols/nix-archive.md` + +## Content addressing File System Objects beyond a single serialisation pass + +Serialising the entire tree and then hashing that binary string is not the only option for content addressing, however. +Another technique is that of a [Merkle graph](https://en.wikipedia.org/wiki/Merkle_tree), where previously computed hashes are included in subsequent byte strings to be hashed. + +In particular, the Merkle graphs can match the original graph structure of file system objects: +we can first hash (serialised) child file system objects, and then hash parent objects using the hashes of their children in the serialisation (to be hashed) of the parent file system objects. + +Currently, there is one such Merkle DAG content addressing method supported. + +### Git ([experimental][xp-feature-git-hashing]) { #git } + +> **Warning** +> +> This method is part of the [`git-hashing`][xp-feature-git-hashing] experimental feature. + +Git's file system model is very close to Nix's, and so Git's content addressing method is a pretty good fit. +Just as with regular Git, files and symlinks are hashed as git "blobs", and directories are hashed as git "trees". + +However, one difference between Nix's and Git's file system model needs special treatment. +Plain files, executable files, and symlinks are not differentiated as distinctly addressable objects, but by their context: by the directory entry that refers to them. +That means so long as the root object is a directory, there is no problem: +every non-directory object is owned by a parent directory, and the entry that refers to it provides the missing information. +However, if the root object is not a directory, then we have no way of knowing which one of an executable file, non-executable file, or symlink it is supposed to be. + +In response to this, we have decided to treat a bare file as non-executable file. +This is similar to do what we do with [flat serialisation](#serial-flat), which also lacks this information. +To avoid an address collision, attempts to hash a bare executable file or symlink will result in an error (just as would happen for flat serialisation also). +Thus, Git can encode some, but not all of Nix's "File System Objects", and this sort of content-addressing is likewise partial. + +In the future, we may support a Git-like hash for such file system objects, or we may adopt another Merkle DAG format which is capable of representing all Nix file system objects. + +[file system object]: ../file-system-object.md +[store object]: ../store-object.md +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/doc/manual/src/store/store-object/content-address.md b/doc/manual/src/store/store-object/content-address.md new file mode 100644 index 000000000..02dce2836 --- /dev/null +++ b/doc/manual/src/store/store-object/content-address.md @@ -0,0 +1,95 @@ +# Content-Addressing Store Objects + +Just [like][fso-ca] [File System Objects][File System Object], +[Store Objects][Store Object] can also be [content-addressed](@docroot@/glossary.md#gloss-content-addressed), +unless they are [input-addressed](@docroot@/glossary.md#gloss-input-addressed-store-object). + +For store objects, the content address we produce will take the form of a [Store Path] rather than regular hash. +In particular, the content-addressing scheme will ensure that the digest of the store path is solely computed from the + +- file system object graph (the root one and its children, if it has any) +- references +- [store directory](../store-path.md#store-directory) +- name + +of the store object, and not any other information, which would not be an intrinsic property of that store object. + +For the full specification of the algorithms involved, see the [specification of store path digests][sp-spec]. + +[File System Object]: ../file-system-object.md +[Store Object]: ../store-object.md +[Store Path]: ../store-path.md + +## Content addressing each part of a store object + +### File System Objects + +With all currently supported store object content addressing methods, the file system object is always [content-addressed][fso-ca] first, and then that hash is incorporated into content address computation for the store object. + +### References + +With all currently supported store object content addressing methods, +other objects are referred to by their regular (string-encoded-) [store paths][Store Path]. + +Self-references however cannot be referred to by their path, because we are in the midst of describing how to compute that path! + +> The alternative would require finding as hash function fixed point, i.e. the solution to an equation in the form +> ``` +> digest = hash(..... || digest || ....) +> ``` +> which is computationally infeasible. +> As far as we know, this is equivalent to finding a hash collision. + +Instead we just have a "has self reference" boolean, which will end up affecting the digest. + +### Name and Store Directory + +These two items affect the digest in a way that is standard for store path digest computations and not specific to content-addressing. +Consult the [specification of store path digests][sp-spec] for further details. + +## Content addressing Methods + +For historical reasons, we don't support all features in all combinations. +Each currently supported method of content addressing chooses a single method of file system object hashing, and may offer some restrictions on references. +The names and store directories are unrestricted however. + +### Flat { #method-flat } + +This uses the corresponding [Flat](../file-system-object/content-address.md#serial-flat) method of file system object content addressing. + +References are not supported: store objects with flat hashing *and* references can not be created. + +### Text { #method-text } + +This also uses the corresponding [Flat](../file-system-object/content-address.md#serial-flat) method of file system object content addressing. + +References to other store objects are supported, but self references are not. + +This is the only store-object content-addressing method that is not named identically with a corresponding file system object method. +It is somewhat obscure, mainly used for "drv files" +(derivations serialized as store objects in their ["ATerm" file format](@docroot@/protocols/derivation-aterm.md)). +Prefer another method if possible. + +### Nix Archive { #method-nix-archive } + +This uses the corresponding [Nix Archive](../file-system-object/content-address.md#serial-nix-archive) method of file system object content addressing. + +References (to other store objects and self references alike) are supported so long as the hash algorithm is SHA-256, but not (neither kind) otherwise. + +### Git { #method-git } + +> **Warning** +> +> This method is part of the [`git-hashing`][xp-feature-git-hashing] experimental feature. + +This uses the corresponding [Git](../file-system-object/content-address.md#serial-git) method of file system object content addressing. + +References are not supported. + +Only SHA-1 is supported at this time. +If [SHA-256-based Git](https://git-scm.com/docs/hash-function-transition) +becomes more widespread, this restriction will be revisited. + +[fso-ca]: ../file-system-object/content-address.md +[sp-spec]: @docroot@/protocols/store-path.md +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/doc/manual/src/store/store-path.md b/doc/manual/src/store/store-path.md index b5ad0c654..beec2389b 100644 --- a/doc/manual/src/store/store-path.md +++ b/doc/manual/src/store/store-path.md @@ -1,5 +1,11 @@ # Store Path +> **Example** +> +> `/nix/store/a040m110amc4h71lds2jmr8qrkj2jhxd-git-2.38.1` +> +> A rendered store path + Nix implements references to [store objects](./index.md#store-object) as *store paths*. Think of a store path as an [opaque], [unique identifier]: @@ -37,6 +43,10 @@ A store path is rendered to a file system path as the concatenation of > store directory digest name > ``` +Exactly how the digest is calculated depends on the type of store path. +Store path digests are *supposed* to be opaque, and so for most operations, it is not necessary to know the details. +That said, the manual has a full [specification of store path digests](@docroot@/protocols/store-path.md). + ## Store Directory Every [Nix store](./index.md) has a store directory. @@ -46,7 +56,7 @@ But if the store has a file system representation, the store directory contains [file system objects]: ./file-system-object.md -This means a store path is not just derived from the referenced store object itself, but depends on the store the store object is in. +This means a store path is not just derived from the referenced store object itself, but depends on the store that the store object is in. > **Note** > diff --git a/flake.lock b/flake.lock index bb2e400c0..2ac413a69 100644 --- a/flake.lock +++ b/flake.lock @@ -3,11 +3,11 @@ "flake-compat": { "flake": false, "locked": { - "lastModified": 1673956053, - "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", "owner": "edolstra", "repo": "flake-compat", - "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", "type": "github" }, "original": { @@ -16,38 +16,100 @@ "type": "github" } }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1719994518, + "narHash": "sha256-pQMhCCHyQGRzdfAkdJ4cIWiw+JNuWsTX7f0ZYSyz0VY=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "9227223f6d922fee3c7b190b2cc238a99527bbb7", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "git-hooks-nix": { + "inputs": { + "flake-compat": [], + "gitignore": [], + "nixpkgs": [ + "nixpkgs" + ], + "nixpkgs-stable": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1721042469, + "narHash": "sha256-6FPUl7HVtvRHCCBQne7Ylp4p+dpP3P/OYuzjztZ4s70=", + "owner": "cachix", + "repo": "git-hooks.nix", + "rev": "f451c19376071a90d8c58ab1a953c6e9840527fd", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "git-hooks.nix", + "type": "github" + } + }, "libgit2": { "flake": false, "locked": { - "lastModified": 1697646580, - "narHash": "sha256-oX4Z3S9WtJlwvj0uH9HlYcWv+x1hqp8mhXl7HsLu2f0=", + "lastModified": 1715853528, + "narHash": "sha256-J2rCxTecyLbbDdsyBWn9w7r3pbKRMkI9E7RvRgAqBdY=", "owner": "libgit2", "repo": "libgit2", - "rev": "45fd9ed7ae1a9b74b957ef4f337bc3c8b3df01b5", + "rev": "36f7e21ad757a3dacc58cf7944329da6bc1d6e96", "type": "github" }, "original": { "owner": "libgit2", + "ref": "v1.8.1", "repo": "libgit2", "type": "github" } }, "nixpkgs": { "locked": { - "lastModified": 1709083642, - "narHash": "sha256-7kkJQd4rZ+vFrzWu8sTRtta5D1kBG0LSRYAfhtmMlSo=", + "lastModified": 1721548954, + "narHash": "sha256-7cCC8+Tdq1+3OPyc3+gVo9dzUNkNIQfwSDJ2HSi2u3o=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "b550fe4b4776908ac2a861124307045f8e717c8e", + "rev": "63d37ccd2d178d54e7fb691d7ec76000740ea24a", "type": "github" }, "original": { "owner": "NixOS", - "ref": "release-23.11", + "ref": "nixos-24.05", "repo": "nixpkgs", "type": "github" } }, + "nixpkgs-23-11": { + "locked": { + "lastModified": 1717159533, + "narHash": "sha256-oamiKNfr2MS6yH64rUn99mIZjc45nGJlj9eGth/3Xuw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446", + "type": "github" + } + }, "nixpkgs-regression": { "locked": { "lastModified": 1643052045, @@ -67,8 +129,11 @@ "root": { "inputs": { "flake-compat": "flake-compat", + "flake-parts": "flake-parts", + "git-hooks-nix": "git-hooks-nix", "libgit2": "libgit2", "nixpkgs": "nixpkgs", + "nixpkgs-23-11": "nixpkgs-23-11", "nixpkgs-regression": "nixpkgs-regression" } } diff --git a/flake.nix b/flake.nix index 42aaace67..9e8592e3a 100644 --- a/flake.nix +++ b/flake.nix @@ -1,18 +1,28 @@ { description = "The purely functional package manager"; - # TODO switch to nixos-23.11-small - # https://nixpk.gs/pr-tracker.html?pr=291954 - inputs.nixpkgs.url = "github:NixOS/nixpkgs/release-23.11"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; + inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446"; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; - inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; }; + inputs.libgit2 = { url = "github:libgit2/libgit2/v1.8.1"; flake = false; }; + + # dev tooling + inputs.flake-parts.url = "github:hercules-ci/flake-parts"; + inputs.git-hooks-nix.url = "github:cachix/git-hooks.nix"; + # work around https://github.com/NixOS/nix/issues/7730 + inputs.flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; + inputs.git-hooks-nix.inputs.nixpkgs.follows = "nixpkgs"; + inputs.git-hooks-nix.inputs.nixpkgs-stable.follows = "nixpkgs"; + # work around 7730 and https://github.com/NixOS/nix/issues/7807 + inputs.git-hooks-nix.inputs.flake-compat.follows = ""; + inputs.git-hooks-nix.inputs.gitignore.follows = ""; + + outputs = inputs@{ self, nixpkgs, nixpkgs-regression, libgit2, ... }: - outputs = { self, nixpkgs, nixpkgs-regression, libgit2, ... }: let inherit (nixpkgs) lib; - inherit (lib) fileset; officialRelease = false; @@ -31,14 +41,9 @@ crossSystems = [ "armv6l-unknown-linux-gnueabihf" "armv7l-unknown-linux-gnueabihf" - "x86_64-unknown-freebsd13" + "riscv64-unknown-linux-gnu" "x86_64-unknown-netbsd" - ]; - - # Nix doesn't yet build on this platform, so we put it in a - # separate list. We just use this for `devShells` and - # `nixpkgsFor`, which this depends on. - shellCrossSystems = crossSystems ++ [ + "x86_64-unknown-freebsd" "x86_64-w64-mingw32" ]; @@ -50,6 +55,18 @@ "stdenv" ]; + /** + `flatMapAttrs attrs f` applies `f` to each attribute in `attrs` and + merges the results into a single attribute set. + + This can be nested to form a build matrix where all the attributes + generated by the innermost `f` are returned as is. + (Provided that the names are unique.) + + See https://nixos.org/manual/nixpkgs/stable/index.html#function-library-lib.attrsets.concatMapAttrs + */ + flatMapAttrs = attrs: f: lib.concatMapAttrs f attrs; + forAllSystems = lib.genAttrs systems; forAllCrossSystems = lib.genAttrs crossSystems; @@ -63,6 +80,17 @@ }) stdenvs); + + # We don't apply flake-parts to the whole flake so that non-development attributes + # load without fetching any development inputs. + devFlake = inputs.flake-parts.lib.mkFlake { inherit inputs; } { + imports = [ ./maintainers/flake-module.nix ]; + systems = lib.subtractLists crossSystems systems; + perSystem = { system, ... }: { + _module.args.pkgs = nixpkgsFor.${system}.native; + }; + }; + # Memoize nixpkgs for different platforms for efficiency. nixpkgsFor = forAllSystems (system: let @@ -84,31 +112,9 @@ in { inherit stdenvs native; static = native.pkgsStatic; - cross = lib.genAttrs shellCrossSystems (crossSystem: make-pkgs crossSystem "stdenv"); + cross = forAllCrossSystems (crossSystem: make-pkgs crossSystem "stdenv"); }); - installScriptFor = tarballs: - nixpkgsFor.x86_64-linux.native.callPackage ./scripts/installer.nix { - inherit tarballs; - }; - - testNixVersions = pkgs: client: daemon: - pkgs.callPackage ./package.nix { - pname = - "nix-tests" - + lib.optionalString - (lib.versionAtLeast daemon.version "2.4pre20211005" && - lib.versionAtLeast client.version "2.4pre20211005") - "-${client.version}-against-${daemon.version}"; - - inherit fileset; - - test-client = client; - test-daemon = daemon; - - doBuild = false; - }; - binaryTarball = nix: pkgs: pkgs.callPackage ./scripts/binary-tarball.nix { inherit nix; }; @@ -120,244 +126,130 @@ { nixStable = prev.nix; - default-busybox-sandbox-shell = final.busybox.override { - useMusl = true; - enableStatic = true; - enableMinimal = true; - extraConfig = '' - CONFIG_FEATURE_FANCY_ECHO y - CONFIG_FEATURE_SH_MATH y - CONFIG_FEATURE_SH_MATH_64 y + # A new scope, so that we can use `callPackage` to inject our own interdependencies + # without "polluting" the top level "`pkgs`" attrset. + # This also has the benefit of providing us with a distinct set of packages + # we can iterate over. + nixComponents = lib.makeScope final.nixDependencies.newScope (import ./packaging/components.nix); - CONFIG_ASH y - CONFIG_ASH_OPTIMIZE_FOR_SIZE y - - CONFIG_ASH_ALIAS y - CONFIG_ASH_BASH_COMPAT y - CONFIG_ASH_CMDCMD y - CONFIG_ASH_ECHO y - CONFIG_ASH_GETOPTS y - CONFIG_ASH_INTERNAL_GLOB y - CONFIG_ASH_JOB_CONTROL y - CONFIG_ASH_PRINTF y - CONFIG_ASH_TEST y - ''; - }; - - libgit2-nix = final.libgit2.overrideAttrs (attrs: { - src = libgit2; - version = libgit2.lastModifiedDate; - cmakeFlags = attrs.cmakeFlags or [] - ++ [ "-DUSE_SSH=exec" ]; + # The dependencies are in their own scope, so that they don't have to be + # in Nixpkgs top level `pkgs` or `nixComponents`. + nixDependencies = lib.makeScope final.newScope (import ./packaging/dependencies.nix { + inherit inputs stdenv versionSuffix; + pkgs = final; }); - boehmgc-nix = (final.boehmgc.override { - enableLargeConfig = true; - }).overrideAttrs(o: { - patches = (o.patches or []) ++ [ - ./dep-patches/boehmgc-coroutine-sp-fallback.diff + nix = final.nixComponents.nix; - # https://github.com/ivmai/bdwgc/pull/586 - ./dep-patches/boehmgc-traceable_allocator-public.diff - ]; - }); - - changelog-d-nix = final.buildPackages.callPackage ./misc/changelog-d.nix { }; - - nix = - let - officialRelease = false; - versionSuffix = - if officialRelease - then "" - else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}"; - - in final.callPackage ./package.nix { - inherit - fileset - stdenv - versionSuffix - ; - officialRelease = false; - boehmgc = final.boehmgc-nix; - libgit2 = final.libgit2-nix; - busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell; - } // { - # this is a proper separate downstream package, but put - # here also for back compat reasons. - perl-bindings = final.nix-perl-bindings; - }; - - nix-perl-bindings = final.callPackage ./perl { - inherit fileset stdenv; + nix_noTests = final.nix.override { + doInstallCheck = false; + doCheck = false; }; + # See https://github.com/NixOS/nixpkgs/pull/214409 + # Remove when fixed in this flake's nixpkgs + pre-commit = + if prev.stdenv.hostPlatform.system == "i686-linux" + then (prev.pre-commit.override (o: { dotnet-sdk = ""; })).overridePythonAttrs (o: { doCheck = false; }) + else prev.pre-commit; + }; in { # A Nixpkgs overlay that overrides the 'nix' and - # 'nix.perl-bindings' packages. + # 'nix-perl-bindings' packages. overlays.default = overlayFor (p: p.stdenv); - hydraJobs = { - - # Binary package for various platforms. - build = forAllSystems (system: self.packages.${system}.nix); - - shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation); - - buildStatic = lib.genAttrs linux64BitSystems (system: self.packages.${system}.nix-static); - - buildCross = forAllCrossSystems (crossSystem: - lib.genAttrs ["x86_64-linux"] (system: self.packages.${system}."nix-${crossSystem}")); - - buildNoGc = forAllSystems (system: - self.packages.${system}.nix.override { enableGC = false; } - ); - - buildNoTests = forAllSystems (system: - self.packages.${system}.nix.override { - doCheck = false; - doInstallCheck = false; - installUnitTests = false; - } - ); - - # Toggles some settings for better coverage. Windows needs these - # library combinations, and Debian build Nix with GNU readline too. - buildReadlineNoMarkdown = forAllSystems (system: - self.packages.${system}.nix.override { - enableMarkdown = false; - readlineFlavor = "readline"; - } - ); - - # Perl bindings for various platforms. - perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nix.perl-bindings); - - # Binary tarball for various platforms, containing a Nix store - # with the closure of 'nix' package, and the second half of - # the installation script. - binaryTarball = forAllSystems (system: binaryTarball nixpkgsFor.${system}.native.nix nixpkgsFor.${system}.native); - - binaryTarballCross = lib.genAttrs ["x86_64-linux"] (system: - forAllCrossSystems (crossSystem: - binaryTarball - self.packages.${system}."nix-${crossSystem}" - nixpkgsFor.${system}.cross.${crossSystem})); - - # The first half of the installation script. This is uploaded - # to https://nixos.org/nix/install. It downloads the binary - # tarball for the user's system and calls the second half of the - # installation script. - installerScript = installScriptFor [ - # Native - self.hydraJobs.binaryTarball."x86_64-linux" - self.hydraJobs.binaryTarball."i686-linux" - self.hydraJobs.binaryTarball."aarch64-linux" - self.hydraJobs.binaryTarball."x86_64-darwin" - self.hydraJobs.binaryTarball."aarch64-darwin" - # Cross - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" - ]; - installerScriptForGHA = installScriptFor [ - # Native - self.hydraJobs.binaryTarball."x86_64-linux" - self.hydraJobs.binaryTarball."x86_64-darwin" - # Cross - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" - ]; - - # docker image with Nix inside - dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); - - # Line coverage analysis. - coverage = nixpkgsFor.x86_64-linux.native.nix.override { - pname = "nix-coverage"; - withCoverageChecks = true; - }; - - # API docs for Nix's unstable internal C++ interfaces. - internal-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ./package.nix { - inherit fileset; - doBuild = false; - enableInternalAPIDocs = true; - }; - - # System tests. - tests = import ./tests/nixos { inherit lib nixpkgs nixpkgsFor; } // { - - # Make sure that nix-env still produces the exact same result - # on a particular version of Nixpkgs. - evalNixpkgs = - let - inherit (nixpkgsFor.x86_64-linux.native) runCommand nix; - in - runCommand "eval-nixos" { buildInputs = [ nix ]; } - '' - type -p nix-env - # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593. - time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages - [[ $(sha1sum < packages | cut -c1-40) = ff451c521e61e4fe72bdbe2d0ca5d1809affa733 ]] - mkdir $out - ''; - - nixpkgsLibTests = - forAllSystems (system: - import (nixpkgs + "/lib/tests/release.nix") - { pkgs = nixpkgsFor.${system}.native; - nixVersions = [ self.packages.${system}.nix ]; - } - ); - }; - - metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" { - pkgs = nixpkgsFor.x86_64-linux.native; - nixpkgs = nixpkgs-regression; - }; - - installTests = forAllSystems (system: - let pkgs = nixpkgsFor.${system}.native; in - pkgs.runCommand "install-tests" { - againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix; - againstCurrentUnstable = - # FIXME: temporarily disable this on macOS because of #3605. - if system == "x86_64-linux" - then testNixVersions pkgs pkgs.nix pkgs.nixUnstable - else null; - # Disabled because the latest stable version doesn't handle - # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work - # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable; - } "touch $out"); - - installerTests = import ./tests/installer { - binaryTarballs = self.hydraJobs.binaryTarball; - inherit nixpkgsFor; - }; - + hydraJobs = import ./packaging/hydra.nix { + inherit + inputs + binaryTarball + forAllCrossSystems + forAllSystems + lib + linux64BitSystems + nixpkgsFor + self + ; }; checks = forAllSystems (system: { binaryTarball = self.hydraJobs.binaryTarball.${system}; - perlBindings = self.hydraJobs.perlBindings.${system}; installTests = self.hydraJobs.installTests.${system}; nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system}; rl-next = let pkgs = nixpkgsFor.${system}.native; in pkgs.buildPackages.runCommand "test-rl-next-release-notes" { } '' - LANG=C.UTF-8 ${pkgs.changelog-d-nix}/bin/changelog-d ${./doc/manual/rl-next} >$out + LANG=C.UTF-8 ${pkgs.changelog-d}/bin/changelog-d ${./doc/manual/rl-next} >$out ''; + repl-completion = nixpkgsFor.${system}.native.callPackage ./tests/repl-completion.nix { }; } // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) { dockerImage = self.hydraJobs.dockerImage.${system}; - }); + } // (lib.optionalAttrs (!(builtins.elem system linux32BitSystems))) { + # Some perl dependencies are broken on i686-linux. + # Since the support is only best-effort there, disable the perl + # bindings - packages = forAllSystems (system: rec { - inherit (nixpkgsFor.${system}.native) nix changelog-d-nix; - default = nix; - } // (lib.optionalAttrs (builtins.elem system linux64BitSystems) { - nix-static = nixpkgsFor.${system}.static.nix; + # Temporarily disabled because GitHub Actions OOM issues. Once + # the old build system is gone and we are back to one build + # system, we should reenable this. + #perlBindings = self.hydraJobs.perlBindings.${system}; + } + # Add "passthru" tests + // flatMapAttrs ({ + "" = nixpkgsFor.${system}.native; + } // lib.optionalAttrs (! nixpkgsFor.${system}.native.stdenv.hostPlatform.isDarwin) { + # TODO: enable static builds for darwin, blocked on: + # https://github.com/NixOS/nixpkgs/issues/320448 + "static-" = nixpkgsFor.${system}.static; + }) + (nixpkgsPrefix: nixpkgs: + flatMapAttrs nixpkgs.nixComponents + (pkgName: pkg: + flatMapAttrs pkg.tests or {} + (testName: test: { + "${nixpkgsPrefix}${pkgName}-${testName}" = test; + }) + ) + ) + // devFlake.checks.${system} or {} + ); + + packages = forAllSystems (system: + { # Here we put attributes that map 1:1 into packages., ie + # for which we don't apply the full build matrix such as cross or static. + inherit (nixpkgsFor.${system}.native) + changelog-d; + default = self.packages.${system}.nix; + nix-internal-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-internal-api-docs; + nix-external-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-external-api-docs; + } + # We need to flatten recursive attribute sets of derivations to pass `flake check`. + // flatMapAttrs + { # Components we'll iterate over in the upcoming lambda + "nix" = { }; + # Temporarily disabled because GitHub Actions OOM issues. Once + # the old build system is gone and we are back to one build + # system, we should reenable these. + #"nix-util" = { }; + #"nix-store" = { }; + #"nix-fetchers" = { }; + } + (pkgName: {}: { + # These attributes go right into `packages.`. + "${pkgName}" = nixpkgsFor.${system}.native.nixComponents.${pkgName}; + "${pkgName}-static" = nixpkgsFor.${system}.static.nixComponents.${pkgName}; + } + // flatMapAttrs (lib.genAttrs crossSystems (_: { })) (crossSystem: {}: { + # These attributes go right into `packages.`. + "${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName}; + }) + // flatMapAttrs (lib.genAttrs stdenvs (_: { })) (stdenvName: {}: { + # These attributes go right into `packages.`. + "${pkgName}-${stdenvName}" = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nixComponents.${pkgName}; + }) + ) + // lib.optionalAttrs (builtins.elem system linux64BitSystems) { dockerImage = let pkgs = nixpkgsFor.${system}.native; @@ -372,21 +264,27 @@ ln -s ${image} $image echo "file binary-dist $image" >> $out/nix-support/hydra-build-products ''; - } // builtins.listToAttrs (map - (crossSystem: { - name = "nix-${crossSystem}"; - value = nixpkgsFor.${system}.cross.${crossSystem}.nix; - }) - crossSystems) - // builtins.listToAttrs (map - (stdenvName: { - name = "nix-${stdenvName}"; - value = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nix; - }) - stdenvs))); + }); devShells = let - makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; forDevShell = true; }).overrideAttrs (attrs: { + makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; forDevShell = true; }).overrideAttrs (attrs: + let + modular = devFlake.getSystem stdenv.buildPlatform.system; + transformFlag = prefix: flag: + assert builtins.isString flag; + let + rest = builtins.substring 2 (builtins.stringLength flag) flag; + in + "-D${prefix}:${rest}"; + havePerl = stdenv.buildPlatform == stdenv.hostPlatform && stdenv.hostPlatform.isUnix; + ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags; + in { + pname = "shell-for-" + attrs.pname; + + # Remove the version suffix to avoid unnecessary attempts to substitute in nix develop + version = lib.fileContents ./.version; + name = attrs.pname; + installFlags = "sysconfdir=$(out)/etc"; shellHook = '' PATH=$prefix/bin:$PATH @@ -397,11 +295,62 @@ XDG_DATA_DIRS+=:$out/share ''; + # We use this shell with the local checkout, not unpackPhase. + src = null; + + env = { + # Needed for Meson to find Boost. + # https://github.com/NixOS/nixpkgs/issues/86131. + BOOST_INCLUDEDIR = "${lib.getDev pkgs.nixDependencies.boost}/include"; + BOOST_LIBRARYDIR = "${lib.getLib pkgs.nixDependencies.boost}/lib"; + # For `make format`, to work without installing pre-commit + _NIX_PRE_COMMIT_HOOKS_CONFIG = + "${(pkgs.formats.yaml { }).generate "pre-commit-config.yaml" modular.pre-commit.settings.rawConfig}"; + }; + + mesonFlags = + map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents.nix-util.mesonFlags) + ++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents.nix-store.mesonFlags) + ++ map (transformFlag "libfetchers") (ignoreCrossFile pkgs.nixComponents.nix-fetchers.mesonFlags) + ++ lib.optionals havePerl (map (transformFlag "perl") (ignoreCrossFile pkgs.nixComponents.nix-perl-bindings.mesonFlags)) + ++ map (transformFlag "libexpr") (ignoreCrossFile pkgs.nixComponents.nix-expr.mesonFlags) + ++ map (transformFlag "libcmd") (ignoreCrossFile pkgs.nixComponents.nix-cmd.mesonFlags) + ; + nativeBuildInputs = attrs.nativeBuildInputs or [] + ++ pkgs.nixComponents.nix-util.nativeBuildInputs + ++ pkgs.nixComponents.nix-store.nativeBuildInputs + ++ pkgs.nixComponents.nix-fetchers.nativeBuildInputs + ++ lib.optionals havePerl pkgs.nixComponents.nix-perl-bindings.nativeBuildInputs + ++ pkgs.nixComponents.nix-internal-api-docs.nativeBuildInputs + ++ pkgs.nixComponents.nix-external-api-docs.nativeBuildInputs + ++ lib.optional + (!stdenv.buildPlatform.canExecute stdenv.hostPlatform + # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479 + && !(stdenv.hostPlatform.isWindows && stdenv.buildPlatform.isDarwin) + && stdenv.hostPlatform.emulatorAvailable pkgs.buildPackages + && lib.meta.availableOn stdenv.buildPlatform (stdenv.hostPlatform.emulator pkgs.buildPackages)) + pkgs.buildPackages.mesonEmulatorHook + ++ [ + pkgs.buildPackages.cmake + pkgs.buildPackages.shellcheck + pkgs.buildPackages.changelog-d + modular.pre-commit.settings.package + (pkgs.writeScriptBin "pre-commit-hooks-install" + modular.pre-commit.settings.installationScript) + ] # TODO: Remove the darwin check once # https://github.com/NixOS/nixpkgs/pull/291814 is available ++ lib.optional (stdenv.cc.isClang && !stdenv.buildPlatform.isDarwin) pkgs.buildPackages.bear ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) pkgs.buildPackages.clang-tools; + + buildInputs = attrs.buildInputs or [] + ++ [ + pkgs.gtest + pkgs.rapidcheck + ] + ++ lib.optional havePerl pkgs.perl + ; }); in forAllSystems (system: @@ -413,8 +362,8 @@ in (makeShells "native" nixpkgsFor.${system}.native) // (lib.optionalAttrs (!nixpkgsFor.${system}.native.stdenv.isDarwin) - (makeShells "static" nixpkgsFor.${system}.static)) // - (lib.genAttrs shellCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv)) // + (makeShells "static" nixpkgsFor.${system}.static) // + (forAllCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv))) // { default = self.devShells.${system}.native-stdenvPackages; } diff --git a/local.mk b/local.mk index 3f3abb9f0..b27c7031e 100644 --- a/local.mk +++ b/local.mk @@ -2,9 +2,14 @@ GLOBAL_CXXFLAGS += -Wno-deprecated-declarations -Werror=switch # Allow switch-enum to be overridden for files that do not support it, usually because of dependency headers. ERROR_SWITCH_ENUM = -Werror=switch-enum -$(foreach i, config.h $(wildcard src/lib*/*.hh), \ +$(foreach i, config.h $(wildcard src/lib*/*.hh) $(filter-out %_internal.h, $(wildcard src/lib*c/*.h)), \ $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) +ifdef HOST_UNIX + $(foreach i, $(wildcard src/lib*/unix/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) +endif + $(GCH): src/libutil/util.hh config.h -GCH_CXXFLAGS = -I src/libutil +GCH_CXXFLAGS = $(INCLUDE_libutil) diff --git a/m4/gcc_bug_80431.m4 b/m4/gcc_bug_80431.m4 index e42f01956..cdc4ddb40 100644 --- a/m4/gcc_bug_80431.m4 +++ b/m4/gcc_bug_80431.m4 @@ -46,11 +46,13 @@ AC_DEFUN([ENSURE_NO_GCC_BUG_80431], ]])], [status_80431=0], [status_80431=$?], - [ - # Assume we're bug-free when cross-compiling - ]) + [status_80431='']) AC_LANG_POP(C++) AS_CASE([$status_80431], + [''],[ + AC_MSG_RESULT(cannot check because cross compiling) + AC_MSG_NOTICE(assume we are bug free) + ], [0],[ AC_MSG_RESULT(yes) ], diff --git a/maintainers/README.md b/maintainers/README.md index fa321c7c0..b92833497 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -30,17 +30,18 @@ We aim to achieve this by improving the contributor experience and attracting mo ## Members - Eelco Dolstra (@edolstra) – Team lead -- Théophane Hufschmitt (@thufschmitt) - Valentin Gagarin (@fricklerhandwerk) - Thomas Bereknyei (@tomberek) - Robert Hensing (@roberth) - John Ericson (@Ericson2314) +The team is on Github as [@NixOS/nix-team](https://github.com/orgs/NixOS/teams/nix-team). + ## Meeting protocol -The team meets twice a week: +The team meets twice a week (times are denoted in the [Europe/Amsterdam](https://en.m.wikipedia.org/wiki/Time_in_the_Netherlands) time zone): -- Discussion meeting: [Fridays 13:00-14:00 CET](https://calendar.google.com/calendar/event?eid=MHNtOGVuNWtrZXNpZHR2bW1sM3QyN2ZjaGNfMjAyMjExMjVUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) +- Discussion meeting: [Wednesday 21:00-22:00 Europe/Amsterdam](https://www.google.com/calendar/event?eid=ZG5rZzNyajRjajducGV2NGY5aGkzYWIwdnJfMjAyNDA1MDhUMTkwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) 1. Triage issues and pull requests from the [No Status](#no-status) column (30 min) 2. Discuss issues and pull requests from the [To discuss](#to-discuss) column (30 min). @@ -49,15 +50,19 @@ The team meets twice a week: - mark it as draft if it is blocked on the contributor - escalate it back to the team by moving it to To discuss, and leaving a comment as to why the issue needs to be discussed again. -- Work meeting: [Mondays 13:00-15:00 CET](https://calendar.google.com/calendar/event?eid=NTM1MG1wNGJnOGpmOTZhYms3bTB1bnY5cWxfMjAyMjExMjFUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) +- Work meeting: [Mondays 14:00-16:00 Europe/Amsterdam](https://www.google.com/calendar/event?eid=Ym52NDdzYnRic2NzcDcybjZiNDhpNzhpa3NfMjAyNDA1MTNUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) 1. Code review on pull requests from [In review](#in-review). 2. Other chores and tasks. Meeting notes are collected on a [collaborative scratchpad](https://pad.lassul.us/Cv7FpYx-Ri-4VjUykQOLAw). -Notes on issues and pull requests are posted as comments and linked from the meeting notes, so they are easy to find from both places. +Notes on issues and pull requests are posted as comments and linked from the meeting notes, so they are can be found from both places. [All meeting notes](https://discourse.nixos.org/search?expanded=true&q=Nix%20team%20meeting%20minutes%20%23%20%23dev%3Anix%20in%3Atitle%20order%3Alatest_topic) are published on Discourse under the [Nix category](https://discourse.nixos.org/c/dev/nix/50). +Team meetings are generally open to anyone interested. +We can make exceptions to discuss sensitive issues, such as security incidents or people matters. +Contact any team member to get a calendar invite for reminders and updates. + ## Project board protocol The team uses a [GitHub project board](https://github.com/orgs/NixOS/projects/19/views/1) for tracking its work. diff --git a/maintainers/data/release-credits-email-to-handle.json b/maintainers/data/release-credits-email-to-handle.json new file mode 100644 index 000000000..cddc1a6e7 --- /dev/null +++ b/maintainers/data/release-credits-email-to-handle.json @@ -0,0 +1,52 @@ +{ + "bogus": "bogus", + "edolstra@gmail.com": "edolstra", + "roberth@users.noreply.github.com": "roberth", + "toscano.pino@tiscali.it": "pinotree", + "valentin@gagarin.work": "fricklerhandwerk", + "mr.trubach@icloud.com": "tie", + "robert@roberthensing.nl": "roberth", + "lix@jade.fyi": "lf-", + "cole.e.helbling@outlook.com": "cole-h", + "joerg@thalheim.io": "Mic92", + "John.Ericson@Obsidian.Systems": "Ericson2314", + "ryan.hendrickson@alum.mit.edu": "rhendric", + "67135060+poweredbypie@users.noreply.github.com": "poweredbypie", + "detroyejr@outlook.com": "detroyejr", + "silvan.mosberger@tweag.io": "infinisil", + "vcs@emily.moe": "emilazy", + "farid.m.zakaria@gmail.com": "fzakaria", + "22859658+RTUnreal@users.noreply.github.com": "RTUnreal", + "me@las.rs": "L-as", + "philip.taron@gmail.com": "philiptaron", + "root@goldstein.rs": "GoldsteinE", + "tomberek@users.noreply.github.com": "tomberek", + "lexi.mattick@neuralink.com": "kognise", + "andrew@johnandrewmarshall.com": "amarshall", + "contact@romain-neil.fr": "romain-neil", + "Mic92@users.noreply.github.com": "Mic92", + "valentin.gagarin@tweag.io": "fricklerhandwerk", + "siddhantk232@gmail.com": "siddhantk232", + "kn@openbsd.org": "klemensn", + "slyich@gmail.com": "trofi", + "theophane.hufschmitt@tweag.io": "thufschmitt", + "alicebob@lijzij.de": "alicebob", + "winter@winter.cafe": "winterqt", + "brian@brianmckenna.org": "puffnfresh", + "git@haenoe.party": "haenoe", + "peshogo@gmail.com": "pineapplehunter", + "poweredbypie@users.noreply.github.com": "poweredbypie", + "arthur200126@gmail.com": "Artoria2e5", + "tomberek@gmail.com": "tomberek", + "jaredbaur@fastmail.com": "jmbaur", + "andreas@rammhold.de": "andir", + "hamirmahal@gmail.com": "hamirmahal", + "git@JohnEricson.me": "Ericson2314", + "8763518+SkamDart@users.noreply.github.com": "SkamDart", + "kirillrdy@gmail.com": "kirillrdy", + "pennae@lix.systems": "pennae", + "delroth@gmail.com": "delroth", + "enno@nerdworks.de": "elohmeier", + "mjbauer95@gmail.com": "matthewbauer", + "MostAwesomeDude@gmail.com": "MostAwesomeDude" +} \ No newline at end of file diff --git a/maintainers/data/release-credits-handle-to-name.json b/maintainers/data/release-credits-handle-to-name.json new file mode 100644 index 000000000..abf9ed05b --- /dev/null +++ b/maintainers/data/release-credits-handle-to-name.json @@ -0,0 +1,45 @@ +{ + "fzakaria": "Farid Zakaria", + "kognise": "Lexi Mattick", + "L-as": "Las Safin", + "haenoe": "HaeNoe", + "andir": "Andreas Rammhold", + "matthewbauer": "Matthew Bauer", + "emilazy": "Emily", + "pineapplehunter": "Shogo Takata", + "RTUnreal": null, + "jmbaur": "Jared Baur", + "Ericson2314": "John Ericson", + "pinotree": "Pino Toscano", + "tie": "Ivan Trubach", + "poweredbypie": null, + "fricklerhandwerk": "Valentin Gagarin", + "Mic92": "J\u00f6rg Thalheim", + "alicebob": "Harmen", + "elohmeier": "Enno Richter", + "delroth": "Pierre Bourdon", + "kirillrdy": null, + "thufschmitt": "Th\u00e9ophane Hufschmitt", + "detroyejr": "Jonathan De Troye", + "klemensn": "Klemens Nanni", + "tomberek": null, + "rhendric": "Ryan Hendrickson", + "philiptaron": "Philip Taron", + "puffnfresh": "Brian McKenna", + "lf-": "jade", + "romain-neil": "Romain Neil", + "hamirmahal": "Hamir Mahal", + "edolstra": "Eelco Dolstra", + "Artoria2e5": "Mingye Wang", + "SkamDart": "Cameron", + "roberth": "Robert Hensing", + "amarshall": "Andrew Marshall", + "trofi": "Sergei Trofimovich", + "cole-h": "Cole Helbling", + "infinisil": "Silvan Mosberger", + "siddhantk232": "Siddhant Kumar", + "winterqt": "Winter", + "GoldsteinE": "Max \u201cGoldstein\u201d Siling", + "pennae": null, + "MostAwesomeDude": "Corbin Simpson" +} \ No newline at end of file diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix new file mode 100644 index 000000000..be91df536 --- /dev/null +++ b/maintainers/flake-module.nix @@ -0,0 +1,677 @@ +{ lib, getSystem, inputs, ... }: + +{ + imports = [ + inputs.git-hooks-nix.flakeModule + ]; + + perSystem = { config, pkgs, ... }: { + + # https://flake.parts/options/pre-commit-hooks-nix.html#options + pre-commit.settings = { + hooks = { + clang-format = { + enable = true; + excludes = [ + # We don't want to format test data + # ''tests/(?!nixos/).*\.nix'' + ''^tests/unit/[^/]*/data/.*$'' + + # Don't format vendored code + ''^doc/manual/redirects\.js$'' + ''^doc/manual/theme/highlight\.js$'' + + # We haven't applied formatting to these files yet + ''^doc/manual/redirects\.js$'' + ''^doc/manual/theme/highlight\.js$'' + ''^precompiled-headers\.h$'' + ''^src/build-remote/build-remote\.cc$'' + ''^src/libcmd/built-path\.cc$'' + ''^src/libcmd/built-path\.hh$'' + ''^src/libcmd/command\.cc$'' + ''^src/libcmd/command\.hh$'' + ''^src/libcmd/common-eval-args\.cc$'' + ''^src/libcmd/common-eval-args\.hh$'' + ''^src/libcmd/editor-for\.cc$'' + ''^src/libcmd/installable-attr-path\.cc$'' + ''^src/libcmd/installable-attr-path\.hh$'' + ''^src/libcmd/installable-derived-path\.cc$'' + ''^src/libcmd/installable-derived-path\.hh$'' + ''^src/libcmd/installable-flake\.cc$'' + ''^src/libcmd/installable-flake\.hh$'' + ''^src/libcmd/installable-value\.cc$'' + ''^src/libcmd/installable-value\.hh$'' + ''^src/libcmd/installables\.cc$'' + ''^src/libcmd/installables\.hh$'' + ''^src/libcmd/legacy\.hh$'' + ''^src/libcmd/markdown\.cc$'' + ''^src/libcmd/misc-store-flags\.cc$'' + ''^src/libcmd/repl-interacter\.cc$'' + ''^src/libcmd/repl-interacter\.hh$'' + ''^src/libcmd/repl\.cc$'' + ''^src/libcmd/repl\.hh$'' + ''^src/libexpr-c/nix_api_expr\.cc$'' + ''^src/libexpr-c/nix_api_external\.cc$'' + ''^src/libexpr/attr-path\.cc$'' + ''^src/libexpr/attr-path\.hh$'' + ''^src/libexpr/attr-set\.cc$'' + ''^src/libexpr/attr-set\.hh$'' + ''^src/libexpr/eval-cache\.cc$'' + ''^src/libexpr/eval-cache\.hh$'' + ''^src/libexpr/eval-error\.cc$'' + ''^src/libexpr/eval-inline\.hh$'' + ''^src/libexpr/eval-settings\.cc$'' + ''^src/libexpr/eval-settings\.hh$'' + ''^src/libexpr/eval\.cc$'' + ''^src/libexpr/eval\.hh$'' + ''^src/libexpr/function-trace\.cc$'' + ''^src/libexpr/gc-small-vector\.hh$'' + ''^src/libexpr/get-drvs\.cc$'' + ''^src/libexpr/get-drvs\.hh$'' + ''^src/libexpr/json-to-value\.cc$'' + ''^src/libexpr/nixexpr\.cc$'' + ''^src/libexpr/nixexpr\.hh$'' + ''^src/libexpr/parser-state\.hh$'' + ''^src/libexpr/pos-table\.hh$'' + ''^src/libexpr/primops\.cc$'' + ''^src/libexpr/primops\.hh$'' + ''^src/libexpr/primops/context\.cc$'' + ''^src/libexpr/primops/fetchClosure\.cc$'' + ''^src/libexpr/primops/fetchMercurial\.cc$'' + ''^src/libexpr/primops/fetchTree\.cc$'' + ''^src/libexpr/primops/fromTOML\.cc$'' + ''^src/libexpr/print-ambiguous\.cc$'' + ''^src/libexpr/print-ambiguous\.hh$'' + ''^src/libexpr/print-options\.hh$'' + ''^src/libexpr/print\.cc$'' + ''^src/libexpr/print\.hh$'' + ''^src/libexpr/search-path\.cc$'' + ''^src/libexpr/symbol-table\.hh$'' + ''^src/libexpr/value-to-json\.cc$'' + ''^src/libexpr/value-to-json\.hh$'' + ''^src/libexpr/value-to-xml\.cc$'' + ''^src/libexpr/value-to-xml\.hh$'' + ''^src/libexpr/value\.hh$'' + ''^src/libexpr/value/context\.cc$'' + ''^src/libexpr/value/context\.hh$'' + ''^src/libfetchers/attrs\.cc$'' + ''^src/libfetchers/cache\.cc$'' + ''^src/libfetchers/cache\.hh$'' + ''^src/libfetchers/fetch-settings\.cc$'' + ''^src/libfetchers/fetch-settings\.hh$'' + ''^src/libfetchers/fetch-to-store\.cc$'' + ''^src/libfetchers/fetchers\.cc$'' + ''^src/libfetchers/fetchers\.hh$'' + ''^src/libfetchers/filtering-source-accessor\.cc$'' + ''^src/libfetchers/filtering-source-accessor\.hh$'' + ''^src/libfetchers/fs-source-accessor\.cc$'' + ''^src/libfetchers/fs-source-accessor\.hh$'' + ''^src/libfetchers/git-utils\.cc$'' + ''^src/libfetchers/git-utils\.hh$'' + ''^src/libfetchers/github\.cc$'' + ''^src/libfetchers/indirect\.cc$'' + ''^src/libfetchers/memory-source-accessor\.cc$'' + ''^src/libfetchers/path\.cc$'' + ''^src/libfetchers/registry\.cc$'' + ''^src/libfetchers/registry\.hh$'' + ''^src/libfetchers/tarball\.cc$'' + ''^src/libfetchers/tarball\.hh$'' + ''^src/libfetchers/git\.cc$'' + ''^src/libfetchers/mercurial\.cc$'' + ''^src/libflake/flake/config\.cc$'' + ''^src/libflake/flake/flake\.cc$'' + ''^src/libflake/flake/flake\.hh$'' + ''^src/libflake/flake/flakeref\.cc$'' + ''^src/libflake/flake/flakeref\.hh$'' + ''^src/libflake/flake/lockfile\.cc$'' + ''^src/libflake/flake/lockfile\.hh$'' + ''^src/libflake/flake/url-name\.cc$'' + ''^src/libmain/common-args\.cc$'' + ''^src/libmain/common-args\.hh$'' + ''^src/libmain/loggers\.cc$'' + ''^src/libmain/loggers\.hh$'' + ''^src/libmain/progress-bar\.cc$'' + ''^src/libmain/shared\.cc$'' + ''^src/libmain/shared\.hh$'' + ''^src/libmain/unix/stack\.cc$'' + ''^src/libstore/binary-cache-store\.cc$'' + ''^src/libstore/binary-cache-store\.hh$'' + ''^src/libstore/build-result\.hh$'' + ''^src/libstore/builtins\.hh$'' + ''^src/libstore/builtins/buildenv\.cc$'' + ''^src/libstore/builtins/buildenv\.hh$'' + ''^src/libstore/common-protocol-impl\.hh$'' + ''^src/libstore/common-protocol\.cc$'' + ''^src/libstore/common-protocol\.hh$'' + ''^src/libstore/common-ssh-store-config\.hh$'' + ''^src/libstore/content-address\.cc$'' + ''^src/libstore/content-address\.hh$'' + ''^src/libstore/daemon\.cc$'' + ''^src/libstore/daemon\.hh$'' + ''^src/libstore/derivations\.cc$'' + ''^src/libstore/derivations\.hh$'' + ''^src/libstore/derived-path-map\.cc$'' + ''^src/libstore/derived-path-map\.hh$'' + ''^src/libstore/derived-path\.cc$'' + ''^src/libstore/derived-path\.hh$'' + ''^src/libstore/downstream-placeholder\.cc$'' + ''^src/libstore/downstream-placeholder\.hh$'' + ''^src/libstore/dummy-store\.cc$'' + ''^src/libstore/export-import\.cc$'' + ''^src/libstore/filetransfer\.cc$'' + ''^src/libstore/filetransfer\.hh$'' + ''^src/libstore/gc-store\.hh$'' + ''^src/libstore/globals\.cc$'' + ''^src/libstore/globals\.hh$'' + ''^src/libstore/http-binary-cache-store\.cc$'' + ''^src/libstore/legacy-ssh-store\.cc$'' + ''^src/libstore/legacy-ssh-store\.hh$'' + ''^src/libstore/length-prefixed-protocol-helper\.hh$'' + ''^src/libstore/linux/personality\.cc$'' + ''^src/libstore/linux/personality\.hh$'' + ''^src/libstore/local-binary-cache-store\.cc$'' + ''^src/libstore/local-fs-store\.cc$'' + ''^src/libstore/local-fs-store\.hh$'' + ''^src/libstore/log-store\.cc$'' + ''^src/libstore/log-store\.hh$'' + ''^src/libstore/machines\.cc$'' + ''^src/libstore/machines\.hh$'' + ''^src/libstore/make-content-addressed\.cc$'' + ''^src/libstore/make-content-addressed\.hh$'' + ''^src/libstore/misc\.cc$'' + ''^src/libstore/names\.cc$'' + ''^src/libstore/names\.hh$'' + ''^src/libstore/nar-accessor\.cc$'' + ''^src/libstore/nar-accessor\.hh$'' + ''^src/libstore/nar-info-disk-cache\.cc$'' + ''^src/libstore/nar-info-disk-cache\.hh$'' + ''^src/libstore/nar-info\.cc$'' + ''^src/libstore/nar-info\.hh$'' + ''^src/libstore/outputs-spec\.cc$'' + ''^src/libstore/outputs-spec\.hh$'' + ''^src/libstore/parsed-derivations\.cc$'' + ''^src/libstore/path-info\.cc$'' + ''^src/libstore/path-info\.hh$'' + ''^src/libstore/path-references\.cc$'' + ''^src/libstore/path-regex\.hh$'' + ''^src/libstore/path-with-outputs\.cc$'' + ''^src/libstore/path\.cc$'' + ''^src/libstore/path\.hh$'' + ''^src/libstore/pathlocks\.cc$'' + ''^src/libstore/pathlocks\.hh$'' + ''^src/libstore/profiles\.cc$'' + ''^src/libstore/profiles\.hh$'' + ''^src/libstore/realisation\.cc$'' + ''^src/libstore/realisation\.hh$'' + ''^src/libstore/remote-fs-accessor\.cc$'' + ''^src/libstore/remote-fs-accessor\.hh$'' + ''^src/libstore/remote-store-connection\.hh$'' + ''^src/libstore/remote-store\.cc$'' + ''^src/libstore/remote-store\.hh$'' + ''^src/libstore/s3-binary-cache-store\.cc$'' + ''^src/libstore/s3\.hh$'' + ''^src/libstore/serve-protocol-impl\.cc$'' + ''^src/libstore/serve-protocol-impl\.hh$'' + ''^src/libstore/serve-protocol\.cc$'' + ''^src/libstore/serve-protocol\.hh$'' + ''^src/libstore/sqlite\.cc$'' + ''^src/libstore/sqlite\.hh$'' + ''^src/libstore/ssh-store\.cc$'' + ''^src/libstore/ssh\.cc$'' + ''^src/libstore/ssh\.hh$'' + ''^src/libstore/store-api\.cc$'' + ''^src/libstore/store-api\.hh$'' + ''^src/libstore/store-dir-config\.hh$'' + ''^src/libstore/build/derivation-goal\.cc$'' + ''^src/libstore/build/derivation-goal\.hh$'' + ''^src/libstore/build/drv-output-substitution-goal\.cc$'' + ''^src/libstore/build/drv-output-substitution-goal\.hh$'' + ''^src/libstore/build/entry-points\.cc$'' + ''^src/libstore/build/goal\.cc$'' + ''^src/libstore/build/goal\.hh$'' + ''^src/libstore/unix/build/hook-instance\.cc$'' + ''^src/libstore/unix/build/local-derivation-goal\.cc$'' + ''^src/libstore/unix/build/local-derivation-goal\.hh$'' + ''^src/libstore/build/substitution-goal\.cc$'' + ''^src/libstore/build/substitution-goal\.hh$'' + ''^src/libstore/build/worker\.cc$'' + ''^src/libstore/build/worker\.hh$'' + ''^src/libstore/builtins/fetchurl\.cc$'' + ''^src/libstore/builtins/unpack-channel\.cc$'' + ''^src/libstore/gc\.cc$'' + ''^src/libstore/local-overlay-store\.cc$'' + ''^src/libstore/local-overlay-store\.hh$'' + ''^src/libstore/local-store\.cc$'' + ''^src/libstore/local-store\.hh$'' + ''^src/libstore/unix/user-lock\.cc$'' + ''^src/libstore/unix/user-lock\.hh$'' + ''^src/libstore/optimise-store\.cc$'' + ''^src/libstore/unix/pathlocks\.cc$'' + ''^src/libstore/posix-fs-canonicalise\.cc$'' + ''^src/libstore/posix-fs-canonicalise\.hh$'' + ''^src/libstore/uds-remote-store\.cc$'' + ''^src/libstore/uds-remote-store\.hh$'' + ''^src/libstore/windows/build\.cc$'' + ''^src/libstore/worker-protocol-impl\.hh$'' + ''^src/libstore/worker-protocol\.cc$'' + ''^src/libstore/worker-protocol\.hh$'' + ''^src/libutil-c/nix_api_util_internal\.h$'' + ''^src/libutil/archive\.cc$'' + ''^src/libutil/archive\.hh$'' + ''^src/libutil/args\.cc$'' + ''^src/libutil/args\.hh$'' + ''^src/libutil/args/root\.hh$'' + ''^src/libutil/callback\.hh$'' + ''^src/libutil/canon-path\.cc$'' + ''^src/libutil/canon-path\.hh$'' + ''^src/libutil/chunked-vector\.hh$'' + ''^src/libutil/closure\.hh$'' + ''^src/libutil/comparator\.hh$'' + ''^src/libutil/compute-levels\.cc$'' + ''^src/libutil/config-impl\.hh$'' + ''^src/libutil/config\.cc$'' + ''^src/libutil/config\.hh$'' + ''^src/libutil/current-process\.cc$'' + ''^src/libutil/current-process\.hh$'' + ''^src/libutil/english\.cc$'' + ''^src/libutil/english\.hh$'' + ''^src/libutil/environment-variables\.cc$'' + ''^src/libutil/error\.cc$'' + ''^src/libutil/error\.hh$'' + ''^src/libutil/exit\.hh$'' + ''^src/libutil/experimental-features\.cc$'' + ''^src/libutil/experimental-features\.hh$'' + ''^src/libutil/file-content-address\.cc$'' + ''^src/libutil/file-content-address\.hh$'' + ''^src/libutil/file-descriptor\.cc$'' + ''^src/libutil/file-descriptor\.hh$'' + ''^src/libutil/file-path-impl\.hh$'' + ''^src/libutil/file-path\.hh$'' + ''^src/libutil/file-system\.cc$'' + ''^src/libutil/file-system\.hh$'' + ''^src/libutil/finally\.hh$'' + ''^src/libutil/fmt\.hh$'' + ''^src/libutil/fs-sink\.cc$'' + ''^src/libutil/fs-sink\.hh$'' + ''^src/libutil/git\.cc$'' + ''^src/libutil/git\.hh$'' + ''^src/libutil/hash\.cc$'' + ''^src/libutil/hash\.hh$'' + ''^src/libutil/hilite\.cc$'' + ''^src/libutil/hilite\.hh$'' + ''^src/libutil/source-accessor\.hh$'' + ''^src/libutil/json-impls\.hh$'' + ''^src/libutil/json-utils\.cc$'' + ''^src/libutil/json-utils\.hh$'' + ''^src/libutil/linux/cgroup\.cc$'' + ''^src/libutil/linux/namespaces\.cc$'' + ''^src/libutil/logging\.cc$'' + ''^src/libutil/logging\.hh$'' + ''^src/libutil/lru-cache\.hh$'' + ''^src/libutil/memory-source-accessor\.cc$'' + ''^src/libutil/memory-source-accessor\.hh$'' + ''^src/libutil/pool\.hh$'' + ''^src/libutil/position\.cc$'' + ''^src/libutil/position\.hh$'' + ''^src/libutil/posix-source-accessor\.cc$'' + ''^src/libutil/posix-source-accessor\.hh$'' + ''^src/libutil/processes\.hh$'' + ''^src/libutil/ref\.hh$'' + ''^src/libutil/references\.cc$'' + ''^src/libutil/references\.hh$'' + ''^src/libutil/regex-combinators\.hh$'' + ''^src/libutil/serialise\.cc$'' + ''^src/libutil/serialise\.hh$'' + ''^src/libutil/signals\.hh$'' + ''^src/libutil/signature/local-keys\.cc$'' + ''^src/libutil/signature/local-keys\.hh$'' + ''^src/libutil/signature/signer\.cc$'' + ''^src/libutil/signature/signer\.hh$'' + ''^src/libutil/source-accessor\.cc$'' + ''^src/libutil/source-accessor\.hh$'' + ''^src/libutil/source-path\.cc$'' + ''^src/libutil/source-path\.hh$'' + ''^src/libutil/split\.hh$'' + ''^src/libutil/suggestions\.cc$'' + ''^src/libutil/suggestions\.hh$'' + ''^src/libutil/sync\.hh$'' + ''^src/libutil/terminal\.cc$'' + ''^src/libutil/terminal\.hh$'' + ''^src/libutil/thread-pool\.cc$'' + ''^src/libutil/thread-pool\.hh$'' + ''^src/libutil/topo-sort\.hh$'' + ''^src/libutil/types\.hh$'' + ''^src/libutil/unix/file-descriptor\.cc$'' + ''^src/libutil/unix/file-path\.cc$'' + ''^src/libutil/unix/monitor-fd\.hh$'' + ''^src/libutil/unix/processes\.cc$'' + ''^src/libutil/unix/signals-impl\.hh$'' + ''^src/libutil/unix/signals\.cc$'' + ''^src/libutil/unix-domain-socket\.cc$'' + ''^src/libutil/unix/users\.cc$'' + ''^src/libutil/url-parts\.hh$'' + ''^src/libutil/url\.cc$'' + ''^src/libutil/url\.hh$'' + ''^src/libutil/users\.cc$'' + ''^src/libutil/users\.hh$'' + ''^src/libutil/util\.cc$'' + ''^src/libutil/util\.hh$'' + ''^src/libutil/variant-wrapper\.hh$'' + ''^src/libutil/windows/environment-variables\.cc$'' + ''^src/libutil/windows/file-descriptor\.cc$'' + ''^src/libutil/windows/file-path\.cc$'' + ''^src/libutil/windows/processes\.cc$'' + ''^src/libutil/windows/users\.cc$'' + ''^src/libutil/windows/windows-error\.cc$'' + ''^src/libutil/windows/windows-error\.hh$'' + ''^src/libutil/xml-writer\.cc$'' + ''^src/libutil/xml-writer\.hh$'' + ''^src/nix-build/nix-build\.cc$'' + ''^src/nix-channel/nix-channel\.cc$'' + ''^src/nix-collect-garbage/nix-collect-garbage\.cc$'' + ''^src/nix-env/buildenv.nix$'' + ''^src/nix-env/nix-env\.cc$'' + ''^src/nix-env/user-env\.cc$'' + ''^src/nix-env/user-env\.hh$'' + ''^src/nix-instantiate/nix-instantiate\.cc$'' + ''^src/nix-store/dotgraph\.cc$'' + ''^src/nix-store/graphml\.cc$'' + ''^src/nix-store/nix-store\.cc$'' + ''^src/nix/add-to-store\.cc$'' + ''^src/nix/app\.cc$'' + ''^src/nix/build\.cc$'' + ''^src/nix/bundle\.cc$'' + ''^src/nix/cat\.cc$'' + ''^src/nix/config-check\.cc$'' + ''^src/nix/config\.cc$'' + ''^src/nix/copy\.cc$'' + ''^src/nix/derivation-add\.cc$'' + ''^src/nix/derivation-show\.cc$'' + ''^src/nix/derivation\.cc$'' + ''^src/nix/develop\.cc$'' + ''^src/nix/diff-closures\.cc$'' + ''^src/nix/dump-path\.cc$'' + ''^src/nix/edit\.cc$'' + ''^src/nix/eval\.cc$'' + ''^src/nix/flake\.cc$'' + ''^src/nix/fmt\.cc$'' + ''^src/nix/hash\.cc$'' + ''^src/nix/log\.cc$'' + ''^src/nix/ls\.cc$'' + ''^src/nix/main\.cc$'' + ''^src/nix/make-content-addressed\.cc$'' + ''^src/nix/nar\.cc$'' + ''^src/nix/optimise-store\.cc$'' + ''^src/nix/path-from-hash-part\.cc$'' + ''^src/nix/path-info\.cc$'' + ''^src/nix/prefetch\.cc$'' + ''^src/nix/profile\.cc$'' + ''^src/nix/realisation\.cc$'' + ''^src/nix/registry\.cc$'' + ''^src/nix/repl\.cc$'' + ''^src/nix/run\.cc$'' + ''^src/nix/run\.hh$'' + ''^src/nix/search\.cc$'' + ''^src/nix/sigs\.cc$'' + ''^src/nix/store-copy-log\.cc$'' + ''^src/nix/store-delete\.cc$'' + ''^src/nix/store-gc\.cc$'' + ''^src/nix/store-info\.cc$'' + ''^src/nix/store-repair\.cc$'' + ''^src/nix/store\.cc$'' + ''^src/nix/unix/daemon\.cc$'' + ''^src/nix/upgrade-nix\.cc$'' + ''^src/nix/verify\.cc$'' + ''^src/nix/why-depends\.cc$'' + + ''^tests/functional/plugins/plugintest\.cc'' + ''^tests/functional/test-libstoreconsumer/main\.cc'' + ''^tests/nixos/ca-fd-leak/sender\.c'' + ''^tests/nixos/ca-fd-leak/smuggler\.c'' + ''^tests/nixos/user-sandboxing/attacker\.c'' + ''^tests/unit/libexpr-support/tests/libexpr\.hh'' + ''^tests/unit/libexpr-support/tests/value/context\.cc'' + ''^tests/unit/libexpr-support/tests/value/context\.hh'' + ''^tests/unit/libexpr/derived-path\.cc'' + ''^tests/unit/libexpr/error_traces\.cc'' + ''^tests/unit/libexpr/eval\.cc'' + ''^tests/unit/libexpr/json\.cc'' + ''^tests/unit/libexpr/main\.cc'' + ''^tests/unit/libexpr/primops\.cc'' + ''^tests/unit/libexpr/search-path\.cc'' + ''^tests/unit/libexpr/trivial\.cc'' + ''^tests/unit/libexpr/value/context\.cc'' + ''^tests/unit/libexpr/value/print\.cc'' + ''^tests/unit/libfetchers/public-key\.cc'' + ''^tests/unit/libflake/flakeref\.cc'' + ''^tests/unit/libflake/url-name\.cc'' + ''^tests/unit/libstore-support/tests/derived-path\.cc'' + ''^tests/unit/libstore-support/tests/derived-path\.hh'' + ''^tests/unit/libstore-support/tests/nix_api_store\.hh'' + ''^tests/unit/libstore-support/tests/outputs-spec\.cc'' + ''^tests/unit/libstore-support/tests/outputs-spec\.hh'' + ''^tests/unit/libstore-support/tests/path\.cc'' + ''^tests/unit/libstore-support/tests/path\.hh'' + ''^tests/unit/libstore-support/tests/protocol\.hh'' + ''^tests/unit/libstore/common-protocol\.cc'' + ''^tests/unit/libstore/content-address\.cc'' + ''^tests/unit/libstore/derivation\.cc'' + ''^tests/unit/libstore/derived-path\.cc'' + ''^tests/unit/libstore/downstream-placeholder\.cc'' + ''^tests/unit/libstore/machines\.cc'' + ''^tests/unit/libstore/nar-info-disk-cache\.cc'' + ''^tests/unit/libstore/nar-info\.cc'' + ''^tests/unit/libstore/outputs-spec\.cc'' + ''^tests/unit/libstore/path-info\.cc'' + ''^tests/unit/libstore/path\.cc'' + ''^tests/unit/libstore/serve-protocol\.cc'' + ''^tests/unit/libstore/worker-protocol\.cc'' + ''^tests/unit/libutil-support/tests/characterization\.hh'' + ''^tests/unit/libutil-support/tests/hash\.cc'' + ''^tests/unit/libutil-support/tests/hash\.hh'' + ''^tests/unit/libutil/args\.cc'' + ''^tests/unit/libutil/canon-path\.cc'' + ''^tests/unit/libutil/chunked-vector\.cc'' + ''^tests/unit/libutil/closure\.cc'' + ''^tests/unit/libutil/compression\.cc'' + ''^tests/unit/libutil/config\.cc'' + ''^tests/unit/libutil/file-content-address\.cc'' + ''^tests/unit/libutil/git\.cc'' + ''^tests/unit/libutil/hash\.cc'' + ''^tests/unit/libutil/hilite\.cc'' + ''^tests/unit/libutil/json-utils\.cc'' + ''^tests/unit/libutil/logging\.cc'' + ''^tests/unit/libutil/lru-cache\.cc'' + ''^tests/unit/libutil/pool\.cc'' + ''^tests/unit/libutil/references\.cc'' + ''^tests/unit/libutil/suggestions\.cc'' + ''^tests/unit/libutil/tests\.cc'' + ''^tests/unit/libutil/url\.cc'' + ''^tests/unit/libutil/xml-writer\.cc'' + ]; + }; + shellcheck = { + enable = true; + excludes = [ + # We haven't linted these files yet + ''^config/install-sh$'' + ''^misc/bash/completion\.sh$'' + ''^misc/fish/completion\.fish$'' + ''^misc/zsh/completion\.zsh$'' + ''^scripts/create-darwin-volume\.sh$'' + ''^scripts/install-darwin-multi-user\.sh$'' + ''^scripts/install-multi-user\.sh$'' + ''^scripts/install-nix-from-closure\.sh$'' + ''^scripts/install-systemd-multi-user\.sh$'' + ''^src/nix/get-env\.sh$'' + ''^tests/functional/build\.sh$'' + ''^tests/functional/ca/build-dry\.sh$'' + ''^tests/functional/ca/build-with-garbage-path\.sh$'' + ''^tests/functional/ca/common\.sh$'' + ''^tests/functional/ca/concurrent-builds\.sh$'' + ''^tests/functional/ca/eval-store\.sh$'' + ''^tests/functional/ca/gc\.sh$'' + ''^tests/functional/ca/import-derivation\.sh$'' + ''^tests/functional/ca/new-build-cmd\.sh$'' + ''^tests/functional/ca/nix-shell\.sh$'' + ''^tests/functional/ca/post-hook\.sh$'' + ''^tests/functional/ca/recursive\.sh$'' + ''^tests/functional/ca/repl\.sh$'' + ''^tests/functional/ca/selfref-gc\.sh$'' + ''^tests/functional/ca/why-depends\.sh$'' + ''^tests/functional/characterisation-test-infra\.sh$'' + ''^tests/functional/check\.sh$'' + ''^tests/functional/common/vars-and-functions\.sh$'' + ''^tests/functional/completions\.sh$'' + ''^tests/functional/compute-levels\.sh$'' + ''^tests/functional/config\.sh$'' + ''^tests/functional/db-migration\.sh$'' + ''^tests/functional/debugger\.sh$'' + ''^tests/functional/dependencies\.builder0\.sh$'' + ''^tests/functional/dependencies\.sh$'' + ''^tests/functional/dump-db\.sh$'' + ''^tests/functional/dyn-drv/build-built-drv\.sh$'' + ''^tests/functional/dyn-drv/common\.sh$'' + ''^tests/functional/dyn-drv/dep-built-drv\.sh$'' + ''^tests/functional/dyn-drv/eval-outputOf\.sh$'' + ''^tests/functional/dyn-drv/old-daemon-error-hack\.sh$'' + ''^tests/functional/dyn-drv/recursive-mod-json\.sh$'' + ''^tests/functional/eval-store\.sh$'' + ''^tests/functional/eval\.sh$'' + ''^tests/functional/export-graph\.sh$'' + ''^tests/functional/export\.sh$'' + ''^tests/functional/extra-sandbox-profile\.sh$'' + ''^tests/functional/fetchClosure\.sh$'' + ''^tests/functional/fetchGit\.sh$'' + ''^tests/functional/fetchGitRefs\.sh$'' + ''^tests/functional/fetchGitSubmodules\.sh$'' + ''^tests/functional/fetchGitVerification\.sh$'' + ''^tests/functional/fetchMercurial\.sh$'' + ''^tests/functional/fetchurl\.sh$'' + ''^tests/functional/fixed\.builder1\.sh$'' + ''^tests/functional/fixed\.builder2\.sh$'' + ''^tests/functional/fixed\.sh$'' + ''^tests/functional/flakes/absolute-paths\.sh$'' + ''^tests/functional/flakes/check\.sh$'' + ''^tests/functional/flakes/common\.sh$'' + ''^tests/functional/flakes/config\.sh$'' + ''^tests/functional/flakes/develop\.sh$'' + ''^tests/functional/flakes/flakes\.sh$'' + ''^tests/functional/flakes/follow-paths\.sh$'' + ''^tests/functional/flakes/prefetch\.sh$'' + ''^tests/functional/flakes/run\.sh$'' + ''^tests/functional/flakes/show\.sh$'' + ''^tests/functional/fmt\.sh$'' + ''^tests/functional/fmt\.simple\.sh$'' + ''^tests/functional/gc-auto\.sh$'' + ''^tests/functional/gc-concurrent\.builder\.sh$'' + ''^tests/functional/gc-concurrent\.sh$'' + ''^tests/functional/gc-concurrent2\.builder\.sh$'' + ''^tests/functional/gc-non-blocking\.sh$'' + ''^tests/functional/gc\.sh$'' + ''^tests/functional/git-hashing/common\.sh$'' + ''^tests/functional/git-hashing/simple\.sh$'' + ''^tests/functional/hash-convert\.sh$'' + ''^tests/functional/help\.sh$'' + ''^tests/functional/impure-derivations\.sh$'' + ''^tests/functional/impure-env\.sh$'' + ''^tests/functional/impure-eval\.sh$'' + ''^tests/functional/install-darwin\.sh$'' + ''^tests/functional/lang\.sh$'' + ''^tests/functional/legacy-ssh-store\.sh$'' + ''^tests/functional/linux-sandbox\.sh$'' + ''^tests/functional/local-overlay-store/add-lower-inner\.sh$'' + ''^tests/functional/local-overlay-store/add-lower\.sh$'' + ''^tests/functional/local-overlay-store/bad-uris\.sh$'' + ''^tests/functional/local-overlay-store/build-inner\.sh$'' + ''^tests/functional/local-overlay-store/build\.sh$'' + ''^tests/functional/local-overlay-store/check-post-init-inner\.sh$'' + ''^tests/functional/local-overlay-store/check-post-init\.sh$'' + ''^tests/functional/local-overlay-store/common\.sh$'' + ''^tests/functional/local-overlay-store/delete-duplicate-inner\.sh$'' + ''^tests/functional/local-overlay-store/delete-duplicate\.sh$'' + ''^tests/functional/local-overlay-store/delete-refs-inner\.sh$'' + ''^tests/functional/local-overlay-store/delete-refs\.sh$'' + ''^tests/functional/local-overlay-store/gc-inner\.sh$'' + ''^tests/functional/local-overlay-store/gc\.sh$'' + ''^tests/functional/local-overlay-store/optimise-inner\.sh$'' + ''^tests/functional/local-overlay-store/optimise\.sh$'' + ''^tests/functional/local-overlay-store/redundant-add-inner\.sh$'' + ''^tests/functional/local-overlay-store/redundant-add\.sh$'' + ''^tests/functional/local-overlay-store/remount\.sh$'' + ''^tests/functional/local-overlay-store/stale-file-handle-inner\.sh$'' + ''^tests/functional/local-overlay-store/stale-file-handle\.sh$'' + ''^tests/functional/local-overlay-store/verify-inner\.sh$'' + ''^tests/functional/local-overlay-store/verify\.sh$'' + ''^tests/functional/logging\.sh$'' + ''^tests/functional/misc\.sh$'' + ''^tests/functional/multiple-outputs\.sh$'' + ''^tests/functional/nar-access\.sh$'' + ''^tests/functional/nested-sandboxing\.sh$'' + ''^tests/functional/nested-sandboxing/command\.sh$'' + ''^tests/functional/nix-build\.sh$'' + ''^tests/functional/nix-channel\.sh$'' + ''^tests/functional/nix-collect-garbage-d\.sh$'' + ''^tests/functional/nix-copy-ssh-common\.sh$'' + ''^tests/functional/nix-copy-ssh-ng\.sh$'' + ''^tests/functional/nix-copy-ssh\.sh$'' + ''^tests/functional/nix-daemon-untrusting\.sh$'' + ''^tests/functional/nix-profile\.sh$'' + ''^tests/functional/nix-shell\.sh$'' + ''^tests/functional/nix_path\.sh$'' + ''^tests/functional/optimise-store\.sh$'' + ''^tests/functional/output-normalization\.sh$'' + ''^tests/functional/parallel\.builder\.sh$'' + ''^tests/functional/parallel\.sh$'' + ''^tests/functional/pass-as-file\.sh$'' + ''^tests/functional/path-from-hash-part\.sh$'' + ''^tests/functional/path-info\.sh$'' + ''^tests/functional/placeholders\.sh$'' + ''^tests/functional/plugins\.sh$'' + ''^tests/functional/post-hook\.sh$'' + ''^tests/functional/pure-eval\.sh$'' + ''^tests/functional/push-to-store-old\.sh$'' + ''^tests/functional/push-to-store\.sh$'' + ''^tests/functional/read-only-store\.sh$'' + ''^tests/functional/readfile-context\.sh$'' + ''^tests/functional/recursive\.sh$'' + ''^tests/functional/referrers\.sh$'' + ''^tests/functional/remote-store\.sh$'' + ''^tests/functional/repair\.sh$'' + ''^tests/functional/restricted\.sh$'' + ''^tests/functional/search\.sh$'' + ''^tests/functional/secure-drv-outputs\.sh$'' + ''^tests/functional/selfref-gc\.sh$'' + ''^tests/functional/shell\.sh$'' + ''^tests/functional/shell\.shebang\.sh$'' + ''^tests/functional/signing\.sh$'' + ''^tests/functional/simple\.builder\.sh$'' + ''^tests/functional/simple\.sh$'' + ''^tests/functional/ssh-relay\.sh$'' + ''^tests/functional/store-info\.sh$'' + ''^tests/functional/structured-attrs\.sh$'' + ''^tests/functional/substitute-with-invalid-ca\.sh$'' + ''^tests/functional/suggestions\.sh$'' + ''^tests/functional/supplementary-groups\.sh$'' + ''^tests/functional/tarball\.sh$'' + ''^tests/functional/test-infra\.sh$'' + ''^tests/functional/test-libstoreconsumer\.sh$'' + ''^tests/functional/timeout\.sh$'' + ''^tests/functional/toString-path\.sh$'' + ''^tests/functional/user-envs-migration\.sh$'' + ''^tests/functional/user-envs-test-case\.sh$'' + ''^tests/functional/user-envs\.builder\.sh$'' + ''^tests/functional/user-envs\.sh$'' + ''^tests/functional/why-depends\.sh$'' + ''^tests/functional/zstd\.sh$'' + ''^tests/unit/libutil/data/git/check-data\.sh$'' + ]; + }; + # TODO: nixfmt, https://github.com/NixOS/nixfmt/issues/153 + }; + }; + }; + + # We'll be pulling from this in the main flake + flake.getSystem = getSystem; +} diff --git a/maintainers/local.mk b/maintainers/local.mk new file mode 100644 index 000000000..88d594d67 --- /dev/null +++ b/maintainers/local.mk @@ -0,0 +1,15 @@ + +.PHONY: format +print-top-help += echo ' format: Format source code' + +# This uses the cached .pre-commit-hooks.yaml file +format: + @if ! type -p pre-commit &>/dev/null; then \ + echo "make format: pre-commit not found. Please use \`nix develop\`."; \ + exit 1; \ + fi; \ + if test -z "$$_NIX_PRE_COMMIT_HOOKS_CONFIG"; then \ + echo "make format: _NIX_PRE_COMMIT_HOOKS_CONFIG not set. Please use \`nix develop\`."; \ + exit 1; \ + fi; \ + pre-commit run --config $$_NIX_PRE_COMMIT_HOOKS_CONFIG --all-files diff --git a/maintainers/release-credits b/maintainers/release-credits new file mode 100755 index 000000000..7a5c87d7d --- /dev/null +++ b/maintainers/release-credits @@ -0,0 +1,184 @@ +#!/usr/bin/env nix +# vim: set filetype=python: +#!nix develop --impure --expr +#!nix `` +#!nix let flake = builtins.getFlake ("git+file://" + toString ../.); +#!nix pkgs = flake.inputs.nixpkgs.legacyPackages.${builtins.currentSystem}; +#!nix in pkgs.mkShell { nativeBuildInputs = [ +#!nix (pkgs.python3.withPackages (ps: with ps; [ requests ])) +#!nix ]; } +#!nix `` --command python3 + +# This script lists out the contributors for a given release. +# It must be run from the root of the Nix repository. + +import os +import sys +import json +import requests + +github_token = os.environ.get("GITHUB_TOKEN") +if not github_token: + print("GITHUB_TOKEN is not set. If you hit the rate limit, set it", file=sys.stderr) + # Might be ok, as we have a cache. + # raise ValueError("GITHUB_TOKEN must be set") + +# 1. Read the current version in .version +version = os.environ.get("VERSION") +if not version: + version = open(".version").read().strip() + +print(f"Generating release credits for Nix {version}", file=sys.stderr) + +# 2. Compute previous version +vcomponents = version.split(".") +if len(vcomponents) >= 2: + prev_version = f"{vcomponents[0]}.{int(vcomponents[1])-1}.0" +else: + raise ValueError(".version must have at least two components") + +# For unreleased versions +endref = "HEAD" +# For older releases +# endref = version + +# 2. Find the merge base between the current version and the previous version +mergeBase = os.popen(f"git merge-base {prev_version} {endref}").read().strip() +print(f"Merge base between {prev_version} and {endref} is {mergeBase}", file=sys.stderr) + +# 3. Find the date of the merge base +mergeBaseDate = os.popen(f"git show -s --format=%ci {mergeBase}").read().strip()[0:10] +print(f"Merge base date is {mergeBaseDate}", file=sys.stderr) + +# 4. Get the commits between the merge base and the current version + +def get_commits(): + raw = os.popen(f"git log --pretty=format:'%H\t%an\t%ae' {mergeBase}..{endref}").read().strip() + lines = raw.split("\n") + return [ { "hash": items[0], "author": items[1], "email": items[2] } + for line in lines + for items in (line.split("\t"),) + ] + +def commits_to_first_commit_by_email(commits): + by_email = dict() + for commit in commits: + email = commit["email"] + if email not in by_email: + by_email[email] = commit + return by_email + + +samples = commits_to_first_commit_by_email(get_commits()) + +# For quick testing, only pick two samples from the dict +# samples = dict(list(samples.items())[:2]) + +# Query the GitHub API to get handle +def get_github_commit(commit): + url = f"https://api.github.com/repos/NixOS/nix/commits/{commit['hash']}" + headers = {'Authorization': f'token {github_token}'} + response = requests.get(url, headers=headers) + response.raise_for_status() + return response.json() + +class Cache: + def __init__(self, filename, require = True): + self.filename = filename + try: + with open(filename, "r") as f: + self.values = json.load(f) + except FileNotFoundError: + if require: + raise + self.values = dict() + def save(self): + with open(self.filename, "w") as f: + json.dump(self.values, f, indent=4) + print(f"Saved cache to {self.filename}", file=sys.stderr) + +# The email to handle cache maps email addresses to either +# - a handle (string) +# - None (if no handle was found) +email_to_handle_cache = Cache("maintainers/data/release-credits-email-to-handle.json") + +handles = set() +emails = dict() + +for sample in samples: + s = samples[sample] + email = s["email"] + if not email in email_to_handle_cache.values: + print(f"Querying GitHub API for {s['hash']}, to get handle for {s['email']}") + ghc = get_github_commit(samples[sample]) + gha = ghc["author"] + if gha and gha["login"]: + handle = gha["login"] + print(f"Handle: {handle}") + email_to_handle_cache.values[email] = handle + else: + print(f"Found no handle for {s['email']}") + email_to_handle_cache.values[email] = None + handle = email_to_handle_cache.values[email] + if handle is not None: + handles.add(handle) + else: + emails[email] = s["author"] + +# print(email_to_handle_cache.values) + +email_to_handle_cache.save() + +handle_to_name_cache = Cache("maintainers/data/release-credits-handle-to-name.json") + +print(f"Found {len(handles)} handles", file=sys.stderr) + +for handle in handles: + if not handle in handle_to_name_cache.values: + print(f"Querying GitHub API for {handle}, to get name", file=sys.stderr) + url = f"https://api.github.com/users/{handle}" + headers = {'Authorization': f'token {github_token}'} + response = requests.get(url, headers=headers) + response.raise_for_status() + user = response.json() + name = user["name"] + print(f"Name: {name}", file=sys.stderr) + handle_to_name_cache.values[handle] = name + +handle_to_name_cache.save() + +entries = list() + +def is_bot(handle): + return handle.endswith("[bot]") + +for handle in handles: + name = handle_to_name_cache.values[handle] + if name is None: + # This way it looks more regular + name = handle + + # Note to bots: I'm only doing this because it would distract humans, and + # therefore delay your creation. I'm sorry; it's for your benefit. + if not is_bot(handle): + entries += [ f"- {name} [**(@{handle})**](https://github.com/{handle})" ] + +def shuffle(entries): + salt = os.urandom(16) + return sorted(entries, key=lambda x: hash((x, salt))) + +# Fair ordering is undecidable +entries = shuffle(entries) + +# For a sanity check, we could sort the entries by handle instead. +# entries = sorted(entries) + +print("") +print(f"This release was made possible by the following {len(entries)} contributors:") +print("") + +for entry in entries: + print(entry) + +for email in emails: + print(f"- {emails[email]}") diff --git a/maintainers/release-notes b/maintainers/release-notes index 2d84485c1..c0c4ee734 100755 --- a/maintainers/release-notes +++ b/maintainers/release-notes @@ -1,5 +1,6 @@ #!/usr/bin/env nix -#!nix shell .#changelog-d-nix --command bash +# vim: set filetype=bash: +#!nix shell .#changelog-d --command bash # --- CONFIGURATION --- @@ -151,6 +152,13 @@ section_title="Release $version_full ($DATE)" echo "# $section_title" echo changelog-d doc/manual/rl-next | sed -e 's/ *$//' + + if ! $IS_PATCH; then + echo + echo "# Contributors" + echo + VERSION=$version_full ./maintainers/release-credits + fi ) | tee -a $file log "Wrote $file" diff --git a/maintainers/release-process.md b/maintainers/release-process.md index da6886ea9..7a2b3c0a7 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -39,6 +39,10 @@ release: * Proof-read / edit / rearrange the release notes if needed. Breaking changes and highlights should go to the top. +* Run `maintainers/release-credits` to make sure the credits script works + and produces a sensible output. Some emails might not automatically map to + a GitHub handle. + * Push. ```console diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 4e2c379f0..731988568 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -11,6 +11,8 @@ use JSON::PP; use LWP::UserAgent; use Net::Amazon::S3; +delete $ENV{'shell'}; # shut up a LWP::UserAgent.pm warning + my $evalId = $ARGV[0] or die "Usage: $0 EVAL-ID\n"; my $releasesBucketName = "nix-releases"; @@ -36,11 +38,11 @@ sub fetch { my $evalUrl = "https://hydra.nixos.org/eval/$evalId"; my $evalInfo = decode_json(fetch($evalUrl, 'application/json')); #print Dumper($evalInfo); -my $flakeUrl = $evalInfo->{flake} or die; -my $flakeInfo = decode_json(`nix flake metadata --json "$flakeUrl"` or die); -my $nixRev = $flakeInfo->{revision} or die; +my $flakeUrl = $evalInfo->{flake}; +my $flakeInfo = decode_json(`nix flake metadata --json "$flakeUrl"` or die) if $flakeUrl; +my $nixRev = ($flakeInfo ? $flakeInfo->{revision} : $evalInfo->{jobsetevalinputs}->{nix}->{revision}) or die; -my $buildInfo = decode_json(fetch("$evalUrl/job/build.x86_64-linux", 'application/json')); +my $buildInfo = decode_json(fetch("$evalUrl/job/build.nix.x86_64-linux", 'application/json')); #print Dumper($buildInfo); my $releaseName = $buildInfo->{nixname}; @@ -83,12 +85,19 @@ my $channelsBucket = $s3_us->bucket($channelsBucketName) or die; sub getStorePath { my ($jobName, $output) = @_; my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json')); - return $buildInfo->{buildoutputs}->{$output or "out"}->{path} or die "cannot get store path for '$jobName'"; + return $buildInfo->{buildoutputs}->{$output or "out"}->{path} // die "cannot get store path for '$jobName'"; } sub copyManual { - my $manual = getStorePath("build.x86_64-linux", "doc"); - print "$manual\n"; + my $manual; + eval { + $manual = getStorePath("build.nix.x86_64-linux", "doc"); + }; + if ($@) { + warn "$@"; + return; + } + print "Manual: $manual\n"; my $manualNar = "$tmpDir/$releaseName-manual.nar.xz"; print "$manualNar\n"; @@ -154,19 +163,34 @@ downloadFile("binaryTarball.x86_64-linux", "1"); downloadFile("binaryTarball.aarch64-linux", "1"); downloadFile("binaryTarball.x86_64-darwin", "1"); downloadFile("binaryTarball.aarch64-darwin", "1"); -downloadFile("binaryTarballCross.x86_64-linux.armv6l-unknown-linux-gnueabihf", "1"); -downloadFile("binaryTarballCross.x86_64-linux.armv7l-unknown-linux-gnueabihf", "1"); +eval { + downloadFile("binaryTarballCross.x86_64-linux.armv6l-unknown-linux-gnueabihf", "1"); +}; +warn "$@" if $@; +eval { + downloadFile("binaryTarballCross.x86_64-linux.armv7l-unknown-linux-gnueabihf", "1"); +}; +warn "$@" if $@; +eval { + downloadFile("binaryTarballCross.x86_64-linux.riscv64-unknown-linux-gnu", "1"); +}; +warn "$@" if $@; downloadFile("installerScript", "1"); # Upload docker images to dockerhub. my $dockerManifest = ""; my $dockerManifestLatest = ""; +my $haveDocker = 0; for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) { my $system = $platforms->[0]; my $dockerPlatform = $platforms->[1]; my $fn = "nix-$version-docker-image-$dockerPlatform.tar.gz"; - downloadFile("dockerImage.$system", "1", $fn); + eval { + downloadFile("dockerImage.$system", "1", $fn); + }; + die "$@" if $@; + $haveDocker = 1; print STDERR "loading docker image for $dockerPlatform...\n"; system("docker load -i $tmpDir/$fn") == 0 or die; @@ -194,31 +218,34 @@ for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) { $dockerManifestLatest .= " --amend $latestTag" } -print STDERR "creating multi-platform docker manifest...\n"; -system("docker manifest rm nixos/nix:$version"); -system("docker manifest create nixos/nix:$version $dockerManifest") == 0 or die; -if ($isLatest) { - print STDERR "creating latest multi-platform docker manifest...\n"; - system("docker manifest rm nixos/nix:latest"); - system("docker manifest create nixos/nix:latest $dockerManifestLatest") == 0 or die; -} +if ($haveDocker) { + print STDERR "creating multi-platform docker manifest...\n"; + system("docker manifest rm nixos/nix:$version"); + system("docker manifest create nixos/nix:$version $dockerManifest") == 0 or die; + if ($isLatest) { + print STDERR "creating latest multi-platform docker manifest...\n"; + system("docker manifest rm nixos/nix:latest"); + system("docker manifest create nixos/nix:latest $dockerManifestLatest") == 0 or die; + } -print STDERR "pushing multi-platform docker manifest...\n"; -system("docker manifest push nixos/nix:$version") == 0 or die; + print STDERR "pushing multi-platform docker manifest...\n"; + system("docker manifest push nixos/nix:$version") == 0 or die; -if ($isLatest) { - print STDERR "pushing latest multi-platform docker manifest...\n"; - system("docker manifest push nixos/nix:latest") == 0 or die; + if ($isLatest) { + print STDERR "pushing latest multi-platform docker manifest...\n"; + system("docker manifest push nixos/nix:latest") == 0 or die; + } } # Upload nix-fallback-paths.nix. write_file("$tmpDir/fallback-paths.nix", "{\n" . - " x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" . - " i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" . - " aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" . - " x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" . - " aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" . + " x86_64-linux = \"" . getStorePath("build.nix.x86_64-linux") . "\";\n" . + " i686-linux = \"" . getStorePath("build.nix.i686-linux") . "\";\n" . + " aarch64-linux = \"" . getStorePath("build.nix.aarch64-linux") . "\";\n" . + " riscv64-linux = \"" . getStorePath("buildCross.nix.riscv64-unknown-linux-gnu.x86_64-linux") . "\";\n" . + " x86_64-darwin = \"" . getStorePath("build.nix.x86_64-darwin") . "\";\n" . + " aarch64-darwin = \"" . getStorePath("build.nix.aarch64-darwin") . "\";\n" . "}\n"); # Upload release files to S3. diff --git a/meson.build b/meson.build new file mode 100644 index 000000000..1554244ab --- /dev/null +++ b/meson.build @@ -0,0 +1,44 @@ +# This is just a stub project to include all the others as subprojects +# for development shell purposes + +project('nix-dev-shell', 'cpp', + version : files('.version'), + subproject_dir : 'src', +) + +# Internal Libraries +subproject('libutil') +subproject('libstore') +subproject('libfetchers') +subproject('libexpr') +subproject('libflake') +subproject('libmain') +subproject('libcmd') + +# Executables +subproject('nix') + +# Docs +subproject('internal-api-docs') +subproject('external-api-docs') + +# External C wrapper libraries +subproject('libutil-c') +subproject('libstore-c') +subproject('libexpr-c') +subproject('libmain-c') + +# Language Bindings +if not meson.is_cross_build() + subproject('perl') +endif + +# Testing +subproject('nix-util-test-support') +subproject('nix-util-tests') +subproject('nix-store-test-support') +subproject('nix-store-tests') +subproject('nix-fetchers-tests') +subproject('nix-expr-test-support') +subproject('nix-expr-tests') +subproject('nix-flake-tests') diff --git a/misc/changelog-d.cabal.nix b/misc/changelog-d.cabal.nix deleted file mode 100644 index 76f9353cd..000000000 --- a/misc/changelog-d.cabal.nix +++ /dev/null @@ -1,31 +0,0 @@ -{ mkDerivation, aeson, base, bytestring, cabal-install-parsers -, Cabal-syntax, containers, directory, filepath, frontmatter -, generic-lens-lite, lib, mtl, optparse-applicative, parsec, pretty -, regex-applicative, text, pkgs -}: -let rev = "f30f6969e9cd8b56242309639d58acea21c99d06"; -in -mkDerivation { - pname = "changelog-d"; - version = "0.1"; - src = pkgs.fetchurl { - name = "changelog-d-${rev}.tar.gz"; - url = "https://codeberg.org/roberth/changelog-d/archive/${rev}.tar.gz"; - hash = "sha256-8a2+i5u7YoszAgd5OIEW0eYUcP8yfhtoOIhLJkylYJ4="; - } // { inherit rev; }; - isLibrary = false; - isExecutable = true; - libraryHaskellDepends = [ - aeson base bytestring cabal-install-parsers Cabal-syntax containers - directory filepath frontmatter generic-lens-lite mtl parsec pretty - regex-applicative text - ]; - executableHaskellDepends = [ - base bytestring Cabal-syntax directory filepath - optparse-applicative - ]; - doHaddock = false; - description = "Concatenate changelog entries into a single one"; - license = lib.licenses.gpl3Plus; - mainProgram = "changelog-d"; -} diff --git a/misc/changelog-d.nix b/misc/changelog-d.nix deleted file mode 100644 index 1b20f4596..000000000 --- a/misc/changelog-d.nix +++ /dev/null @@ -1,31 +0,0 @@ -# Taken temporarily from -{ - callPackage, - lib, - haskell, - haskellPackages, -}: - -let - hsPkg = haskellPackages.callPackage ./changelog-d.cabal.nix { }; - - addCompletions = haskellPackages.generateOptparseApplicativeCompletions ["changelog-d"]; - - haskellModifications = - lib.flip lib.pipe [ - addCompletions - haskell.lib.justStaticExecutables - ]; - - mkDerivationOverrides = finalAttrs: oldAttrs: { - - version = oldAttrs.version + "-git-${lib.strings.substring 0 7 oldAttrs.src.rev}"; - - meta = oldAttrs.meta // { - homepage = "https://codeberg.org/roberth/changelog-d"; - maintainers = [ lib.maintainers.roberth ]; - }; - - }; -in - (haskellModifications hsPkg).overrideAttrs mkDerivationOverrides diff --git a/misc/systemv/nix-daemon b/misc/systemv/nix-daemon index fea537167..e8326f947 100755 --- a/misc/systemv/nix-daemon +++ b/misc/systemv/nix-daemon @@ -34,6 +34,7 @@ else fi # Source function library. +# shellcheck source=/dev/null . /etc/init.d/functions LOCKFILE=/var/lock/subsys/nix-daemon @@ -41,14 +42,20 @@ RUNDIR=/var/run/nix PIDFILE=${RUNDIR}/nix-daemon.pid RETVAL=0 -base=${0##*/} +# https://www.shellcheck.net/wiki/SC3004 +# Check if gettext exists +if ! type gettext > /dev/null 2>&1 +then + # If not, create a dummy function that returns the input verbatim + gettext() { printf '%s' "$1"; } +fi start() { mkdir -p ${RUNDIR} chown ${NIX_DAEMON_USER}:${NIX_DAEMON_USER} ${RUNDIR} - echo -n $"Starting nix daemon... " + printf '%s' "$(gettext 'Starting nix daemon... ')" daemonize -u $NIX_DAEMON_USER -p ${PIDFILE} $NIX_DAEMON_BIN $NIX_DAEMON_OPTS RETVAL=$? @@ -58,7 +65,7 @@ start() { } stop() { - echo -n $"Shutting down nix daemon: " + printf '%s' "$(gettext 'Shutting down nix daemon: ')" killproc -p ${PIDFILE} $NIX_DAEMON_BIN RETVAL=$? [ $RETVAL -eq 0 ] && rm -f ${LOCKFILE} ${PIDFILE} @@ -67,7 +74,7 @@ stop() { } reload() { - echo -n $"Reloading nix daemon... " + printf '%s' "$(gettext 'Reloading nix daemon... ')" killproc -p ${PIDFILE} $NIX_DAEMON_BIN -HUP RETVAL=$? echo @@ -105,7 +112,7 @@ case "$1" in fi ;; *) - echo $"Usage: $0 {start|stop|status|restart|condrestart}" + printf '%s' "$(gettext "Usage: $0 {start|stop|status|restart|condrestart}")" exit 2 ;; esac diff --git a/mk/common-test.sh b/mk/common-test.sh index 2783d293b..817422c40 100644 --- a/mk/common-test.sh +++ b/mk/common-test.sh @@ -1,27 +1,23 @@ +# shellcheck shell=bash + # Remove overall test dir (at most one of the two should match) and # remove file extension. -test_name=$(echo -n "$test" | sed \ - -e "s|^tests/unit/[^/]*/data/||" \ + +test_name=$(echo -n "${test?must be defined by caller (test runner)}" | sed \ + -e "s|^src/[^/]*-test/data/||" \ -e "s|^tests/functional/||" \ -e "s|\.sh$||" \ ) +# shellcheck disable=SC2016 TESTS_ENVIRONMENT=( "TEST_NAME=$test_name" 'NIX_REMOTE=' 'PS4=+(${BASH_SOURCE[0]-$0}:$LINENO) ' ) -: ${BASH:=/usr/bin/env bash} +read -r -a bash <<< "${BASH:-/usr/bin/env bash}" run () { - cd "$(dirname $1)" && env "${TESTS_ENVIRONMENT[@]}" $BASH -x -e -u -o pipefail $(basename $1) -} - -init_test () { - run "$init" 2>/dev/null > /dev/null -} - -run_test_proper () { - run "$test" + cd "$(dirname "$1")" && env "${TESTS_ENVIRONMENT[@]}" "${bash[@]}" -x -e -u -o pipefail "$(basename "$1")" } diff --git a/mk/compilation-database.mk b/mk/compilation-database.mk new file mode 100644 index 000000000..f69dc0de0 --- /dev/null +++ b/mk/compilation-database.mk @@ -0,0 +1,11 @@ +compile-commands-json-files := + +define write-compile-commands + _srcs := $$(sort $$(foreach src, $$($(1)_SOURCES), $$(src))) + + $(1)_COMPILE_COMMANDS_JSON := $$(addprefix $(buildprefix), $$(addsuffix .compile_commands.json, $$(basename $$(_srcs)))) + + compile-commands-json-files += $$($(1)_COMPILE_COMMANDS_JSON) + + clean-files += $$($(1)_COMPILE_COMMANDS_JSON) +endef diff --git a/mk/cxx-big-literal.mk b/mk/cxx-big-literal.mk index 85365df8e..d64a171c8 100644 --- a/mk/cxx-big-literal.mk +++ b/mk/cxx-big-literal.mk @@ -1,5 +1,5 @@ %.gen.hh: % - @echo 'R"foo(' >> $@.tmp + @echo 'R"__NIX_STR(' >> $@.tmp $(trace-gen) cat $< >> $@.tmp - @echo ')foo"' >> $@.tmp + @echo ')__NIX_STR"' >> $@.tmp @mv $@.tmp $@ diff --git a/mk/debug-test.sh b/mk/debug-test.sh index 52482c01e..0dd4406c3 100755 --- a/mk/debug-test.sh +++ b/mk/debug-test.sh @@ -3,12 +3,8 @@ set -eu -o pipefail test=$1 -init=${2-} dir="$(dirname "${BASH_SOURCE[0]}")" source "$dir/common-test.sh" -if [ -n "$init" ]; then - (init_test) -fi -run_test_proper +run "$test" diff --git a/mk/lib.mk b/mk/lib.mk index fe0add1c9..1e7af6ad5 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -68,6 +68,7 @@ include mk/patterns.mk include mk/templates.mk include mk/cxx-big-literal.mk include mk/tests.mk +include mk/compilation-database.mk # Include all sub-Makefiles. @@ -86,17 +87,23 @@ $(foreach script, $(bin-scripts), $(eval $(call install-program-in,$(script),$(b $(foreach script, $(bin-scripts), $(eval programs-list += $(script))) $(foreach script, $(noinst-scripts), $(eval programs-list += $(script))) $(foreach template, $(template-files), $(eval $(call instantiate-template,$(template)))) -install_test_init=tests/functional/init.sh $(foreach test, $(install-tests), \ - $(eval $(call run-test,$(test),$(install_test_init))) \ + $(eval $(call run-test,$(test))) \ $(eval installcheck: $(test).test)) $(foreach test-group, $(install-tests-groups), \ - $(eval $(call run-test-group,$(test-group),$(install_test_init))) \ + $(eval $(call run-test-group,$(test-group))) \ $(eval installcheck: $(test-group).test-group) \ $(foreach test, $($(test-group)-tests), \ - $(eval $(call run-test,$(test),$(install_test_init))) \ + $(eval $(call run-test,$(test))) \ $(eval $(test-group).test-group: $(test).test))) +# Compilation database. +$(foreach lib, $(libraries), $(eval $(call write-compile-commands,$(lib)))) +$(foreach prog, $(programs), $(eval $(call write-compile-commands,$(prog)))) + +compile_commands.json: $(compile-commands-json-files) + @jq --slurp '.' $^ >$@ + # Include makefiles requiring built programs. $(foreach mf, $(makefiles-late), $(eval $(call include-sub-makefile,$(mf)))) diff --git a/mk/patterns.mk b/mk/patterns.mk index c81150260..4caa2039e 100644 --- a/mk/patterns.mk +++ b/mk/patterns.mk @@ -1,11 +1,41 @@ + +# These are the complete command lines we use to compile C and C++ files. +# - $< is the source file. +# - $1 is the object file to create. +CC_CMD=$(CC) -o $1 -c $< $(CPPFLAGS) $(GLOBAL_CFLAGS) $(CFLAGS) $($1_CFLAGS) -MMD -MF $(call filename-to-dep,$1) -MP +CXX_CMD=$(CXX) -o $1 -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($1_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep,$1) -MP + +# We use COMPILE_COMMANDS_JSON_CMD to turn a compilation command (like CC_CMD +# or CXX_CMD above) into a comple_commands.json file. We rely on bash native +# word splitting to define the positional arguments. +# - $< is the source file being compiled. +COMPILE_COMMANDS_JSON_CMD=jq --null-input '{ directory: $$ENV.PWD, file: "$<", arguments: $$ARGS.positional }' --args -- + + $(buildprefix)%.o: %.cc @mkdir -p "$(dir $@)" - $(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cxx) $(call CXX_CMD,$@) $(buildprefix)%.o: %.cpp @mkdir -p "$(dir $@)" - $(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cxx) $(call CXX_CMD,$@) $(buildprefix)%.o: %.c @mkdir -p "$(dir $@)" - $(trace-cc) $(CC) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CFLAGS) $(CFLAGS) $($@_CFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cc) $(call CC_CMD,$@) + +# In the following we need to replace the .compile_commands.json extension in $@ with .o +# to make the object file. This is needed because CC_CMD and CXX_CMD do further expansions +# based on the object file name (i.e. *_CXXFLAGS and filename-to-dep). + +$(buildprefix)%.compile_commands.json: %.cc + @mkdir -p "$(dir $@)" + $(trace-jq) $(COMPILE_COMMANDS_JSON_CMD) $(call CXX_CMD,$(@:.compile_commands.json=.o)) > $@ + +$(buildprefix)%.compile_commands.json: %.cpp + @mkdir -p "$(dir $@)" + $(trace-jq) $(COMPILE_COMMANDS_JSON_CMD) $(call CXX_CMD,$(@:.compile_commands.json=.o)) > $@ + +$(buildprefix)%.compile_commands.json: %.c + @mkdir -p "$(dir $@)" + $(trace-jq) $(COMPILE_COMMANDS_JSON_CMD) $(call CC_CMD,$(@:.compile_commands.json=.o)) > $@ diff --git a/mk/platform.mk b/mk/platform.mk index fe960dedf..22c114a20 100644 --- a/mk/platform.mk +++ b/mk/platform.mk @@ -29,4 +29,8 @@ ifdef HOST_OS HOST_SOLARIS = 1 HOST_UNIX = 1 endif + ifeq ($(HOST_KERNEL), gnu) + HOST_HURD = 1 + HOST_UNIX = 1 + endif endif diff --git a/mk/precompiled-headers.mk b/mk/precompiled-headers.mk index cdd3daecd..f2803eb79 100644 --- a/mk/precompiled-headers.mk +++ b/mk/precompiled-headers.mk @@ -8,7 +8,7 @@ GCH = $(buildprefix)precompiled-headers.h.gch $(GCH): precompiled-headers.h @rm -f $@ @mkdir -p "$(dir $@)" - $(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS) + $(trace-gen) $(CXX) -c -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS) clean-files += $(GCH) diff --git a/mk/run-test.sh b/mk/run-test.sh index da9c5a473..7f9f1d5f8 100755 --- a/mk/run-test.sh +++ b/mk/run-test.sh @@ -8,7 +8,6 @@ yellow="" normal="" test=$1 -init=${2-} dir="$(dirname "${BASH_SOURCE[0]}")" source "$dir/common-test.sh" @@ -22,20 +21,18 @@ if [ -t 1 ]; then fi run_test () { - if [ -n "$init" ]; then - (init_test 2>/dev/null > /dev/null) - fi - log="$(run_test_proper 2>&1)" && status=0 || status=$? + log="$(run "$test" 2>&1)" && status=0 || status=$? } run_test -if [ $status -eq 0 ]; then +if [[ "$status" = 0 ]]; then echo "$post_run_msg [${green}PASS$normal]" -elif [ $status -eq 99 ]; then +elif [[ "$status" = 77 ]]; then echo "$post_run_msg [${yellow}SKIP$normal]" else echo "$post_run_msg [${red}FAIL$normal]" + # shellcheck disable=SC2001 echo "$log" | sed 's/^/ /' exit "$status" fi diff --git a/mk/tests.mk b/mk/tests.mk index bac9b704a..0a10f6d3b 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -12,8 +12,8 @@ endef define run-test - $(eval $(call run-bash,$1.test,$1 $(test-deps),mk/run-test.sh $1 $2)) - $(eval $(call run-bash,$1.test-debug,$1 $(test-deps),mk/debug-test.sh $1 $2)) + $(eval $(call run-bash,$1.test,$1 $(test-deps),mk/run-test.sh $1)) + $(eval $(call run-bash,$1.test-debug,$1 $(test-deps),mk/debug-test.sh $1)) endef diff --git a/mk/tracing.mk b/mk/tracing.mk index 1fc5573d7..09db1e617 100644 --- a/mk/tracing.mk +++ b/mk/tracing.mk @@ -10,6 +10,8 @@ ifeq ($(V), 0) trace-install = @echo " INST " $@; trace-mkdir = @echo " MKDIR " $@; trace-test = @echo " TEST " $@; + trace-sh = @echo " SH " $@; + trace-jq = @echo " JQ " $@; suppress = @ diff --git a/package.nix b/package.nix index 20796a386..a7c8923e8 100644 --- a/package.nix +++ b/package.nix @@ -13,17 +13,16 @@ , curl , editline , readline -, fileset , flex , git , gtest , jq -, doxygen , libarchive , libcpuid , libgit2 , libseccomp , libsodium +, man , lowdown , mdbook , mdbook-linkcheck @@ -33,7 +32,8 @@ , pkg-config , rapidcheck , sqlite -, util-linux +, toml11 +, unixtools , xz , busybox-sandbox-shell ? null @@ -48,10 +48,8 @@ , pname ? "nix" , versionSuffix ? "" -, officialRelease ? false -# Whether to build Nix. Useful to skip for tasks like (a) just -# generating API docs or (b) testing existing pre-built versions of Nix +# Whether to build Nix. Useful to skip for tasks like testing existing pre-built versions of Nix , doBuild ? true # Run the unit tests as part of the build. See `installUnitTests` for an @@ -74,7 +72,10 @@ # sounds so long as evaluation just takes places within short-lived # processes. (When the process exits, the memory is reclaimed; it is # only leaked *within* the process.) -, enableGC ? true +# +# Temporarily disabled on Windows because the `GC_throw_bad_alloc` +# symbol is missing during linking. +, enableGC ? !stdenv.hostPlatform.isWindows # Whether to enable Markdown rendering in the Nix binary. , enableMarkdown ? !stdenv.hostPlatform.isWindows @@ -87,10 +88,6 @@ # - readline , readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline" -# Whether to build the internal API docs, can be done separately from -# everything else. -, enableInternalAPIDocs ? false - # Whether to install unit tests. This is useful when cross compiling # since we cannot run them natively during the build, but can do so # later. @@ -113,6 +110,8 @@ }: let + inherit (lib) fileset; + version = lib.fileContents ./.version + versionSuffix; # selected attributes with defaults, will be used to define some @@ -161,6 +160,8 @@ in { ./m4 # TODO: do we really need README.md? It doesn't seem used in the build. ./README.md + # This could be put behind a conditional + ./maintainers/local.mk # For make, regardless of what we are building ./local.mk ./Makefile @@ -171,17 +172,11 @@ in { ./doc ./misc ./precompiled-headers.h - ./src + (fileset.difference ./src ./src/perl) ./COPYING ./scripts/local.mk - ] ++ lib.optionals buildUnitTests [ + ] ++ lib.optionals enableManual [ ./doc/manual - ] ++ lib.optionals enableInternalAPIDocs [ - ./doc/internal-api - # Source might not be compiled, but still must be available - # for Doxygen to gather comments. - ./src - ./tests/unit ] ++ lib.optionals buildUnitTests [ ./tests/unit ] ++ lib.optionals doInstallCheck [ @@ -195,8 +190,10 @@ in { ++ lib.optional doBuild "dev" # If we are doing just build or just docs, the one thing will use # "out". We only need additional outputs if we are doing both. - ++ lib.optional (doBuild && (enableManual || enableInternalAPIDocs)) "doc" - ++ lib.optional installUnitTests "check"; + ++ lib.optional (doBuild && enableManual) "doc" + ++ lib.optional installUnitTests "check" + ++ lib.optional doCheck "testresults" + ; nativeBuildInputs = [ autoconf-archive @@ -213,14 +210,14 @@ in { git mercurial openssh + man # for testing `nix-* --help` ] ++ lib.optionals (doInstallCheck || enableManual) [ jq # Also for custom mdBook preprocessor. - ] ++ lib.optional stdenv.hostPlatform.isLinux util-linux - ++ lib.optional enableInternalAPIDocs doxygen + ] ++ lib.optional stdenv.hostPlatform.isStatic unixtools.hexdump ; - buildInputs = lib.optionals doBuild [ - boost + buildInputs = lib.optionals doBuild ( + [ brotli bzip2 curl @@ -229,6 +226,7 @@ in { libsodium openssl sqlite + toml11 xz ({ inherit readline editline; }.${readlineFlavor}) ] ++ lib.optionals enableMarkdown [ @@ -240,46 +238,22 @@ in { ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid # There have been issues building these dependencies ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin)) - (aws-sdk-cpp.override { - apis = ["s3" "transfer"]; - customMemoryManagement = false; - }) - ; + aws-sdk-cpp + ); - propagatedBuildInputs = [ + propagatedBuildInputs = lib.optionals doBuild ([ + boost nlohmann_json - ] ++ lib.optional enableGC boehmgc; + ] ++ lib.optional enableGC boehmgc + ); dontBuild = !attrs.doBuild; doCheck = attrs.doCheck; - disallowedReferences = [ boost ]; - - preConfigure = lib.optionalString (doBuild && ! stdenv.hostPlatform.isStatic) ( - '' - # Copy libboost_context so we don't get all of Boost in our closure. - # https://github.com/NixOS/nixpkgs/issues/45462 - mkdir -p $out/lib - cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib - rm -f $out/lib/*.a - '' + lib.optionalString stdenv.hostPlatform.isLinux '' - chmod u+w $out/lib/*.so.* - patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* - '' + lib.optionalString stdenv.hostPlatform.isDarwin '' - for LIB in $out/lib/*.dylib; do - chmod u+w $LIB - install_name_tool -id $LIB $LIB - install_name_tool -delete_rpath ${boost}/lib/ $LIB || true - done - install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib - '' - ); - configureFlags = [ (lib.enableFeature doBuild "build") (lib.enableFeature buildUnitTests "unit-tests") (lib.enableFeature doInstallCheck "functional-tests") - (lib.enableFeature enableInternalAPIDocs "internal-api-docs") (lib.enableFeature enableManual "doc-gen") (lib.enableFeature enableGC "gc") (lib.enableFeature enableMarkdown "markdown") @@ -303,8 +277,11 @@ in { makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1"; - installTargets = lib.optional doBuild "install" - ++ lib.optional enableInternalAPIDocs "internal-api-html"; + preCheck = '' + mkdir $testresults + ''; + + installTargets = lib.optional doBuild "install"; installFlags = "sysconfdir=$(out)/etc"; @@ -319,18 +296,16 @@ in { lib.optionalString stdenv.hostPlatform.isStatic '' mkdir -p $out/nix-support echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products - '' + lib.optionalString stdenv.isDarwin '' - install_name_tool \ - -change ${boost}/lib/libboost_context.dylib \ - $out/lib/libboost_context.dylib \ - $out/lib/libnixutil.dylib '' ) + lib.optionalString enableManual '' mkdir -p ''${!outputDoc}/nix-support echo "doc manual ''${!outputDoc}/share/doc/nix/manual" >> ''${!outputDoc}/nix-support/hydra-build-products - '' + lib.optionalString enableInternalAPIDocs '' - mkdir -p ''${!outputDoc}/nix-support - echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products + ''; + + # So the check output gets links for DLLs in the out output. + preFixup = lib.optionalString (stdenv.hostPlatform.isWindows && builtins.elem "check" finalAttrs.outputs) '' + ln -s "$check/lib/"*.dll "$check/bin" + ln -s "$out/bin/"*.dll "$check/bin" ''; doInstallCheck = attrs.doInstallCheck; @@ -341,20 +316,26 @@ in { # Work around weird bug where it doesn't think there is a Makefile. installCheckPhase = if (!doBuild && doInstallCheck) then '' + runHook preInstallCheck mkdir -p src/nix-channel make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES '' else null; # Needed for tests if we are not doing a build, but testing existing # built Nix. - preInstallCheck = lib.optionalString (! doBuild) '' - mkdir -p src/nix-channel - ''; + preInstallCheck = + lib.optionalString (! doBuild) '' + mkdir -p src/nix-channel + '' + # See https://github.com/NixOS/nix/issues/2523 + # Occurs often in tests since https://github.com/NixOS/nix/pull/9900 + + lib.optionalString stdenv.hostPlatform.isDarwin '' + export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES + ''; separateDebugInfo = !stdenv.hostPlatform.isStatic; - # TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated - # to work with `strictDeps`. + # TODO Always true after https://github.com/NixOS/nixpkgs/issues/318564 strictDeps = !withCoverageChecks; hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; diff --git a/packaging/components.nix b/packaging/components.nix new file mode 100644 index 000000000..870e9ae61 --- /dev/null +++ b/packaging/components.nix @@ -0,0 +1,43 @@ +scope: +let + inherit (scope) callPackage; +in + +# This becomes the pkgs.nixComponents attribute set +{ + nix = callPackage ../package.nix { }; + + nix-util = callPackage ../src/libutil/package.nix { }; + nix-util-c = callPackage ../src/libutil-c/package.nix { }; + nix-util-test-support = callPackage ../tests/unit/libutil-support/package.nix { }; + nix-util-tests = callPackage ../tests/unit/libutil/package.nix { }; + + nix-store = callPackage ../src/libstore/package.nix { }; + nix-store-c = callPackage ../src/libstore-c/package.nix { }; + nix-store-test-support = callPackage ../tests/unit/libstore-support/package.nix { }; + nix-store-tests = callPackage ../tests/unit/libstore/package.nix { }; + + nix-fetchers = callPackage ../src/libfetchers/package.nix { }; + nix-fetchers-tests = callPackage ../tests/unit/libfetchers/package.nix { }; + + nix-expr = callPackage ../src/libexpr/package.nix { }; + nix-expr-c = callPackage ../src/libexpr-c/package.nix { }; + nix-expr-test-support = callPackage ../tests/unit/libexpr-support/package.nix { }; + nix-expr-tests = callPackage ../tests/unit/libexpr/package.nix { }; + + nix-flake = callPackage ../src/libflake/package.nix { }; + nix-flake-tests = callPackage ../tests/unit/libflake/package.nix { }; + + nix-main = callPackage ../src/libmain/package.nix { }; + nix-main-c = callPackage ../src/libmain-c/package.nix { }; + + nix-cmd = callPackage ../src/libcmd/package.nix { }; + + # Will replace `nix` once the old build system is gone. + nix-ng = callPackage ../src/nix/package.nix { }; + + nix-internal-api-docs = callPackage ../src/internal-api-docs/package.nix { }; + nix-external-api-docs = callPackage ../src/external-api-docs/package.nix { }; + + nix-perl-bindings = callPackage ../src/perl/package.nix { }; +} diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix new file mode 100644 index 000000000..e0737593f --- /dev/null +++ b/packaging/dependencies.nix @@ -0,0 +1,165 @@ +# These overrides are applied to the dependencies of the Nix components. + +{ + # Flake inputs; used for sources + inputs, + + # The raw Nixpkgs, not affected by this scope + pkgs, + + stdenv, + versionSuffix, +}: + +let + prevStdenv = stdenv; +in + +let + inherit (pkgs) lib; + + root = ../.; + + stdenv = if prevStdenv.isDarwin && prevStdenv.isx86_64 + then darwinStdenv + else prevStdenv; + + # Fix the following error with the default x86_64-darwin SDK: + # + # error: aligned allocation function of type 'void *(std::size_t, std::align_val_t)' is only available on macOS 10.13 or newer + # + # Despite the use of the 10.13 deployment target here, the aligned + # allocation function Clang uses with this setting actually works + # all the way back to 10.6. + darwinStdenv = pkgs.overrideSDK prevStdenv { darwinMinVersion = "10.13"; }; + + # Nixpkgs implements this by returning a subpath into the fetched Nix sources. + resolvePath = p: p; + + # Indirection for Nixpkgs to override when package.nix files are vendored + filesetToSource = lib.fileset.toSource; + + localSourceLayer = finalAttrs: prevAttrs: + let + workDirPath = + # Ideally we'd pick finalAttrs.workDir, but for now `mkDerivation` has + # the requirement that everything except passthru and meta must be + # serialized by mkDerivation, which doesn't work for this. + prevAttrs.workDir; + + workDirSubpath = lib.path.removePrefix root workDirPath; + sources = assert prevAttrs.fileset._type == "fileset"; prevAttrs.fileset; + src = lib.fileset.toSource { fileset = sources; inherit root; }; + + in + { + sourceRoot = "${src.name}/" + workDirSubpath; + inherit src; + + # Clear what `derivation` can't/shouldn't serialize; see prevAttrs.workDir. + fileset = null; + workDir = null; + }; + + # Work around weird `--as-needed` linker behavior with BSD, see + # https://github.com/mesonbuild/meson/issues/3593 + bsdNoLinkAsNeeded = finalAttrs: prevAttrs: + lib.optionalAttrs stdenv.hostPlatform.isBSD { + mesonFlags = [ (lib.mesonBool "b_asneeded" false) ] ++ prevAttrs.mesonFlags or []; + }; + + miscGoodPractice = finalAttrs: prevAttrs: + { + strictDeps = prevAttrs.strictDeps or true; + enableParallelBuilding = true; + }; + +in +scope: { + inherit stdenv versionSuffix; + version = lib.fileContents ../.version + versionSuffix; + + aws-sdk-cpp = (pkgs.aws-sdk-cpp.override { + apis = [ "s3" "transfer" ]; + customMemoryManagement = false; + }).overrideAttrs { + # only a stripped down version is built, which takes a lot less resources + # to build, so we don't need a "big-parallel" machine. + requiredSystemFeatures = [ ]; + }; + + libseccomp = pkgs.libseccomp.overrideAttrs (_: rec { + version = "2.5.5"; + src = pkgs.fetchurl { + url = "https://github.com/seccomp/libseccomp/releases/download/v${version}/libseccomp-${version}.tar.gz"; + hash = "sha256-JIosik2bmFiqa69ScSw0r+/PnJ6Ut23OAsHJqiX7M3U="; + }; + }); + + boehmgc = pkgs.boehmgc.override { + enableLargeConfig = true; + }; + + # TODO Hack until https://github.com/NixOS/nixpkgs/issues/45462 is fixed. + boost = (pkgs.boost.override { + extraB2Args = [ + "--with-container" + "--with-context" + "--with-coroutine" + ]; + }).overrideAttrs (old: { + # Need to remove `--with-*` to use `--with-libraries=...` + buildPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.buildPhase; + installPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.installPhase; + }); + + libgit2 = pkgs.libgit2.overrideAttrs (attrs: { + src = inputs.libgit2; + version = inputs.libgit2.lastModifiedDate; + cmakeFlags = attrs.cmakeFlags or [] + ++ [ "-DUSE_SSH=exec" ]; + }); + + busybox-sandbox-shell = pkgs.busybox-sandbox-shell or (pkgs.busybox.override { + useMusl = true; + enableStatic = true; + enableMinimal = true; + extraConfig = '' + CONFIG_FEATURE_FANCY_ECHO y + CONFIG_FEATURE_SH_MATH y + CONFIG_FEATURE_SH_MATH_64 y + + CONFIG_ASH y + CONFIG_ASH_OPTIMIZE_FOR_SIZE y + + CONFIG_ASH_ALIAS y + CONFIG_ASH_BASH_COMPAT y + CONFIG_ASH_CMDCMD y + CONFIG_ASH_ECHO y + CONFIG_ASH_GETOPTS y + CONFIG_ASH_INTERNAL_GLOB y + CONFIG_ASH_JOB_CONTROL y + CONFIG_ASH_PRINTF y + CONFIG_ASH_TEST y + ''; + }); + + # TODO change in Nixpkgs, Windows works fine. First commit of + # https://github.com/NixOS/nixpkgs/pull/322977 backported will fix. + toml11 = pkgs.toml11.overrideAttrs (old: { + meta.platforms = lib.platforms.all; + }); + + inherit resolvePath filesetToSource; + + mkMesonDerivation = f: let + exts = [ + miscGoodPractice + bsdNoLinkAsNeeded + localSourceLayer + ]; + in stdenv.mkDerivation + (lib.extends + (lib.foldr lib.composeExtensions (_: _: {}) exts) + f); +} diff --git a/packaging/hydra.nix b/packaging/hydra.nix new file mode 100644 index 000000000..dbe992476 --- /dev/null +++ b/packaging/hydra.nix @@ -0,0 +1,200 @@ +{ inputs +, binaryTarball +, forAllCrossSystems +, forAllSystems +, lib +, linux64BitSystems +, nixpkgsFor +, self +}: +let + inherit (inputs) nixpkgs nixpkgs-regression; + + installScriptFor = tarballs: + nixpkgsFor.x86_64-linux.native.callPackage ../scripts/installer.nix { + inherit tarballs; + }; + + testNixVersions = pkgs: client: daemon: + pkgs.callPackage ../package.nix { + pname = + "nix-tests" + + lib.optionalString + (lib.versionAtLeast daemon.version "2.4pre20211005" && + lib.versionAtLeast client.version "2.4pre20211005") + "-${client.version}-against-${daemon.version}"; + + test-client = client; + test-daemon = daemon; + + doBuild = false; + }; + + # Technically we could just return `pkgs.nixComponents`, but for Hydra it's + # convention to transpose it, and to transpose it efficiently, we need to + # enumerate them manually, so that we don't evaluate unnecessary package sets. + forAllPackages = lib.genAttrs [ + "nix" + "nix-util" + "nix-util-c" + "nix-util-test-support" + "nix-util-tests" + "nix-store" + "nix-store-c" + "nix-store-test-support" + "nix-store-tests" + "nix-fetchers" + "nix-fetchers-tests" + "nix-expr" + "nix-expr-c" + "nix-expr-test-support" + "nix-expr-tests" + "nix-flake" + "nix-flake-tests" + "nix-main" + "nix-main-c" + "nix-cmd" + "nix-ng" + ]; +in +{ + # Binary package for various platforms. + build = forAllPackages (pkgName: + forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.${pkgName})); + + shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation); + + buildStatic = forAllPackages (pkgName: + lib.genAttrs linux64BitSystems (system: nixpkgsFor.${system}.static.nixComponents.${pkgName})); + + buildCross = forAllPackages (pkgName: + forAllCrossSystems (crossSystem: + lib.genAttrs [ "x86_64-linux" ] (system: nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName}))); + + buildNoGc = forAllSystems (system: + self.packages.${system}.nix.override { enableGC = false; } + ); + + buildNoTests = forAllSystems (system: nixpkgsFor.${system}.native.nix_noTests); + + # Toggles some settings for better coverage. Windows needs these + # library combinations, and Debian build Nix with GNU readline too. + buildReadlineNoMarkdown = forAllSystems (system: + self.packages.${system}.nix.override { + enableMarkdown = false; + readlineFlavor = "readline"; + } + ); + + # Perl bindings for various platforms. + perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.nix-perl-bindings); + + # Binary tarball for various platforms, containing a Nix store + # with the closure of 'nix' package, and the second half of + # the installation script. + binaryTarball = forAllSystems (system: binaryTarball nixpkgsFor.${system}.native.nix nixpkgsFor.${system}.native); + + binaryTarballCross = lib.genAttrs [ "x86_64-linux" ] (system: + forAllCrossSystems (crossSystem: + binaryTarball + nixpkgsFor.${system}.cross.${crossSystem}.nix + nixpkgsFor.${system}.cross.${crossSystem})); + + # The first half of the installation script. This is uploaded + # to https://nixos.org/nix/install. It downloads the binary + # tarball for the user's system and calls the second half of the + # installation script. + installerScript = installScriptFor [ + # Native + self.hydraJobs.binaryTarball."x86_64-linux" + self.hydraJobs.binaryTarball."i686-linux" + self.hydraJobs.binaryTarball."aarch64-linux" + self.hydraJobs.binaryTarball."x86_64-darwin" + self.hydraJobs.binaryTarball."aarch64-darwin" + # Cross + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu" + ]; + installerScriptForGHA = installScriptFor [ + # Native + self.hydraJobs.binaryTarball."x86_64-linux" + self.hydraJobs.binaryTarball."aarch64-darwin" + # Cross + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu" + ]; + + # docker image with Nix inside + dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); + + # Line coverage analysis. + coverage = nixpkgsFor.x86_64-linux.native.nix.override { + pname = "nix-coverage"; + withCoverageChecks = true; + }; + + # API docs for Nix's unstable internal C++ interfaces. + internal-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-internal-api-docs; + + # API docs for Nix's C bindings. + external-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-external-api-docs; + + # System tests. + tests = import ../tests/nixos { inherit lib nixpkgs nixpkgsFor self; } // { + + # Make sure that nix-env still produces the exact same result + # on a particular version of Nixpkgs. + evalNixpkgs = + let + inherit (nixpkgsFor.x86_64-linux.native) runCommand nix; + in + runCommand "eval-nixos" { buildInputs = [ nix ]; } + '' + type -p nix-env + # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593. + ( + set -x + time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages + [[ $(sha1sum < packages | cut -c1-40) = e01b031fc9785a572a38be6bc473957e3b6faad7 ]] + ) + mkdir $out + ''; + + nixpkgsLibTests = + forAllSystems (system: + import (nixpkgs + "/lib/tests/test-with-nix.nix") + { + lib = nixpkgsFor.${system}.native.lib; + nix = self.packages.${system}.nix; + pkgs = nixpkgsFor.${system}.native; + } + ); + }; + + metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" { + pkgs = nixpkgsFor.x86_64-linux.native; + nixpkgs = nixpkgs-regression; + }; + + installTests = forAllSystems (system: + let pkgs = nixpkgsFor.${system}.native; in + pkgs.runCommand "install-tests" + { + againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix; + againstCurrentLatest = + # FIXME: temporarily disable this on macOS because of #3605. + if system == "x86_64-linux" + then testNixVersions pkgs pkgs.nix pkgs.nixVersions.latest + else null; + # Disabled because the latest stable version doesn't handle + # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work + # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable; + } "touch $out"); + + installerTests = import ../tests/installer { + binaryTarballs = self.hydraJobs.binaryTarball; + inherit nixpkgsFor; + }; +} diff --git a/perl/.yath.rc b/perl/.yath.rc deleted file mode 100644 index 118bf80c8..000000000 --- a/perl/.yath.rc +++ /dev/null @@ -1,2 +0,0 @@ -[test] --I=rel(lib/Nix) diff --git a/perl/Makefile b/perl/Makefile deleted file mode 100644 index 832668dd1..000000000 --- a/perl/Makefile +++ /dev/null @@ -1,21 +0,0 @@ -makefiles = local.mk - -GLOBAL_CXXFLAGS += -g -Wall -std=c++2a - -# A convenience for concurrent development of Nix and its Perl bindings. -# Not needed in a standalone build of the Perl bindings. -ifneq ("$(wildcard ../src)", "") - GLOBAL_CXXFLAGS += -I ../src -endif - --include Makefile.config - -OPTIMIZE = 1 - -ifeq ($(OPTIMIZE), 1) - GLOBAL_CXXFLAGS += -O3 -else - GLOBAL_CXXFLAGS += -O0 -endif - -include mk/lib.mk diff --git a/perl/Makefile.config.in b/perl/Makefile.config.in deleted file mode 100644 index d856de3ad..000000000 --- a/perl/Makefile.config.in +++ /dev/null @@ -1,18 +0,0 @@ -HOST_OS = @host_os@ -CC = @CC@ -CFLAGS = @CFLAGS@ -CXX = @CXX@ -CXXFLAGS = @CXXFLAGS@ -PACKAGE_NAME = @PACKAGE_NAME@ -PACKAGE_VERSION = @PACKAGE_VERSION@ -SODIUM_LIBS = @SODIUM_LIBS@ -NIX_CFLAGS = @NIX_CFLAGS@ -NIX_LIBS = @NIX_LIBS@ -nixbindir = @nixbindir@ -curl = @curl@ -nixlibexecdir = @nixlibexecdir@ -nixlocalstatedir = @nixlocalstatedir@ -perl = @perl@ -perllibdir = @perllibdir@ -nixstoredir = @nixstoredir@ -nixsysconfdir = @nixsysconfdir@ diff --git a/perl/configure.ac b/perl/configure.ac deleted file mode 100644 index a02cb06c9..000000000 --- a/perl/configure.ac +++ /dev/null @@ -1,84 +0,0 @@ -AC_INIT(nix-perl, m4_esyscmd([bash -c "echo -n $(cat ../.version)$VERSION_SUFFIX"])) -AC_CONFIG_SRCDIR(MANIFEST) -AC_CONFIG_AUX_DIR(../config) - -CFLAGS= -CXXFLAGS= -AC_PROG_CC -AC_PROG_CXX - -AC_CANONICAL_HOST - -# Use 64-bit file system calls so that we can support files > 2 GiB. -AC_SYS_LARGEFILE - -AC_DEFUN([NEED_PROG], -[ -AC_PATH_PROG($1, $2) -if test -z "$$1"; then - AC_MSG_ERROR([$2 is required]) -fi -]) - -NEED_PROG(perl, perl) -NEED_PROG(curl, curl) -NEED_PROG(bzip2, bzip2) -NEED_PROG(xz, xz) - -# Test that Perl has the open/fork feature (Perl 5.8.0 and beyond). -AC_MSG_CHECKING([whether Perl is recent enough]) -if ! $perl -e 'open(FOO, "-|", "true"); while () { print; }; close FOO or die;'; then - AC_MSG_RESULT(no) - AC_MSG_ERROR([Your Perl version is too old. Nix requires Perl 5.8.0 or newer.]) -fi -AC_MSG_RESULT(yes) - - -# Figure out where to install Perl modules. -AC_MSG_CHECKING([for the Perl installation prefix]) -perlversion=$($perl -e 'use Config; print $Config{version};') -perlarchname=$($perl -e 'use Config; print $Config{archname};') -AC_SUBST(perllibdir, [${libdir}/perl5/site_perl/$perlversion/$perlarchname]) -AC_MSG_RESULT($perllibdir) - -# Look for libsodium. -PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"]) - -# Check for the required Perl dependencies (DBI and DBD::SQLite). -perlFlags="-I$perllibdir" - -AC_ARG_WITH(dbi, AC_HELP_STRING([--with-dbi=PATH], - [prefix of the Perl DBI library]), - perlFlags="$perlFlags -I$withval") - -AC_ARG_WITH(dbd-sqlite, AC_HELP_STRING([--with-dbd-sqlite=PATH], - [prefix of the Perl DBD::SQLite library]), - perlFlags="$perlFlags -I$withval") - -AC_MSG_CHECKING([whether DBD::SQLite works]) -if ! $perl $perlFlags -e 'use DBI; use DBD::SQLite;' 2>&5; then - AC_MSG_RESULT(no) - AC_MSG_FAILURE([The Perl modules DBI and/or DBD::SQLite are missing.]) -fi -AC_MSG_RESULT(yes) - -AC_SUBST(perlFlags) - -PKG_CHECK_MODULES([NIX], [nix-store]) - -NEED_PROG([NIX], [nix]) - -# Expand all variables in config.status. -test "$prefix" = NONE && prefix=$ac_default_prefix -test "$exec_prefix" = NONE && exec_prefix='${prefix}' -for name in $ac_subst_vars; do - declare $name="$(eval echo "${!name}")" - declare $name="$(eval echo "${!name}")" - declare $name="$(eval echo "${!name}")" -done - -rm -f Makefile.config -ln -sfn ../mk mk - -AC_CONFIG_FILES([]) -AC_OUTPUT diff --git a/perl/default.nix b/perl/default.nix deleted file mode 100644 index 7103574c9..000000000 --- a/perl/default.nix +++ /dev/null @@ -1,61 +0,0 @@ -{ lib, fileset -, stdenv -, perl, perlPackages -, autoconf-archive, autoreconfHook, pkg-config -, nix, curl, bzip2, xz, boost, libsodium, darwin -}: - -perl.pkgs.toPerlModule (stdenv.mkDerivation (finalAttrs: { - name = "nix-perl-${nix.version}"; - - src = fileset.toSource { - root = ../.; - fileset = fileset.unions ([ - ../.version - ../m4 - ../mk - ./MANIFEST - ./Makefile - ./Makefile.config.in - ./configure.ac - ./lib - ./local.mk - ] ++ lib.optionals finalAttrs.doCheck [ - ./.yath.rc - ./t - ]); - }; - - nativeBuildInputs = - [ autoconf-archive - autoreconfHook - pkg-config - ]; - - buildInputs = - [ nix - curl - bzip2 - xz - perl - boost - ] - ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium - ++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security; - - # `perlPackages.Test2Harness` is marked broken for Darwin - doCheck = !stdenv.isDarwin; - - nativeCheckInputs = [ - perlPackages.Test2Harness - ]; - - configureFlags = [ - "--with-dbi=${perlPackages.DBI}/${perl.libPrefix}" - "--with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix}" - ]; - - enableParallelBuilding = true; - - postUnpack = "sourceRoot=$sourceRoot/perl"; -})) diff --git a/perl/local.mk b/perl/local.mk deleted file mode 100644 index ed4764eb9..000000000 --- a/perl/local.mk +++ /dev/null @@ -1,46 +0,0 @@ -nix_perl_sources := \ - lib/Nix/Store.pm \ - lib/Nix/Manifest.pm \ - lib/Nix/SSH.pm \ - lib/Nix/CopyClosure.pm \ - lib/Nix/Config.pm.in \ - lib/Nix/Utils.pm - -nix_perl_modules := $(nix_perl_sources:.in=) - -$(foreach x, $(nix_perl_modules), $(eval $(call install-data-in, $(x), $(perllibdir)/Nix))) - -lib/Nix/Store.cc: lib/Nix/Store.xs - $(trace-gen) xsubpp $^ -output $@ - -libraries += Store - -Store_DIR := lib/Nix - -Store_SOURCES := $(Store_DIR)/Store.cc - -Store_CXXFLAGS = \ - $(NIX_CFLAGS) \ - -I$(shell perl -e 'use Config; print $$Config{archlibexp};')/CORE \ - -D_FILE_OFFSET_BITS=64 \ - -Wno-unknown-warning-option -Wno-unused-variable -Wno-literal-suffix \ - -Wno-reserved-user-defined-literal -Wno-duplicate-decl-specifier -Wno-pointer-bool-conversion - -Store_LDFLAGS := $(SODIUM_LIBS) $(NIX_LIBS) - -ifdef HOST_CYGWIN - archlib = $(shell perl -E 'use Config; print $$Config{archlib};') - libperl = $(shell perl -E 'use Config; print $$Config{libperl};') - Store_LDFLAGS += $(shell find ${archlib} -name ${libperl}) -endif - -Store_ALLOW_UNDEFINED = 1 - -Store_FORCE_INSTALL = 1 - -Store_INSTALL_DIR = $(perllibdir)/auto/Nix/Store - -clean-files += lib/Nix/Config.pm lib/Nix/Store.cc Makefile.config - -check: all - yath test diff --git a/precompiled-headers.h b/precompiled-headers.h index f52f1cab8..e1a3f8cc0 100644 --- a/precompiled-headers.h +++ b/precompiled-headers.h @@ -42,19 +42,22 @@ #include #include #include -#include -#include -#include #include -#include -#include -#include #include #include #include -#include -#include -#include #include +#ifndef _WIN32 +# include +# include +# include +# include +# include +# include +# include +# include +# include +#endif + #include diff --git a/scripts/bigsur-nixbld-user-migration.sh b/scripts/bigsur-nixbld-user-migration.sh index f1619fd56..0eb312e07 100755 --- a/scripts/bigsur-nixbld-user-migration.sh +++ b/scripts/bigsur-nixbld-user-migration.sh @@ -3,7 +3,7 @@ ((NEW_NIX_FIRST_BUILD_UID=301)) id_available(){ - dscl . list /Users UniqueID | grep -E '\b'$1'\b' >/dev/null + dscl . list /Users UniqueID | grep -E '\b'"$1"'\b' >/dev/null } change_nixbld_names_and_ids(){ @@ -26,18 +26,18 @@ change_nixbld_names_and_ids(){ fi done - if [[ $name == _* ]]; then + if [[ "$name" == _* ]]; then echo " It looks like $name has already been renamed--skipping." else # first 3 are cleanup, it's OK if they aren't here - sudo dscl . delete /Users/$name dsAttrTypeNative:_writers_passwd &>/dev/null || true - sudo dscl . change /Users/$name NFSHomeDirectory "/private/var/empty 1" "/var/empty" &>/dev/null || true + sudo dscl . delete "/Users/$name" dsAttrTypeNative:_writers_passwd &>/dev/null || true + sudo dscl . change "/Users/$name" NFSHomeDirectory "/private/var/empty 1" "/var/empty" &>/dev/null || true # remove existing user from group - sudo dseditgroup -o edit -t user -d $name nixbld || true - sudo dscl . change /Users/$name UniqueID $uid $next_id - sudo dscl . change /Users/$name RecordName $name _$name + sudo dseditgroup -o edit -t user -d "$name" nixbld || true + sudo dscl . change "/Users/$name" UniqueID "$uid" "$next_id" + sudo dscl . change "/Users/$name" RecordName "$name" "_$name" # add renamed user to group - sudo dseditgroup -o edit -t user -a _$name nixbld + sudo dseditgroup -o edit -t user -a "_$name" nixbld echo " $name migrated to _$name (uid: $next_id)" fi done < <(dscl . list /Users UniqueID | grep nixbld | sort -n -k2) diff --git a/scripts/check-hydra-status.sh b/scripts/check-hydra-status.sh deleted file mode 100644 index e62705e94..000000000 --- a/scripts/check-hydra-status.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail -# set -x - - -# mapfile BUILDS_FOR_LATEST_EVAL < <( -# curl -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master/evals | \ -# jq -r '.evals[0].builds[] | @sh') -BUILDS_FOR_LATEST_EVAL=$( -curl -sS -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master/evals | \ - jq -r '.evals[0].builds[]') - -someBuildFailed=0 - -for buildId in $BUILDS_FOR_LATEST_EVAL; do - buildInfo=$(curl --fail -sS -H 'Accept: application/json' "https://hydra.nixos.org/build/$buildId") - - finished=$(echo "$buildInfo" | jq -r '.finished') - - if [[ $finished = 0 ]]; then - continue - fi - - buildStatus=$(echo "$buildInfo" | jq -r '.buildstatus') - - if [[ $buildStatus != 0 ]]; then - someBuildFailed=1 - echo "Job “$(echo "$buildInfo" | jq -r '.job')†failed on hydra: $buildInfo" - fi -done - -exit "$someBuildFailed" diff --git a/scripts/flake-regressions.sh b/scripts/flake-regressions.sh new file mode 100755 index 000000000..d76531134 --- /dev/null +++ b/scripts/flake-regressions.sh @@ -0,0 +1,27 @@ +#! /usr/bin/env bash + +set -e + +echo "Nix version:" +nix --version + +cd flake-regressions + +status=0 + +flakes=$(find tests -mindepth 3 -maxdepth 3 -type d -not -path '*/.*' | sort | head -n25) + +echo "Running flake tests..." + +for flake in $flakes; do + + if ! REGENERATE=0 ./eval-flake.sh "$flake"; then + status=1 + echo "⌠$flake" + else + echo "✅ $flake" + fi + +done + +exit "$status" diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index ad3ee8881..6aee073e3 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -754,7 +754,7 @@ I will: (if it does, I will tell you how to clean them up.) - create local users (see the list above for the users I'll make) - create a local group ($NIX_BUILD_GROUP_NAME) - - install Nix in to $NIX_ROOT + - install Nix in $NIX_ROOT - create a configuration file in /etc/nix - set up the "default profile" by creating some Nix-related files in $ROOT_HOME diff --git a/scripts/install-systemd-multi-user.sh b/scripts/install-systemd-multi-user.sh index 202a9bb54..a62ed7e3a 100755 --- a/scripts/install-systemd-multi-user.sh +++ b/scripts/install-systemd-multi-user.sh @@ -35,7 +35,7 @@ escape_systemd_env() { # Gather all non-empty proxy environment variables into a string create_systemd_proxy_env() { - vars="http_proxy https_proxy ftp_proxy no_proxy HTTP_PROXY HTTPS_PROXY FTP_PROXY NO_PROXY" + vars="http_proxy https_proxy ftp_proxy all_proxy no_proxy HTTP_PROXY HTTPS_PROXY FTP_PROXY ALL_PROXY NO_PROXY" for v in $vars; do if [ "x${!v:-}" != "x" ]; then echo "Environment=${v}=$(escape_systemd_env ${!v})" diff --git a/scripts/install.in b/scripts/install.in index 7d2e52b26..b4e808d8e 100755 --- a/scripts/install.in +++ b/scripts/install.in @@ -50,6 +50,11 @@ case "$(uname -s).$(uname -m)" in path=@tarballPath_armv7l-linux@ system=armv7l-linux ;; + Linux.riscv64) + hash=@tarballHash_riscv64-linux@ + path=@tarballPath_riscv64-linux@ + system=riscv64-linux + ;; Darwin.x86_64) hash=@tarballHash_x86_64-darwin@ path=@tarballPath_x86_64-darwin@ diff --git a/scripts/nix-profile-daemon.fish.in b/scripts/nix-profile-daemon.fish.in index c23aa64f0..346dce5dd 100644 --- a/scripts/nix-profile-daemon.fish.in +++ b/scripts/nix-profile-daemon.fish.in @@ -28,7 +28,7 @@ else end # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. -if test -n "$NIX_SSH_CERT_FILE" +if test -n "$NIX_SSL_CERT_FILE" : # Allow users to override the NIX_SSL_CERT_FILE else if test -e /etc/ssl/certs/ca-certificates.crt # NixOS, Ubuntu, Debian, Gentoo, Arch set --export NIX_SSL_CERT_FILE /etc/ssl/certs/ca-certificates.crt @@ -44,7 +44,7 @@ else if test -e "$NIX_LINK/etc/ca-bundle.crt" # old cacert in Nix profile set --export NIX_SSL_CERT_FILE "$NIX_LINK/etc/ca-bundle.crt" else # Fall back to what is in the nix profiles, favouring whatever is defined last. - for i in $NIX_PROFILES + for i in (string split ' ' $NIX_PROFILES) if test -e "$i/etc/ssl/certs/ca-bundle.crt" set --export NIX_SSL_CERT_FILE "$i/etc/ssl/certs/ca-bundle.crt" end diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index d256b24ed..eb124c0b5 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -1,4 +1,5 @@ # Only execute this file once per shell. +# This file is tested by tests/installer/default.nix. if [ -n "${__ETC_PROFILE_NIX_SOURCED:-}" ]; then return; fi __ETC_PROFILE_NIX_SOURCED=1 @@ -9,11 +10,9 @@ else NIX_LINK_NEW=$HOME/.local/state/nix/profile fi if [ -e "$NIX_LINK_NEW" ]; then - NIX_LINK="$NIX_LINK_NEW" -else - if [ -t 2 ] && [ -e "$NIX_LINK_NEW" ]; then + if [ -t 2 ] && [ -e "$NIX_LINK" ]; then warning="\033[1;35mwarning:\033[0m" - printf "$warning Both %s and legacy %s exist; using the latter.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2 + printf "$warning Both %s and legacy %s exist; using the former.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2 if [ "$(realpath "$NIX_LINK")" = "$(realpath "$NIX_LINK_NEW")" ]; then printf " Since the profiles match, you can safely delete either of them.\n" 1>&2 else @@ -26,6 +25,7 @@ else printf "$warning Profiles do not match. You should manually migrate from %s to %s.\n" "$NIX_LINK" "$NIX_LINK_NEW" 1>&2 fi fi + NIX_LINK="$NIX_LINK_NEW" fi export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK" @@ -69,4 +69,4 @@ else fi export PATH="$NIX_LINK/bin:@localstatedir@/nix/profiles/default/bin:$PATH" -unset NIX_LINK +unset NIX_LINK NIX_LINK_NEW diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index 44bc96e89..e868399b1 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -1,3 +1,4 @@ +# This file is tested by tests/installer/default.nix. if [ -n "$HOME" ] && [ -n "$USER" ]; then # Set up the per-user profile. @@ -9,11 +10,9 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then NIX_LINK_NEW="$HOME/.local/state/nix/profile" fi if [ -e "$NIX_LINK_NEW" ]; then - NIX_LINK="$NIX_LINK_NEW" - else - if [ -t 2 ] && [ -e "$NIX_LINK_NEW" ]; then + if [ -t 2 ] && [ -e "$NIX_LINK" ]; then warning="\033[1;35mwarning:\033[0m" - printf "$warning Both %s and legacy %s exist; using the latter.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2 + printf "$warning Both %s and legacy %s exist; using the former.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2 if [ "$(realpath "$NIX_LINK")" = "$(realpath "$NIX_LINK_NEW")" ]; then printf " Since the profiles match, you can safely delete either of them.\n" 1>&2 else @@ -26,6 +25,7 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then printf "$warning Profiles do not match. You should manually migrate from %s to %s.\n" "$NIX_LINK" "$NIX_LINK_NEW" 1>&2 fi fi + NIX_LINK="$NIX_LINK_NEW" fi # Set up environment. diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 118468477..82ad7d862 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -11,11 +11,13 @@ #include "machines.hh" #include "shared.hh" +#include "plugin.hh" #include "pathlocks.hh" #include "globals.hh" #include "serialise.hh" #include "build-result.hh" #include "store-api.hh" +#include "strings.hh" #include "derivations.hh" #include "local-store.hh" #include "legacy.hh" @@ -37,7 +39,7 @@ static std::string currentLoad; static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot) { - return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true); + return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri.render()), slot), true); } static bool allSupportedLocally(Store & store, const std::set& requiredFeatures) { @@ -135,7 +137,7 @@ static int main_build_remote(int argc, char * * argv) Machine * bestMachine = nullptr; uint64_t bestLoad = 0; for (auto & m : machines) { - debug("considering building on remote machine '%s'", m.storeUri); + debug("considering building on remote machine '%s'", m.storeUri.render()); if (m.enabled && m.systemSupported(neededSystem) && @@ -202,7 +204,7 @@ static int main_build_remote(int argc, char * * argv) else drvstr = ""; - auto error = HintFmt(errorText); + auto error = HintFmt::fromFormatString(errorText); error % drvstr % neededSystem @@ -232,17 +234,16 @@ static int main_build_remote(int argc, char * * argv) lock = -1; try { + storeUri = bestMachine->storeUri.render(); - Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri)); + Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", storeUri)); sshStore = bestMachine->openStore(); sshStore->connect(); - storeUri = bestMachine->storeUri; - } catch (std::exception & e) { auto msg = chomp(drainFD(5, false)); printError("cannot build on '%s': %s%s", - bestMachine->storeUri, e.what(), + storeUri, e.what(), msg.empty() ? "" : ": " + msg); bestMachine->enabled = false; continue; @@ -262,7 +263,20 @@ connected: auto inputs = readStrings(source); auto wantedOutputs = readStrings(source); - AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true); + AutoCloseFD uploadLock; + { + auto setUpdateLock = [&](auto && fileName){ + uploadLock = openLockFile(currentLoad + "/" + escapeUri(fileName) + ".upload-lock", true); + }; + try { + setUpdateLock(storeUri); + } catch (SysError & e) { + if (e.errNo != ENAMETOOLONG) throw; + // Try again hashing the store URL so we have a shorter path + auto h = hashString(HashAlgorithm::MD5, storeUri); + setUpdateLock(h.to_string(HashFormat::Base64, false)); + } + } { Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri)); diff --git a/doc/internal-api/.gitignore b/src/external-api-docs/.gitignore similarity index 100% rename from doc/internal-api/.gitignore rename to src/external-api-docs/.gitignore diff --git a/src/external-api-docs/.version b/src/external-api-docs/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/external-api-docs/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/external-api-docs/README.md b/src/external-api-docs/README.md new file mode 100644 index 000000000..8760ac88b --- /dev/null +++ b/src/external-api-docs/README.md @@ -0,0 +1,121 @@ +# Getting started + +> **Warning** These bindings are **experimental**, which means they can change +> at any time or be removed outright; nevertheless the plan is to provide a +> stable external C API to the Nix language and the Nix store. + +The language library allows evaluating Nix expressions and interacting with Nix +language values. The Nix store API is still rudimentary, and only allows +initialising and connecting to a store for the Nix language evaluator to +interact with. + +Currently there are two ways to interface with the Nix language evaluator +programmatically: + +1. Embedding the evaluator +2. Writing language plug-ins + +Embedding means you link the Nix C libraries in your program and use them from +there. Adding a plug-in means you make a library that gets loaded by the Nix +language evaluator, specified through a configuration option. + +Many of the components and mechanisms involved are not yet documented, therefore +please refer to the [Nix source code](https://github.com/NixOS/nix/) for +details. Additions to in-code documentation and the reference manual are highly +appreciated. + +The following examples, for simplicity, don't include error handling. See the +[Handling errors](@ref errors) section for more information. + +# Embedding the Nix Evaluator{#nix_evaluator_example} + +In this example we programmatically start the Nix language evaluator with a +dummy store (that has no store paths and cannot be written to), and evaluate the +Nix expression `builtins.nixVersion`. + +**main.c:** + +```C +#include +#include +#include +#include +#include +#include + +// NOTE: This example lacks all error handling. Production code must check for +// errors, as some return values will be undefined. + +void my_get_string_cb(const char * start, unsigned int n, void * user_data) +{ + *((char **) user_data) = strdup(start); +} + +int main() +{ + nix_libexpr_init(NULL); + + Store * store = nix_store_open(NULL, "dummy://", NULL); + EvalState * state = nix_state_create(NULL, NULL, store); // empty search path (NIX_PATH) + Value * value = nix_alloc_value(NULL, state); + + nix_expr_eval_from_string(NULL, state, "builtins.nixVersion", ".", value); + nix_value_force(NULL, state, value); + + char * version; + nix_get_string(NULL, value, my_get_string_cb, &version); + printf("Nix version: %s\n", version); + + free(version); + nix_gc_decref(NULL, value); + nix_state_free(state); + nix_store_free(store); + return 0; +} +``` + +**Usage:** + +```ShellSession +$ gcc main.c $(pkg-config nix-expr-c --libs --cflags) -o main +$ ./main +Nix version: 2.17 +``` + +# Writing a Nix language plug-in + +In this example we add a custom primitive operation (_primop_) to `builtins`. It +will increment the argument if it is an integer and throw an error otherwise. + +**plugin.c:** + +```C +#include +#include +#include + +void increment(void* user_data, nix_c_context* ctx, EvalState* state, Value** args, Value* v) { + nix_value_force(NULL, state, args[0]); + if (nix_get_type(NULL, args[0]) == NIX_TYPE_INT) { + nix_init_int(NULL, v, nix_get_int(NULL, args[0]) + 1); + } else { + nix_set_err_msg(ctx, NIX_ERR_UNKNOWN, "First argument should be an integer."); + } +} + +void nix_plugin_entry() { + const char* args[] = {"n", NULL}; + PrimOp *p = nix_alloc_primop(NULL, increment, 1, "increment", args, "Example custom built-in function: increments an integer", NULL); + nix_register_primop(NULL, p); + nix_gc_decref(NULL, p); +} +``` + +**Usage:** + +```ShellSession +$ gcc plugin.c $(pkg-config nix-expr-c --libs --cflags) -shared -o plugin.so +$ nix --plugin-files ./plugin.so repl +nix-repl> builtins.increment 1 +2 +``` diff --git a/src/external-api-docs/doxygen.cfg.in b/src/external-api-docs/doxygen.cfg.in new file mode 100644 index 000000000..1be71d895 --- /dev/null +++ b/src/external-api-docs/doxygen.cfg.in @@ -0,0 +1,58 @@ +# Doxyfile 1.9.5 + +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by +# double-quotes, unless you are using Doxywizard) that should identify the +# project for which the documentation is generated. This name is used in the +# title of most generated pages and in a few other places. +# The default value is: My Project. + +PROJECT_NAME = "Nix" + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. This +# could be handy for archiving the generated documentation or if some version +# control system is used. + +PROJECT_NUMBER = @PROJECT_NUMBER@ + +OUTPUT_DIRECTORY = @OUTPUT_DIRECTORY@ + +# Using the PROJECT_BRIEF tag one can provide an optional one line description +# for a project that appears at the top of each page and should give viewer a +# quick idea about the purpose of the project. Keep the description short. + +PROJECT_BRIEF = "Nix, the purely functional package manager: C API (experimental)" + +# If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output. +# The default value is: YES. + +GENERATE_LATEX = NO + +# The INPUT tag is used to specify the files and/or directories that contain +# documented source files. You may enter file names like myfile.cpp or +# directories like /usr/src/myproject. Separate the files or directories with +# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING +# Note: If this tag is empty the current directory is searched. + +# FIXME Make this list more maintainable somehow. We could maybe generate this +# in the Makefile, but we would need to change how `.in` files are preprocessed +# so they can expand variables despite configure variables. + +INPUT = \ + @src@/src/libutil-c \ + @src@/src/libexpr-c \ + @src@/src/libstore-c \ + @src@/doc/external-api/README.md + +FILE_PATTERNS = nix_api_*.h *.md + +# The INCLUDE_PATH tag can be used to specify one or more directories that +# contain include files that are not input files but should be processed by the +# preprocessor. Note that the INCLUDE_PATH is not recursive, so the setting of +# RECURSIVE has no effect here. +# This tag requires that the tag SEARCH_INCLUDES is set to YES. + +EXCLUDE_PATTERNS = *_internal.h +GENERATE_TREEVIEW = YES +OPTIMIZE_OUTPUT_FOR_C = YES + +USE_MDFILE_AS_MAINPAGE = doc/external-api/README.md diff --git a/src/external-api-docs/meson.build b/src/external-api-docs/meson.build new file mode 100644 index 000000000..62474ffe4 --- /dev/null +++ b/src/external-api-docs/meson.build @@ -0,0 +1,31 @@ +project('nix-external-api-docs', + version : files('.version'), + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +fs = import('fs') + +doxygen_cfg = configure_file( + input : 'doxygen.cfg.in', + output : 'doxygen.cfg', + configuration : { + 'PROJECT_NUMBER': meson.project_version(), + 'OUTPUT_DIRECTORY' : meson.current_build_dir(), + 'src' : fs.parent(fs.parent(meson.project_source_root())), + }, +) + +doxygen = find_program('doxygen', native : true, required : true) + +custom_target( + 'external-api-docs', + command : [ doxygen , doxygen_cfg ], + input : [ + doxygen_cfg, + ], + output : 'html', + install : true, + install_dir : get_option('datadir') / 'doc/nix/external-api', + build_always_stale : true, +) diff --git a/src/external-api-docs/package.nix b/src/external-api-docs/package.nix new file mode 100644 index 000000000..743b3e9b7 --- /dev/null +++ b/src/external-api-docs/package.nix @@ -0,0 +1,59 @@ +{ lib +, mkMesonDerivation + +, meson +, ninja +, doxygen + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-external-api-docs"; + inherit version; + + workDir = ./.; + fileset = + let + cpp = fileset.fileFilter (file: file.hasExt "cc" || file.hasExt "h"); + in + fileset.unions [ + ./.version + ../../.version + ./meson.build + ./doxygen.cfg.in + ./README.md + # Source is not compiled, but still must be available for Doxygen + # to gather comments. + (cpp ../libexpr-c) + (cpp ../libstore-c) + (cpp ../libutil-c) + ]; + + nativeBuildInputs = [ + meson + ninja + doxygen + ]; + + preConfigure = + '' + chmod u+w ./.version + echo ${finalAttrs.version} > ./.version + ''; + + postInstall = '' + mkdir -p ''${!outputDoc}/nix-support + echo "doc external-api-docs $out/share/doc/nix/external-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products + ''; + + meta = { + platforms = lib.platforms.all; + }; +}) diff --git a/src/internal-api-docs/.gitignore b/src/internal-api-docs/.gitignore new file mode 100644 index 000000000..dab28b6b0 --- /dev/null +++ b/src/internal-api-docs/.gitignore @@ -0,0 +1,3 @@ +/doxygen.cfg +/html +/latex diff --git a/src/internal-api-docs/.version b/src/internal-api-docs/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/internal-api-docs/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/doc/internal-api/doxygen.cfg.in b/src/internal-api-docs/doxygen.cfg.in similarity index 84% rename from doc/internal-api/doxygen.cfg.in rename to src/internal-api-docs/doxygen.cfg.in index 6c6c325bd..f1ef75b38 100644 --- a/doc/internal-api/doxygen.cfg.in +++ b/src/internal-api-docs/doxygen.cfg.in @@ -12,7 +12,9 @@ PROJECT_NAME = "Nix" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = @PACKAGE_VERSION@ +PROJECT_NUMBER = @PROJECT_NUMBER@ + +OUTPUT_DIRECTORY = @OUTPUT_DIRECTORY@ # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a @@ -36,27 +38,27 @@ GENERATE_LATEX = NO # so they can expand variables despite configure variables. INPUT = \ - src/libcmd \ - src/libexpr \ - src/libexpr/flake \ - tests/unit/libexpr \ - tests/unit/libexpr/value \ - tests/unit/libexpr/test \ - tests/unit/libexpr/test/value \ - src/libexpr/value \ - src/libfetchers \ - src/libmain \ - src/libstore \ - src/libstore/build \ - src/libstore/builtins \ - tests/unit/libstore \ - tests/unit/libstore/test \ - src/libutil \ - tests/unit/libutil \ - tests/unit/libutil/test \ - src/nix \ - src/nix-env \ - src/nix-store + @src@/libcmd \ + @src@/libexpr \ + @src@/libexpr/flake \ + @src@/nix-expr-tests \ + @src@/nix-expr-tests/value \ + @src@/nix-expr-test-support/test \ + @src@/nix-expr-test-support/test/value \ + @src@/libexpr/value \ + @src@/libfetchers \ + @src@/libmain \ + @src@/libstore \ + @src@/libstore/build \ + @src@/libstore/builtins \ + @src@/nix-store-tests \ + @src@/nix-store-test-support/test \ + @src@/libutil \ + @src@/nix-util-tests \ + @src@/nix-util-test-support/test \ + @src@/nix \ + @src@/nix-env \ + @src@/nix-store # If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names # in the source code. If set to NO, only conditional compilation will be diff --git a/src/internal-api-docs/meson.build b/src/internal-api-docs/meson.build new file mode 100644 index 000000000..54eb7e5dd --- /dev/null +++ b/src/internal-api-docs/meson.build @@ -0,0 +1,31 @@ +project('nix-internal-api-docs', + version : files('.version'), + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +fs = import('fs') + +doxygen_cfg = configure_file( + input : 'doxygen.cfg.in', + output : 'doxygen.cfg', + configuration : { + 'PROJECT_NUMBER': meson.project_version(), + 'OUTPUT_DIRECTORY' : meson.current_build_dir(), + 'src' : fs.parent(fs.parent(meson.project_source_root())) / 'src', + }, +) + +doxygen = find_program('doxygen', native : true, required : true) + +custom_target( + 'internal-api-docs', + command : [ doxygen , doxygen_cfg ], + input : [ + doxygen_cfg, + ], + output : 'html', + install : true, + install_dir : get_option('datadir') / 'doc/nix/internal-api', + build_always_stale : true, +) diff --git a/src/internal-api-docs/package.nix b/src/internal-api-docs/package.nix new file mode 100644 index 000000000..07ca6d4d9 --- /dev/null +++ b/src/internal-api-docs/package.nix @@ -0,0 +1,54 @@ +{ lib +, mkMesonDerivation + +, meson +, ninja +, doxygen + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-internal-api-docs"; + inherit version; + + workDir = ./.; + fileset = let + cpp = fileset.fileFilter (file: file.hasExt "cc" || file.hasExt "hh"); + in fileset.unions [ + ./.version + ../../.version + ./meson.build + ./doxygen.cfg.in + # Source is not compiled, but still must be available for Doxygen + # to gather comments. + (cpp ../.) + ]; + + nativeBuildInputs = [ + meson + ninja + doxygen + ]; + + preConfigure = + '' + chmod u+w ./.version + echo ${finalAttrs.version} > ./.version + ''; + + postInstall = '' + mkdir -p ''${!outputDoc}/nix-support + echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products + ''; + + meta = { + platforms = lib.platforms.all; + }; +}) diff --git a/src/libcmd/.version b/src/libcmd/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libcmd/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libcmd/build-utils-meson b/src/libcmd/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libcmd/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc index c5eb93c5d..905e70f32 100644 --- a/src/libcmd/built-path.cc +++ b/src/libcmd/built-path.cc @@ -1,6 +1,7 @@ #include "built-path.hh" #include "derivations.hh" #include "store-api.hh" +#include "comparator.hh" #include @@ -8,30 +9,24 @@ namespace nix { -#define CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, COMPARATOR) \ - bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \ - { \ - const MY_TYPE* me = this; \ - auto fields1 = std::tie(*me->drvPath, me->FIELD); \ - me = &other; \ - auto fields2 = std::tie(*me->drvPath, me->FIELD); \ - return fields1 COMPARATOR fields2; \ - } -#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, ==) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <) +// Custom implementation to avoid `ref` ptr equality +GENERATE_CMP_EXT( + , + std::strong_ordering, + SingleBuiltPathBuilt, + *me->drvPath, + me->output); -#define FIELD_TYPE std::pair -CMP(SingleBuiltPath, SingleBuiltPathBuilt, output) -#undef FIELD_TYPE +// Custom implementation to avoid `ref` ptr equality -#define FIELD_TYPE std::map -CMP(SingleBuiltPath, BuiltPathBuilt, outputs) -#undef FIELD_TYPE - -#undef CMP -#undef CMP_ONE +// TODO no `GENERATE_CMP_EXT` because no `std::set::operator<=>` on +// Darwin, per header. +GENERATE_EQUAL( + , + BuiltPathBuilt ::, + BuiltPathBuilt, + *me->drvPath, + me->outputs); StorePath SingleBuiltPath::outPath() const { diff --git a/src/libcmd/built-path.hh b/src/libcmd/built-path.hh index 99917e0ee..dc78d3e59 100644 --- a/src/libcmd/built-path.hh +++ b/src/libcmd/built-path.hh @@ -18,7 +18,8 @@ struct SingleBuiltPathBuilt { static SingleBuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view); nlohmann::json toJSON(const StoreDirConfig & store) const; - DECLARE_CMP(SingleBuiltPathBuilt); + bool operator ==(const SingleBuiltPathBuilt &) const noexcept; + std::strong_ordering operator <=>(const SingleBuiltPathBuilt &) const noexcept; }; using _SingleBuiltPathRaw = std::variant< @@ -33,6 +34,9 @@ struct SingleBuiltPath : _SingleBuiltPathRaw { using Opaque = DerivedPathOpaque; using Built = SingleBuiltPathBuilt; + bool operator == (const SingleBuiltPath &) const = default; + auto operator <=> (const SingleBuiltPath &) const = default; + inline const Raw & raw() const { return static_cast(*this); } @@ -59,11 +63,13 @@ struct BuiltPathBuilt { ref drvPath; std::map outputs; + bool operator == (const BuiltPathBuilt &) const noexcept; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //std::strong_ordering operator <=> (const BuiltPathBuilt &) const noexcept; + std::string to_string(const StoreDirConfig & store) const; static BuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view); nlohmann::json toJSON(const StoreDirConfig & store) const; - - DECLARE_CMP(BuiltPathBuilt); }; using _BuiltPathRaw = std::variant< @@ -82,6 +88,10 @@ struct BuiltPath : _BuiltPathRaw { using Opaque = DerivedPathOpaque; using Built = BuiltPathBuilt; + bool operator == (const BuiltPath &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=> (const BuiltPath &) const = default; + inline const Raw & raw() const { return static_cast(*this); } diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 369fa6004..67fef1909 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -1,3 +1,5 @@ +#include + #include "command.hh" #include "markdown.hh" #include "store-api.hh" @@ -6,8 +8,7 @@ #include "nixexpr.hh" #include "profiles.hh" #include "repl.hh" - -#include +#include "strings.hh" extern char * * environ __attribute__((weak)); @@ -127,12 +128,12 @@ ref EvalCommand::getEvalState() if (!evalState) { evalState = #if HAVE_BOEHMGC - std::allocate_shared(traceable_allocator(), - searchPath, getEvalStore(), getStore()) + std::allocate_shared( + traceable_allocator(), #else std::make_shared( - searchPath, getEvalStore(), getStore()) #endif + lookupPath, getEvalStore(), fetchSettings, evalSettings, getStore()) ; evalState->repair = repair; @@ -148,7 +149,7 @@ MixOperateOnOptions::MixOperateOnOptions() { addFlag({ .longName = "derivation", - .description = "Operate on the [store derivation](../../glossary.md#gloss-store-derivation) rather than its outputs.", + .description = "Operate on the [store derivation](@docroot@/glossary.md#gloss-store-derivation) rather than its outputs.", .category = installablesCategory, .handler = {&operateOn, OperateOn::Derivation}, }); diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 444ff81c9..fcef92487 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -1,18 +1,57 @@ +#include "fetch-settings.hh" #include "eval-settings.hh" #include "common-eval-args.hh" #include "shared.hh" +#include "config-global.hh" #include "filetransfer.hh" #include "eval.hh" #include "fetchers.hh" #include "registry.hh" #include "flake/flakeref.hh" +#include "flake/settings.hh" #include "store-api.hh" #include "command.hh" #include "tarball.hh" #include "fetch-to-store.hh" +#include "compatibility-settings.hh" +#include "eval-settings.hh" namespace nix { +fetchers::Settings fetchSettings; + +static GlobalConfig::Register rFetchSettings(&fetchSettings); + +EvalSettings evalSettings { + settings.readOnlyMode, + { + { + "flake", + [](ref store, std::string_view rest) { + experimentalFeatureSettings.require(Xp::Flakes); + // FIXME `parseFlakeRef` should take a `std::string_view`. + auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false); + debug("fetching flake search path element '%s''", rest); + auto storePath = flakeRef.resolve(store).fetchTree(store).first; + return store->toRealPath(storePath); + }, + }, + }, +}; + +static GlobalConfig::Register rEvalSettings(&evalSettings); + + +flake::Settings flakeSettings; + +static GlobalConfig::Register rFlakeSettings(&flakeSettings); + + +CompatibilitySettings compatibilitySettings {}; + +static GlobalConfig::Register rCompatibilitySettings(&compatibilitySettings); + + MixEvalArgs::MixEvalArgs() { addFlag({ @@ -20,7 +59,7 @@ MixEvalArgs::MixEvalArgs() .description = "Pass the value *expr* as the argument *name* to Nix functions.", .category = category, .labels = {"name", "expr"}, - .handler = {[&](std::string name, std::string expr) { autoArgs[name] = 'E' + expr; }} + .handler = {[&](std::string name, std::string expr) { autoArgs.insert_or_assign(name, AutoArg{AutoArgExpr{expr}}); }} }); addFlag({ @@ -28,87 +67,40 @@ MixEvalArgs::MixEvalArgs() .description = "Pass the string *string* as the argument *name* to Nix functions.", .category = category, .labels = {"name", "string"}, - .handler = {[&](std::string name, std::string s) { autoArgs[name] = 'S' + s; }}, + .handler = {[&](std::string name, std::string s) { autoArgs.insert_or_assign(name, AutoArg{AutoArgString{s}}); }}, + }); + + addFlag({ + .longName = "arg-from-file", + .description = "Pass the contents of file *path* as the argument *name* to Nix functions.", + .category = category, + .labels = {"name", "path"}, + .handler = {[&](std::string name, std::string path) { autoArgs.insert_or_assign(name, AutoArg{AutoArgFile{path}}); }}, + .completer = completePath + }); + + addFlag({ + .longName = "arg-from-stdin", + .description = "Pass the contents of stdin as the argument *name* to Nix functions.", + .category = category, + .labels = {"name"}, + .handler = {[&](std::string name) { autoArgs.insert_or_assign(name, AutoArg{AutoArgStdin{}}); }}, }); addFlag({ .longName = "include", .shortName = 'I', .description = R"( - Add *path* to the Nix search path. The Nix search path is - initialized from the colon-separated [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) environment - variable, and is used to look up the location of Nix expressions using [paths](@docroot@/language/values.md#type-path) enclosed in angle - brackets (i.e., ``). + Add *path* to search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md) - For instance, passing + This option may be given multiple times. - ``` - -I /home/eelco/Dev - -I /etc/nixos - ``` - - will cause Nix to look for paths relative to `/home/eelco/Dev` and - `/etc/nixos`, in that order. This is equivalent to setting the - `NIX_PATH` environment variable to - - ``` - /home/eelco/Dev:/etc/nixos - ``` - - It is also possible to match paths against a prefix. For example, - passing - - ``` - -I nixpkgs=/home/eelco/Dev/nixpkgs-branch - -I /etc/nixos - ``` - - will cause Nix to search for `` in - `/home/eelco/Dev/nixpkgs-branch/path` and `/etc/nixos/nixpkgs/path`. - - If a path in the Nix search path starts with `http://` or `https://`, - it is interpreted as the URL of a tarball that will be downloaded and - unpacked to a temporary location. The tarball must consist of a single - top-level directory. For example, passing - - ``` - -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/master.tar.gz - ``` - - tells Nix to download and use the current contents of the `master` - branch in the `nixpkgs` repository. - - The URLs of the tarballs from the official `nixos.org` channels - (see [the manual page for `nix-channel`](../nix-channel.md)) can be - abbreviated as `channel:`. For instance, the - following two flags are equivalent: - - ``` - -I nixpkgs=channel:nixos-21.05 - -I nixpkgs=https://nixos.org/channels/nixos-21.05/nixexprs.tar.xz - ``` - - You can also fetch source trees using [flake URLs](./nix3-flake.md#url-like-syntax) and add them to the - search path. For instance, - - ``` - -I nixpkgs=flake:nixpkgs - ``` - - specifies that the prefix `nixpkgs` shall refer to the source tree - downloaded from the `nixpkgs` entry in the flake registry. Similarly, - - ``` - -I nixpkgs=flake:github:NixOS/nixpkgs/nixos-22.05 - ``` - - makes `` refer to a particular branch of the - `NixOS/nixpkgs` repository on GitHub. + Paths added through `-I` take precedence over the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path) and the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH). )", .category = category, .labels = {"path"}, .handler = {[&](std::string s) { - searchPath.elements.emplace_back(SearchPath::Elem::parse(s)); + lookupPath.elements.emplace_back(LookupPath::Elem::parse(s)); }} }); @@ -127,8 +119,8 @@ MixEvalArgs::MixEvalArgs() .category = category, .labels = {"original-ref", "resolved-ref"}, .handler = {[&](std::string _from, std::string _to) { - auto from = parseFlakeRef(_from, absPath(".")); - auto to = parseFlakeRef(_to, absPath(".")); + auto from = parseFlakeRef(fetchSettings, _from, absPath(".")); + auto to = parseFlakeRef(fetchSettings, _to, absPath(".")); fetchers::Attrs extraAttrs; if (to.subdir != "") extraAttrs["dir"] = to.subdir; fetchers::overrideRegistry(from.input, to.input, extraAttrs); @@ -154,13 +146,23 @@ MixEvalArgs::MixEvalArgs() Bindings * MixEvalArgs::getAutoArgs(EvalState & state) { auto res = state.buildBindings(autoArgs.size()); - for (auto & i : autoArgs) { + for (auto & [name, arg] : autoArgs) { auto v = state.allocValue(); - if (i.second[0] == 'E') - state.mkThunk_(*v, state.parseExprFromString(i.second.substr(1), state.rootPath("."))); - else - v->mkString(((std::string_view) i.second).substr(1)); - res.insert(state.symbols.create(i.first), v); + std::visit(overloaded { + [&](const AutoArgExpr & arg) { + state.mkThunk_(*v, state.parseExprFromString(arg.expr, compatibilitySettings.nixShellShebangArgumentsRelativeToScript ? state.rootPath(absPath(getCommandBaseDir())) : state.rootPath("."))); + }, + [&](const AutoArgString & arg) { + v->mkString(arg.s); + }, + [&](const AutoArgFile & arg) { + v->mkString(readFile(arg.path.string())); + }, + [&](const AutoArgStdin & arg) { + v->mkString(readFile(STDIN_FILENO)); + } + }, arg); + res.insert(state.symbols.create(name), v); } return res.finish(); } @@ -176,7 +178,7 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas else if (hasPrefix(s, "flake:")) { experimentalFeatureSettings.require(Xp::Flakes); - auto flakeRef = parseFlakeRef(std::string(s.substr(6)), {}, true, false); + auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false); auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first; return state.rootPath(CanonPath(state.store->toRealPath(storePath))); } diff --git a/src/libcmd/common-eval-args.hh b/src/libcmd/common-eval-args.hh index 2eb63e15d..c62365b32 100644 --- a/src/libcmd/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh @@ -6,13 +6,42 @@ #include "common-args.hh" #include "search-path.hh" +#include + namespace nix { class Store; + +namespace fetchers { struct Settings; } + class EvalState; +struct EvalSettings; +struct CompatibilitySettings; class Bindings; struct SourcePath; +namespace flake { struct Settings; } + +/** + * @todo Get rid of global setttings variables + */ +extern fetchers::Settings fetchSettings; + +/** + * @todo Get rid of global setttings variables + */ +extern EvalSettings evalSettings; + +/** + * @todo Get rid of global setttings variables + */ +extern flake::Settings flakeSettings; + +/** + * Settings that control behaviors that have changed since Nix 2.3. + */ +extern CompatibilitySettings compatibilitySettings; + struct MixEvalArgs : virtual Args, virtual MixRepair { static constexpr auto category = "Common evaluation options"; @@ -21,14 +50,24 @@ struct MixEvalArgs : virtual Args, virtual MixRepair Bindings * getAutoArgs(EvalState & state); - SearchPath searchPath; + LookupPath lookupPath; std::optional evalStoreUrl; private: - std::map autoArgs; + struct AutoArgExpr { std::string expr; }; + struct AutoArgString { std::string s; }; + struct AutoArgFile { std::filesystem::path path; }; + struct AutoArgStdin { }; + + using AutoArg = std::variant; + + std::map autoArgs; }; +/** + * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) + */ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * baseDir = nullptr); } diff --git a/src/libcmd/compatibility-settings.hh b/src/libcmd/compatibility-settings.hh new file mode 100644 index 000000000..a129a957a --- /dev/null +++ b/src/libcmd/compatibility-settings.hh @@ -0,0 +1,36 @@ +#pragma once +#include "config.hh" + +namespace nix { +struct CompatibilitySettings : public Config +{ + + CompatibilitySettings() = default; + + // Added in Nix 2.24, July 2024. + Setting nixShellAlwaysLooksForShellNix{this, true, "nix-shell-always-looks-for-shell-nix", R"( + Before Nix 2.24, [`nix-shell`](@docroot@/command-ref/nix-shell.md) would only look at `shell.nix` if it was in the working directory - when no file was specified. + + Since Nix 2.24, `nix-shell` always looks for a `shell.nix`, whether that's in the working directory, or in a directory that was passed as an argument. + + You may set this to `false` to temporarily revert to the behavior of Nix 2.23 and older. + + Using this setting is not recommended. + It will be deprecated and removed. + )"}; + + // Added in Nix 2.24, July 2024. + Setting nixShellShebangArgumentsRelativeToScript{ + this, true, "nix-shell-shebang-arguments-relative-to-script", R"( + Before Nix 2.24, relative file path expressions in arguments in a `nix-shell` shebang were resolved relative to the working directory. + + Since Nix 2.24, `nix-shell` resolves these paths in a manner that is relative to the [base directory](@docroot@/glossary.md#gloss-base-directory), defined as the script's directory. + + You may set this to `false` to temporarily revert to the behavior of Nix 2.23 and older. + + Using this setting is not recommended. + It will be deprecated and removed. + )"}; +}; + +}; diff --git a/src/libcmd/installable-attr-path.cc b/src/libcmd/installable-attr-path.cc index 3ec1c1614..8917e7a01 100644 --- a/src/libcmd/installable-attr-path.cc +++ b/src/libcmd/installable-attr-path.cc @@ -75,6 +75,8 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths() std::set outputsToInstall; for (auto & output : packageInfo.queryOutputs(false, true)) outputsToInstall.insert(output.first); + if (outputsToInstall.empty()) + outputsToInstall.insert("out"); return OutputsSpec::Names { std::move(outputsToInstall) }; }, [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec { diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index ddec7537b..8796ad5ba 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -43,20 +43,6 @@ std::vector InstallableFlake::getActualAttrPaths() return res; } -Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake) -{ - auto vFlake = state.allocValue(); - - callFlake(state, lockedFlake, *vFlake); - - auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); - assert(aOutputs); - - state.forceValue(*aOutputs->value, aOutputs->value->determinePos(noPos)); - - return aOutputs->value; -} - static std::string showAttrPaths(const std::vector & paths) { std::string s; @@ -106,9 +92,14 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() fmt("while evaluating the flake output attribute '%s'", attrPath))) { return { *derivedPathWithInfo }; + } else { + throw Error( + "expected flake output attribute '%s' to be a derivation or path but found %s: %s", + attrPath, + showType(v), + ValuePrinter(*this->state, v, errorPrintOptions) + ); } - else - throw Error("flake output attribute '%s' is not a derivation or path", attrPath); } auto drvPath = attr->forceDerivation(); @@ -205,7 +196,8 @@ std::shared_ptr InstallableFlake::getLockedFlake() const flake::LockFlags lockFlagsApplyConfig = lockFlags; // FIXME why this side effect? lockFlagsApplyConfig.applyNixConfig = true; - _lockedFlake = std::make_shared(lockFlake(*state, flakeRef, lockFlagsApplyConfig)); + _lockedFlake = std::make_shared(lockFlake( + flakeSettings, *state, flakeRef, lockFlagsApplyConfig)); } return _lockedFlake; } diff --git a/src/libcmd/installable-flake.hh b/src/libcmd/installable-flake.hh index 314918c14..8e0a232ef 100644 --- a/src/libcmd/installable-flake.hh +++ b/src/libcmd/installable-flake.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "common-eval-args.hh" #include "installable-value.hh" namespace nix { @@ -52,8 +53,6 @@ struct InstallableFlake : InstallableValue std::vector getActualAttrPaths(); - Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake); - DerivedPathsWithInfo toDerivedPaths() override; std::pair toValue(EvalState & state) override; @@ -80,7 +79,7 @@ struct InstallableFlake : InstallableValue */ static inline FlakeRef defaultNixpkgsFlakeRef() { - return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}}); + return FlakeRef::fromAttrs(fetchSettings, {{"type","indirect"}, {"id", "nixpkgs"}}); } ref openEvalCache( diff --git a/src/libcmd/installable-value.hh b/src/libcmd/installable-value.hh index f300d392b..798cb5e1a 100644 --- a/src/libcmd/installable-value.hh +++ b/src/libcmd/installable-value.hh @@ -66,7 +66,7 @@ struct ExtraPathInfoValue : ExtraPathInfo }; /** - * An Installable which corresponds a Nix langauge value, in addition to + * An Installable which corresponds a Nix language value, in addition to * a collection of \ref DerivedPath "derived paths". */ struct InstallableValue : Installable diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 16d25d3cf..0fe956ec0 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -27,6 +27,8 @@ #include +#include "strings-inline.hh" + namespace nix { void completeFlakeInputPath( @@ -129,7 +131,7 @@ MixFlakeOptions::MixFlakeOptions() lockFlags.writeLockFile = false; lockFlags.inputOverrides.insert_or_assign( flake::parseInputPath(inputPath), - parseFlakeRef(flakeRef, absPath(getCommandBaseDir()), true)); + parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir()), true)); }}, .completer = {[&](AddCompletions & completions, size_t n, std::string_view prefix) { if (n == 0) { @@ -146,7 +148,7 @@ MixFlakeOptions::MixFlakeOptions() .category = category, .labels = {"flake-lock-path"}, .handler = {[&](std::string lockFilePath) { - lockFlags.referenceLockFilePath = lockFilePath; + lockFlags.referenceLockFilePath = {getFSSourceAccessor(), CanonPath(absPath(lockFilePath))}; }}, .completer = completePath }); @@ -170,14 +172,15 @@ MixFlakeOptions::MixFlakeOptions() .handler = {[&](std::string flakeRef) { auto evalState = getEvalState(); auto flake = flake::lockFlake( + flakeSettings, *evalState, - parseFlakeRef(flakeRef, absPath(getCommandBaseDir())), + parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir())), { .writeLockFile = false }); for (auto & [inputName, input] : flake.lockFile.root->inputs) { auto input2 = flake.lockFile.findInput({inputName}); // resolve 'follows' nodes if (auto input3 = std::dynamic_pointer_cast(input2)) { overrideRegistry( - fetchers::Input::fromAttrs({{"type","indirect"}, {"id", inputName}}), + fetchers::Input::fromAttrs(fetchSettings, {{"type","indirect"}, {"id", inputName}}), input3->lockedRef.input, {}); } @@ -288,11 +291,11 @@ void SourceExprCommand::completeInstallable(AddCompletions & completions, std::s state->autoCallFunction(*autoArgs, v1, v2); if (v2.type() == nAttrs) { - for (auto & i : *v2.attrs) { - std::string name = state->symbols[i.name]; + for (auto & i : *v2.attrs()) { + std::string_view name = state->symbols[i.name]; if (name.find(searchWord) == 0) { if (prefix_ == "") - completions.add(name); + completions.add(std::string(name)); else completions.add(prefix_ + "." + name); } @@ -338,10 +341,11 @@ void completeFlakeRefWithFragment( auto flakeRefS = std::string(prefix.substr(0, hash)); // TODO: ideally this would use the command base directory instead of assuming ".". - auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath(".")); + auto flakeRef = parseFlakeRef(fetchSettings, expandTilde(flakeRefS), absPath(".")); auto evalCache = openEvalCache(*evalState, - std::make_shared(lockFlake(*evalState, flakeRef, lockFlags))); + std::make_shared(lockFlake( + flakeSettings, *evalState, flakeRef, lockFlags))); auto root = evalCache->getRoot(); @@ -372,6 +376,7 @@ void completeFlakeRefWithFragment( auto attrPath2 = (*attr)->getAttrPath(attr2); /* Strip the attrpath prefix. */ attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size()); + // FIXME: handle names with dots completions.add(flakeRefS + "#" + prefixRoot + concatStringsSep(".", evalState->symbols.resolve(attrPath2))); } } @@ -403,7 +408,7 @@ void completeFlakeRef(AddCompletions & completions, ref store, std::strin Args::completeDir(completions, 0, prefix); /* Look for registry entries that match the prefix. */ - for (auto & registry : fetchers::getRegistries(store)) { + for (auto & registry : fetchers::getRegistries(fetchSettings, store)) { for (auto & entry : registry->entries) { auto from = entry.from.to_string(); if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) { @@ -442,13 +447,10 @@ ref openEvalCache( EvalState & state, std::shared_ptr lockedFlake) { - auto fingerprint = lockedFlake->getFingerprint(); - return make_ref( - evalSettings.useEvalCache && evalSettings.pureEval - ? std::optional { std::cref(fingerprint) } - : std::nullopt, - state, - [&state, lockedFlake]() + auto fingerprint = evalSettings.useEvalCache && evalSettings.pureEval + ? lockedFlake->getFingerprint(state.store) + : std::nullopt; + auto rootLoader = [&state, lockedFlake]() { /* For testing whether the evaluation cache is complete. */ @@ -460,11 +462,21 @@ ref openEvalCache( state.forceAttrs(*vFlake, noPos, "while parsing cached flake data"); - auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); + auto aOutputs = vFlake->attrs()->get(state.symbols.create("outputs")); assert(aOutputs); return aOutputs->value; - }); + }; + + if (fingerprint) { + auto search = state.evalCaches.find(fingerprint.value()); + if (search == state.evalCaches.end()) { + search = state.evalCaches.emplace(fingerprint.value(), make_ref(fingerprint, state, rootLoader)).first; + } + return search->second; + } else { + return make_ref(std::nullopt, state, rootLoader); + } } Installables SourceExprCommand::parseInstallables( @@ -527,7 +539,8 @@ Installables SourceExprCommand::parseInstallables( } try { - auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, absPath(getCommandBaseDir())); + auto [flakeRef, fragment] = parseFlakeRefWithFragment( + fetchSettings, std::string { prefix }, absPath(getCommandBaseDir())); result.push_back(make_ref( this, getEvalState(), @@ -594,6 +607,37 @@ std::vector Installable::build( return res; } +static void throwBuildErrors( + std::vector & buildResults, + const Store & store) +{ + std::vector failed; + for (auto & buildResult : buildResults) { + if (!buildResult.success()) { + failed.push_back(buildResult); + } + } + + auto failedResult = failed.begin(); + if (failedResult != failed.end()) { + if (failed.size() == 1) { + failedResult->rethrow(); + } else { + StringSet failedPaths; + for (; failedResult != failed.end(); failedResult++) { + if (!failedResult->errorMsg.empty()) { + logError(ErrorInfo{ + .level = lvlError, + .msg = failedResult->errorMsg, + }); + } + failedPaths.insert(failedResult->path.to_string(store)); + } + throw Error("build of %s failed", concatStringsSep(", ", quoteStrings(failedPaths))); + } + } +} + std::vector, BuiltPathWithResult>> Installable::build2( ref evalStore, ref store, @@ -655,10 +699,9 @@ std::vector, BuiltPathWithResult>> Installable::build if (settings.printMissing) printMissing(store, pathsToBuild, lvlInfo); - for (auto & buildResult : store->buildPathsWithResults(pathsToBuild, bMode, evalStore)) { - if (!buildResult.success()) - buildResult.rethrow(); - + auto buildResults = store->buildPathsWithResults(pathsToBuild, bMode, evalStore); + throwBuildErrors(buildResults, *store); + for (auto & buildResult : buildResults) { for (auto & aux : backmap[buildResult.path]) { std::visit(overloaded { [&](const DerivedPath::Built & bfd) { @@ -814,6 +857,7 @@ std::vector RawInstallablesCommand::getFlakeRefsForCompletion() std::vector res; for (auto i : rawInstallables) res.push_back(parseFlakeRefWithFragment( + fetchSettings, expandTilde(i), absPath(getCommandBaseDir())).first); return res; @@ -836,6 +880,7 @@ std::vector InstallableCommand::getFlakeRefsForCompletion() { return { parseFlakeRefWithFragment( + fetchSettings, expandTilde(_installable), absPath(getCommandBaseDir())).first }; diff --git a/src/libcmd/local.mk b/src/libcmd/local.mk index abb7459a7..a270333f4 100644 --- a/src/libcmd/local.mk +++ b/src/libcmd/local.mk @@ -6,10 +6,10 @@ libcmd_DIR := $(d) libcmd_SOURCES := $(wildcard $(d)/*.cc) -libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers +libcmd_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) $(INCLUDE_libflake) $(INCLUDE_libmain) libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) $(THREAD_LDFLAGS) -libcmd_LIBS = libstore libutil libexpr libmain libfetchers +libcmd_LIBS = libutil libstore libfetchers libflake libexpr libmain $(eval $(call install-file-in, $(buildprefix)$(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644)) diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc index a4e3c5a77..6a0d05d9f 100644 --- a/src/libcmd/markdown.cc +++ b/src/libcmd/markdown.cc @@ -1,21 +1,23 @@ #include "markdown.hh" -#include "util.hh" +#include "environment-variables.hh" +#include "error.hh" #include "finally.hh" #include "terminal.hh" -#include #if HAVE_LOWDOWN -#include +# include +# include #endif namespace nix { -std::string renderMarkdownToTerminal(std::string_view markdown) -{ #if HAVE_LOWDOWN +static std::string doRenderMarkdownToTerminal(std::string_view markdown) +{ int windowWidth = getWindowSize().second; - struct lowdown_opts opts { + struct lowdown_opts opts + { .type = LOWDOWN_TERM, .maxdepth = 20, .cols = (size_t) std::max(windowWidth - 5, 60), @@ -50,10 +52,22 @@ std::string renderMarkdownToTerminal(std::string_view markdown) if (!rndr_res) throw Error("allocation error while rendering Markdown"); - return filterANSIEscapes(std::string(buf->data, buf->size), !shouldANSI()); -#else - return std::string(markdown); -#endif + return filterANSIEscapes(std::string(buf->data, buf->size), !isTTY()); } +std::string renderMarkdownToTerminal(std::string_view markdown) +{ + if (auto e = getEnv("_NIX_TEST_RAW_MARKDOWN"); e && *e == "1") + return std::string(markdown); + else + return doRenderMarkdownToTerminal(markdown); } + +#else +std::string renderMarkdownToTerminal(std::string_view markdown) +{ + return std::string(markdown); +} +#endif + +} // namespace nix diff --git a/src/libcmd/markdown.hh b/src/libcmd/markdown.hh index a04d32a4f..66db1736c 100644 --- a/src/libcmd/markdown.hh +++ b/src/libcmd/markdown.hh @@ -1,10 +1,17 @@ #pragma once ///@file -#include "types.hh" +#include namespace nix { +/** + * Render the given Markdown text to the terminal. + * + * If Nix is compiled without Markdown support, this function will return the input text as-is. + * + * The renderer takes into account the terminal width, and wraps text accordingly. + */ std::string renderMarkdownToTerminal(std::string_view markdown); } diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build new file mode 100644 index 000000000..c484cf998 --- /dev/null +++ b/src/libcmd/meson.build @@ -0,0 +1,130 @@ +project('nix-cmd', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ +] +deps_public_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), + dependency('nix-fetchers'), + dependency('nix-expr'), + dependency('nix-flake'), + dependency('nix-main'), +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') +deps_public += nlohmann_json + +lowdown = dependency('lowdown', version : '>= 0.9.0', required : get_option('markdown')) +deps_private += lowdown +configdata.set('HAVE_LOWDOWN', lowdown.found().to_int()) + +readline_flavor = get_option('readline-flavor') +if readline_flavor == 'editline' + editline = dependency('libeditline', 'editline', version : '>=1.14') + deps_private += editline +elif readline_flavor == 'readline' + readline = dependency('readline') + deps_private += readline + configdata.set( + 'USE_READLINE', + 1, + description: 'Use readline instead of editline', + ) +else + error('illegal editline flavor', readline_flavor) +endif + +config_h = configure_file( + configuration : configdata, + output : 'config-cmd.hh', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + # '-include', 'config-fetchers.h', + '-include', 'config-expr.hh', + '-include', 'config-main.hh', + '-include', 'config-cmd.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'built-path.cc', + 'command-installable-value.cc', + 'command.cc', + 'common-eval-args.cc', + 'editor-for.cc', + 'installable-attr-path.cc', + 'installable-derived-path.cc', + 'installable-flake.cc', + 'installable-value.cc', + 'installables.cc', + 'legacy.cc', + 'markdown.cc', + 'misc-store-flags.cc', + 'network-proxy.cc', + 'repl-interacter.cc', + 'repl.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'built-path.hh', + 'command-installable-value.hh', + 'command.hh', + 'common-eval-args.hh', + 'compatibility-settings.hh', + 'editor-for.hh', + 'installable-attr-path.hh', + 'installable-derived-path.hh', + 'installable-flake.hh', + 'installable-value.hh', + 'installables.hh', + 'legacy.hh', + 'markdown.hh', + 'misc-store-flags.hh', + 'network-proxy.hh', + 'repl-interacter.hh', + 'repl.hh', +) + +this_library = library( + 'nixcmd', + sources, + dependencies : deps_public + deps_private + deps_other, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libcmd/meson.options b/src/libcmd/meson.options new file mode 100644 index 000000000..79ae4fa55 --- /dev/null +++ b/src/libcmd/meson.options @@ -0,0 +1,15 @@ +# vim: filetype=meson + +option( + 'markdown', + type: 'feature', + description: 'Enable Markdown rendering in the Nix binary (requires lowdown)', +) + +option( + 'readline-flavor', + type : 'combo', + choices : ['editline', 'readline'], + value : 'editline', + description : 'Which library to use for nice line editing with the Nix language REPL', +) diff --git a/src/libcmd/misc-store-flags.cc b/src/libcmd/misc-store-flags.cc index e66d3f63b..06552c032 100644 --- a/src/libcmd/misc-store-flags.cc +++ b/src/libcmd/misc-store-flags.cc @@ -81,9 +81,15 @@ Args::Flag fileIngestionMethod(FileIngestionMethod * method) How to compute the hash of the input. One of: - - `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function. + - `nar` (the default): + Serialises the input as a + [Nix Archive](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) + and passes that to the hash function. - - `flat`: Assumes that the input is a single file and directly passes it to the hash function; + - `flat`: + Assumes that the input is a single file and + [directly passes](@docroot@/store/file-system-object/content-address.md#serial-flat) + it to the hash function. )", .labels = {"file-ingestion-method"}, .handler = {[method](std::string s) { @@ -101,15 +107,23 @@ Args::Flag contentAddressMethod(ContentAddressMethod * method) How to compute the content-address of the store object. One of: - - `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function. + - [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive) + (the default): + Serialises the input as a + [Nix Archive](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) + and passes that to the hash function. - - `flat`: Assumes that the input is a single file and directly passes it to the hash function; + - [`flat`](@docroot@/store/store-object/content-address.md#method-flat): + Assumes that the input is a single file and + [directly passes](@docroot@/store/file-system-object/content-address.md#serial-flat) + it to the hash function. - - `text`: Like `flat`, but used for - [derivations](@docroot@/glossary.md#store-derivation) serialized in store object and + - [`text`](@docroot@/store/store-object/content-address.md#method-text): + Like `flat`, but used for + [derivations](@docroot@/glossary.md#store-derivation) serialized in store object and [`builtins.toFile`](@docroot@/language/builtins.html#builtins-toFile). For advanced use-cases only; - for regular usage prefer `nar` and `flat. + for regular usage prefer `nar` and `flat`. )", .labels = {"content-address-method"}, .handler = {[method](std::string s) { diff --git a/src/libcmd/network-proxy.cc b/src/libcmd/network-proxy.cc new file mode 100644 index 000000000..738bf6147 --- /dev/null +++ b/src/libcmd/network-proxy.cc @@ -0,0 +1,50 @@ +#include "network-proxy.hh" + +#include + +#include "environment-variables.hh" + +namespace nix { + +static const StringSet lowercaseVariables{"http_proxy", "https_proxy", "ftp_proxy", "all_proxy", "no_proxy"}; + +static StringSet getAllVariables() +{ + StringSet variables = lowercaseVariables; + for (const auto & variable : lowercaseVariables) { + std::string upperVariable; + std::transform( + variable.begin(), variable.end(), upperVariable.begin(), [](unsigned char c) { return std::toupper(c); }); + variables.insert(std::move(upperVariable)); + } + return variables; +} + +const StringSet networkProxyVariables = getAllVariables(); + +static StringSet getExcludingNoProxyVariables() +{ + static const StringSet excludeVariables{"no_proxy", "NO_PROXY"}; + StringSet variables; + std::set_difference( + networkProxyVariables.begin(), + networkProxyVariables.end(), + excludeVariables.begin(), + excludeVariables.end(), + std::inserter(variables, variables.begin())); + return variables; +} + +static const StringSet excludingNoProxyVariables = getExcludingNoProxyVariables(); + +bool haveNetworkProxyConnection() +{ + for (const auto & variable : excludingNoProxyVariables) { + if (getEnv(variable).has_value()) { + return true; + } + } + return false; +} + +} diff --git a/src/libcmd/network-proxy.hh b/src/libcmd/network-proxy.hh new file mode 100644 index 000000000..0b6856acb --- /dev/null +++ b/src/libcmd/network-proxy.hh @@ -0,0 +1,22 @@ +#pragma once +///@file + +#include "types.hh" + +namespace nix { + +/** + * Environment variables relating to network proxying. These are used by + * a few misc commands. + * + * See the Environment section of https://curl.se/docs/manpage.html for details. + */ +extern const StringSet networkProxyVariables; + +/** + * Heuristically check if there is a proxy connection by checking for defined + * proxy variables. + */ +bool haveNetworkProxyConnection(); + +} diff --git a/src/libcmd/package.nix b/src/libcmd/package.nix new file mode 100644 index 000000000..cde494901 --- /dev/null +++ b/src/libcmd/package.nix @@ -0,0 +1,104 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-util +, nix-store +, nix-fetchers +, nix-expr +, nix-flake +, nix-main +, editline +, readline +, lowdown +, nlohmann_json + +# Configuration Options + +, version + +# Whether to enable Markdown rendering in the Nix binary. +, enableMarkdown ? !stdenv.hostPlatform.isWindows + +# Which interactive line editor library to use for Nix's repl. +# +# Currently supported choices are: +# +# - editline (default) +# - readline +, readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline" +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-cmd"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + ({ inherit editline readline; }.${readlineFlavor}) + ] ++ lib.optional enableMarkdown lowdown; + + propagatedBuildInputs = [ + nix-util + nix-store + nix-fetchers + nix-expr + nix-flake + nix-main + nlohmann_json + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + mesonFlags = [ + (lib.mesonEnable "markdown" enableMarkdown) + (lib.mesonOption "readline-flavor" readlineFlavor) + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libcmd/repl-interacter.cc b/src/libcmd/repl-interacter.cc new file mode 100644 index 000000000..187af46ea --- /dev/null +++ b/src/libcmd/repl-interacter.cc @@ -0,0 +1,206 @@ +#include + +#ifdef USE_READLINE +#include +#include +#else +// editline < 1.15.2 don't wrap their API for C++ usage +// (added in https://github.com/troglobit/editline/commit/91398ceb3427b730995357e9d120539fb9bb7461). +// This results in linker errors due to to name-mangling of editline C symbols. +// For compatibility with these versions, we wrap the API here +// (wrapping multiple times on newer versions is no problem). +extern "C" { +#include +} +#endif + +#include "signals.hh" +#include "finally.hh" +#include "repl-interacter.hh" +#include "file-system.hh" +#include "repl.hh" +#include "environment-variables.hh" + +namespace nix { + +namespace { +// Used to communicate to NixRepl::getLine whether a signal occurred in ::readline. +volatile sig_atomic_t g_signal_received = 0; + +void sigintHandler(int signo) +{ + g_signal_received = signo; +} +}; + +static detail::ReplCompleterMixin * curRepl; // ugly + +#ifndef USE_READLINE +static char * completionCallback(char * s, int * match) +{ + auto possible = curRepl->completePrefix(s); + if (possible.size() == 1) { + *match = 1; + auto * res = strdup(possible.begin()->c_str() + strlen(s)); + if (!res) + throw Error("allocation failure"); + return res; + } else if (possible.size() > 1) { + auto checkAllHaveSameAt = [&](size_t pos) { + auto & first = *possible.begin(); + for (auto & p : possible) { + if (p.size() <= pos || p[pos] != first[pos]) + return false; + } + return true; + }; + size_t start = strlen(s); + size_t len = 0; + while (checkAllHaveSameAt(start + len)) + ++len; + if (len > 0) { + *match = 1; + auto * res = strdup(std::string(*possible.begin(), start, len).c_str()); + if (!res) + throw Error("allocation failure"); + return res; + } + } + + *match = 0; + return nullptr; +} + +static int listPossibleCallback(char * s, char *** avp) +{ + auto possible = curRepl->completePrefix(s); + + if (possible.size() > (std::numeric_limits::max() / sizeof(char *))) + throw Error("too many completions"); + + int ac = 0; + char ** vp = nullptr; + + auto check = [&](auto * p) { + if (!p) { + if (vp) { + while (--ac >= 0) + free(vp[ac]); + free(vp); + } + throw Error("allocation failure"); + } + return p; + }; + + vp = check((char **) malloc(possible.size() * sizeof(char *))); + + for (auto & p : possible) + vp[ac++] = check(strdup(p.c_str())); + + *avp = vp; + + return ac; +} +#endif + +ReadlineLikeInteracter::Guard ReadlineLikeInteracter::init(detail::ReplCompleterMixin * repl) +{ + // Allow nix-repl specific settings in .inputrc + rl_readline_name = "nix-repl"; + try { + createDirs(dirOf(historyFile)); + } catch (SystemError & e) { + logWarning(e.info()); + } +#ifndef USE_READLINE + el_hist_size = 1000; +#endif + read_history(historyFile.c_str()); + auto oldRepl = curRepl; + curRepl = repl; + Guard restoreRepl([oldRepl] { curRepl = oldRepl; }); +#ifndef USE_READLINE + rl_set_complete_func(completionCallback); + rl_set_list_possib_func(listPossibleCallback); +#endif + return restoreRepl; +} + +static constexpr const char * promptForType(ReplPromptType promptType) +{ + switch (promptType) { + case ReplPromptType::ReplPrompt: + return "nix-repl> "; + case ReplPromptType::ContinuationPrompt: + return " "; + } + assert(false); +} + +bool ReadlineLikeInteracter::getLine(std::string & input, ReplPromptType promptType) +{ +#ifndef _WIN32 // TODO use more signals.hh for this + struct sigaction act, old; + sigset_t savedSignalMask, set; + + auto setupSignals = [&]() { + act.sa_handler = sigintHandler; + sigfillset(&act.sa_mask); + act.sa_flags = 0; + if (sigaction(SIGINT, &act, &old)) + throw SysError("installing handler for SIGINT"); + + sigemptyset(&set); + sigaddset(&set, SIGINT); + if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask)) + throw SysError("unblocking SIGINT"); + }; + auto restoreSignals = [&]() { + if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) + throw SysError("restoring signals"); + + if (sigaction(SIGINT, &old, 0)) + throw SysError("restoring handler for SIGINT"); + }; + + setupSignals(); +#endif + char * s = readline(promptForType(promptType)); + Finally doFree([&]() { free(s); }); +#ifndef _WIN32 // TODO use more signals.hh for this + restoreSignals(); +#endif + + if (g_signal_received) { + g_signal_received = 0; + input.clear(); + return true; + } + + // editline doesn't echo the input to the output when non-interactive, unlike readline + // this results in a different behavior when running tests. The echoing is + // quite useful for reading the test output, so we add it here. + if (auto e = getEnv("_NIX_TEST_REPL_ECHO"); s && e && *e == "1") + { +#ifndef USE_READLINE + // This is probably not right for multi-line input, but we don't use that + // in the characterisation tests, so it's fine. + std::cout << promptForType(promptType) << s << std::endl; +#endif + } + + if (!s) + return false; + input += s; + input += '\n'; + + return true; +} + +ReadlineLikeInteracter::~ReadlineLikeInteracter() +{ + write_history(historyFile.c_str()); +} + +}; diff --git a/src/libcmd/repl-interacter.hh b/src/libcmd/repl-interacter.hh new file mode 100644 index 000000000..cc70efd07 --- /dev/null +++ b/src/libcmd/repl-interacter.hh @@ -0,0 +1,48 @@ +#pragma once +/// @file + +#include "finally.hh" +#include "types.hh" +#include +#include + +namespace nix { + +namespace detail { +/** Provides the completion hooks for the repl, without exposing its complete + * internals. */ +struct ReplCompleterMixin { + virtual StringSet completePrefix(const std::string & prefix) = 0; +}; +}; + +enum class ReplPromptType { + ReplPrompt, + ContinuationPrompt, +}; + +class ReplInteracter +{ +public: + using Guard = Finally>; + + virtual Guard init(detail::ReplCompleterMixin * repl) = 0; + /** Returns a boolean of whether the interacter got EOF */ + virtual bool getLine(std::string & input, ReplPromptType promptType) = 0; + virtual ~ReplInteracter(){}; +}; + +class ReadlineLikeInteracter : public virtual ReplInteracter +{ + std::string historyFile; +public: + ReadlineLikeInteracter(std::string historyFile) + : historyFile(historyFile) + { + } + virtual Guard init(detail::ReplCompleterMixin * repl) override; + virtual bool getLine(std::string & input, ReplPromptType promptType) override; + virtual ~ReadlineLikeInteracter() override; +}; + +}; diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 8b83608fa..efc04b029 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -1,34 +1,17 @@ #include #include #include -#include - -#include - -#ifdef USE_READLINE -#include -#include -#else -// editline < 1.15.2 don't wrap their API for C++ usage -// (added in https://github.com/troglobit/editline/commit/91398ceb3427b730995357e9d120539fb9bb7461). -// This results in linker errors due to to name-mangling of editline C symbols. -// For compatibility with these versions, we wrap the API here -// (wrapping multiple times on newer versions is no problem). -extern "C" { -#include -} -#endif +#include "repl-interacter.hh" #include "repl.hh" #include "ansicolor.hh" -#include "signals.hh" #include "shared.hh" +#include "config-global.hh" #include "eval.hh" -#include "eval-cache.hh" -#include "eval-inline.hh" #include "eval-settings.hh" #include "attr-path.hh" +#include "signals.hh" #include "store-api.hh" #include "log-store.hh" #include "common-eval-args.hh" @@ -38,18 +21,21 @@ extern "C" { #include "flake/flake.hh" #include "flake/lockfile.hh" #include "users.hh" -#include "terminal.hh" #include "editor-for.hh" #include "finally.hh" #include "markdown.hh" #include "local-fs-store.hh" #include "print.hh" +#include "ref.hh" +#include "value.hh" #if HAVE_BOEHMGC #define GC_INCLUDE_NEW #include #endif +#include "strings.hh" + namespace nix { /** @@ -75,6 +61,7 @@ enum class ProcessLineResult { struct NixRepl : AbstractNixRepl + , detail::ReplCompleterMixin #if HAVE_BOEHMGC , gc #endif @@ -90,17 +77,16 @@ struct NixRepl int displ; StringSet varNames; - const Path historyFile; + std::unique_ptr interacter; - NixRepl(const SearchPath & searchPath, nix::ref store,ref state, + NixRepl(const LookupPath & lookupPath, nix::ref store,ref state, std::function getValues); - virtual ~NixRepl(); + virtual ~NixRepl() = default; ReplExitStatus mainLoop() override; void initEnv() override; - StringSet completePrefix(const std::string & prefix); - bool getLine(std::string & input, const std::string & prompt); + virtual StringSet completePrefix(const std::string & prefix) override; StorePath getDerivationPath(Value & v); ProcessLineResult processLine(std::string line); @@ -123,7 +109,8 @@ struct NixRepl .force = true, .derivationPaths = true, .maxDepth = maxDepth, - .prettyIndent = 2 + .prettyIndent = 2, + .errors = ErrorPrintBehavior::ThrowTopLevel, }); } }; @@ -137,111 +124,33 @@ std::string removeWhitespace(std::string s) } -NixRepl::NixRepl(const SearchPath & searchPath, nix::ref store, ref state, +NixRepl::NixRepl(const LookupPath & lookupPath, nix::ref store, ref state, std::function getValues) : AbstractNixRepl(state) , debugTraceIndex(0) , getValues(getValues) , staticEnv(new StaticEnv(nullptr, state->staticBaseEnv.get())) - , historyFile(getDataDir() + "/nix/repl-history") + , interacter(make_unique(getDataDir() + "/nix/repl-history")) { } - -NixRepl::~NixRepl() -{ - write_history(historyFile.c_str()); -} - void runNix(Path program, const Strings & args, const std::optional & input = {}) { auto subprocessEnv = getEnv(); subprocessEnv["NIX_CONFIG"] = globalConfig.toKeyValue(); - + //isInteractive avoid grabling interactive commands runProgram2(RunOptions { .program = settings.nixBinDir+ "/" + program, .args = args, .environment = subprocessEnv, .input = input, + .isInteractive = true, }); return; } -static NixRepl * curRepl; // ugly - -static char * completionCallback(char * s, int *match) { - auto possible = curRepl->completePrefix(s); - if (possible.size() == 1) { - *match = 1; - auto *res = strdup(possible.begin()->c_str() + strlen(s)); - if (!res) throw Error("allocation failure"); - return res; - } else if (possible.size() > 1) { - auto checkAllHaveSameAt = [&](size_t pos) { - auto &first = *possible.begin(); - for (auto &p : possible) { - if (p.size() <= pos || p[pos] != first[pos]) - return false; - } - return true; - }; - size_t start = strlen(s); - size_t len = 0; - while (checkAllHaveSameAt(start + len)) ++len; - if (len > 0) { - *match = 1; - auto *res = strdup(std::string(*possible.begin(), start, len).c_str()); - if (!res) throw Error("allocation failure"); - return res; - } - } - - *match = 0; - return nullptr; -} - -static int listPossibleCallback(char *s, char ***avp) { - auto possible = curRepl->completePrefix(s); - - if (possible.size() > (INT_MAX / sizeof(char*))) - throw Error("too many completions"); - - int ac = 0; - char **vp = nullptr; - - auto check = [&](auto *p) { - if (!p) { - if (vp) { - while (--ac >= 0) - free(vp[ac]); - free(vp); - } - throw Error("allocation failure"); - } - return p; - }; - - vp = check((char **)malloc(possible.size() * sizeof(char*))); - - for (auto & p : possible) - vp[ac++] = check(strdup(p.c_str())); - - *avp = vp; - - return ac; -} - -namespace { - // Used to communicate to NixRepl::getLine whether a signal occurred in ::readline. - volatile sig_atomic_t g_signal_received = 0; - - void sigintHandler(int signo) { - g_signal_received = signo; - } -} - static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positions, const DebugTrace & dt) { if (dt.isError) @@ -281,24 +190,7 @@ ReplExitStatus NixRepl::mainLoop() loadFiles(); - // Allow nix-repl specific settings in .inputrc - rl_readline_name = "nix-repl"; - try { - createDirs(dirOf(historyFile)); - } catch (SystemError & e) { - logWarning(e.info()); - } -#ifndef USE_READLINE - el_hist_size = 1000; -#endif - read_history(historyFile.c_str()); - auto oldRepl = curRepl; - curRepl = this; - Finally restoreRepl([&] { curRepl = oldRepl; }); -#ifndef USE_READLINE - rl_set_complete_func(completionCallback); - rl_set_list_possib_func(listPossibleCallback); -#endif + auto _guard = interacter->init(static_cast(this)); std::string input; @@ -307,7 +199,7 @@ ReplExitStatus NixRepl::mainLoop() logger->pause(); // When continuing input from previous lines, don't print a prompt, just align to the same // number of chars as the prompt. - if (!getLine(input, input.empty() ? "nix-repl> " : " ")) { + if (!interacter->getLine(input, input.empty() ? ReplPromptType::ReplPrompt : ReplPromptType::ContinuationPrompt)) { // Ctrl-D should exit the debugger. state->debugStop = false; logger->cout(""); @@ -325,7 +217,7 @@ ReplExitStatus NixRepl::mainLoop() case ProcessLineResult::PromptAgain: break; default: - abort(); + unreachable(); } } catch (ParseError & e) { if (e.msg().find("unexpected end of file") != std::string::npos) { @@ -336,13 +228,7 @@ ReplExitStatus NixRepl::mainLoop() printMsg(lvlError, e.msg()); } } catch (EvalError & e) { - // in debugger mode, an EvalError should trigger another repl session. - // when that session returns the exception will land here. No need to show it again; - // show the error for this repl session instead. - if (state->debugRepl && !state->debugTraces.empty()) - showDebugTrace(std::cout, state->positions, state->debugTraces.front()); - else - printMsg(lvlError, e.msg()); + printMsg(lvlError, e.msg()); } catch (Error & e) { printMsg(lvlError, e.msg()); } catch (Interrupted & e) { @@ -356,51 +242,6 @@ ReplExitStatus NixRepl::mainLoop() } } - -bool NixRepl::getLine(std::string & input, const std::string & prompt) -{ - struct sigaction act, old; - sigset_t savedSignalMask, set; - - auto setupSignals = [&]() { - act.sa_handler = sigintHandler; - sigfillset(&act.sa_mask); - act.sa_flags = 0; - if (sigaction(SIGINT, &act, &old)) - throw SysError("installing handler for SIGINT"); - - sigemptyset(&set); - sigaddset(&set, SIGINT); - if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask)) - throw SysError("unblocking SIGINT"); - }; - auto restoreSignals = [&]() { - if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) - throw SysError("restoring signals"); - - if (sigaction(SIGINT, &old, 0)) - throw SysError("restoring handler for SIGINT"); - }; - - setupSignals(); - char * s = readline(prompt.c_str()); - Finally doFree([&]() { free(s); }); - restoreSignals(); - - if (g_signal_received) { - g_signal_received = 0; - input.clear(); - return true; - } - - if (!s) - return false; - input += s; - input += '\n'; - return true; -} - - StringSet NixRepl::completePrefix(const std::string & prefix) { StringSet completions; @@ -421,11 +262,14 @@ StringSet NixRepl::completePrefix(const std::string & prefix) try { auto dir = std::string(cur, 0, slash); auto prefix2 = std::string(cur, slash + 1); - for (auto & entry : readDirectory(dir == "" ? "/" : dir)) { - if (entry.name[0] != '.' && hasPrefix(entry.name, prefix2)) - completions.insert(prev + dir + "/" + entry.name); + for (auto & entry : std::filesystem::directory_iterator{dir == "" ? "/" : dir}) { + checkInterrupt(); + auto name = entry.path().filename().string(); + if (name[0] != '.' && hasPrefix(name, prefix2)) + completions.insert(prev + entry.path().string()); } } catch (Error &) { + } catch (std::filesystem::filesystem_error &) { } } else if ((dot = cur.rfind('.')) == std::string::npos) { /* This is a variable name; look it up in the current scope. */ @@ -452,7 +296,7 @@ StringSet NixRepl::completePrefix(const std::string & prefix) e->eval(*state, *env, v); state->forceAttrs(v, noPos, "while evaluating an attrset for the purpose of completion (this error should not be displayed; file an issue?)"); - for (auto & i : *v.attrs) { + for (auto & i : *v.attrs()) { std::string_view name = state->symbols[i.name]; if (name.substr(0, cur2.size()) != cur2) continue; completions.insert(concatStrings(prev, expr, ".", name)); @@ -464,6 +308,8 @@ StringSet NixRepl::completePrefix(const std::string & prefix) // Quietly ignore evaluation errors. } catch (BadURL & e) { // Quietly ignore BadURL flake-related errors. + } catch (FileNotFound & e) { + // Quietly ignore non-existent file beeing `import`-ed. } } @@ -519,7 +365,7 @@ ProcessLineResult NixRepl::processLine(std::string line) if (line.empty()) return ProcessLineResult::PromptAgain; - _isInterrupted = false; + setInterrupted(false); std::string command, arg; @@ -548,6 +394,7 @@ ProcessLineResult NixRepl::processLine(std::string line) << " :l, :load Load Nix expression and add it to scope\n" << " :lf, :load-flake Load Nix flake and add it to scope\n" << " :p, :print Evaluate and print expression recursively\n" + << " Strings are printed directly, without escaping.\n" << " :q, :quit Exit nix-repl\n" << " :r, :reload Reload all files\n" << " :sh Build dependencies of derivation, then start\n" @@ -651,7 +498,7 @@ ProcessLineResult NixRepl::processLine(std::string line) auto path = state->coerceToPath(noPos, v, context, "while evaluating the filename to edit"); return {path, 0}; } else if (v.isLambda()) { - auto pos = state->positions[v.lambda.fun->pos]; + auto pos = state->positions[v.payload.lambda.fun->pos]; if (auto path = std::get_if(&pos.origin)) return {*path, pos.line}; else @@ -669,7 +516,7 @@ ProcessLineResult NixRepl::processLine(std::string line) // runProgram redirects stdout to a StringSink, // using runProgram2 to allow editors to display their UI - runProgram2(RunOptions { .program = editor, .searchPath = true, .args = args }); + runProgram2(RunOptions { .program = editor, .lookupPath = true, .args = args , .isInteractive = true }); // Reload right after exiting the editor state->resetFileCache(); @@ -755,7 +602,11 @@ ProcessLineResult NixRepl::processLine(std::string line) else if (command == ":p" || command == ":print") { Value v; evalString(arg, v); - printValue(std::cout, v); + if (v.type() == nString) { + std::cout << v.string_view(); + } else { + printValue(std::cout, v); + } std::cout << std::endl; } @@ -766,6 +617,35 @@ ProcessLineResult NixRepl::processLine(std::string line) else if (command == ":doc") { Value v; + + auto expr = parseString(arg); + std::string fallbackName; + PosIdx fallbackPos; + DocComment fallbackDoc; + if (auto select = dynamic_cast(expr)) { + Value vAttrs; + auto name = select->evalExceptFinalSelect(*state, *env, vAttrs); + fallbackName = state->symbols[name]; + + state->forceAttrs(vAttrs, noPos, "while evaluating an attribute set to look for documentation"); + auto attrs = vAttrs.attrs(); + assert(attrs); + auto attr = attrs->get(name); + if (!attr) { + // When missing, trigger the normal exception + // e.g. :doc builtins.foo + // behaves like + // nix-repl> builtins.foo + // error: attribute 'foo' missing + evalString(arg, v); + assert(false); + } + if (attr->pos) { + fallbackPos = attr->pos; + fallbackDoc = state->getDocCommentForPos(fallbackPos); + } + } + evalString(arg, v); if (auto doc = state->getDoc(v)) { std::string markdown; @@ -783,6 +663,19 @@ ProcessLineResult NixRepl::processLine(std::string line) markdown += stripIndentation(doc->doc); logger->cout(trim(renderMarkdownToTerminal(markdown))); + } else if (fallbackPos) { + std::stringstream ss; + ss << "Attribute `" << fallbackName << "`\n\n"; + ss << " … defined at " << state->positions[fallbackPos] << "\n\n"; + if (fallbackDoc) { + ss << fallbackDoc.getInnerText(state->positions); + } else { + ss << "No documentation found.\n\n"; + } + + auto markdown = ss.str(); + logger->cout(trim(renderMarkdownToTerminal(markdown))); + } else throw Error("value does not have documentation"); } @@ -840,14 +733,14 @@ void NixRepl::loadFlake(const std::string & flakeRefS) if (flakeRefS.empty()) throw Error("cannot use ':load-flake' without a path specified. (Use '.' for the current working directory.)"); - auto flakeRef = parseFlakeRef(flakeRefS, absPath("."), true); + auto flakeRef = parseFlakeRef(fetchSettings, flakeRefS, absPath("."), true); if (evalSettings.pureEval && !flakeRef.input.isLocked()) throw Error("cannot use ':load-flake' on locked flake reference '%s' (use --impure to override)", flakeRefS); Value v; flake::callFlake(*state, - flake::lockFlake(*state, flakeRef, + flake::lockFlake(flakeSettings, *state, flakeRef, flake::LockFlags { .updateLockFile = false, .useRegistries = !evalSettings.pureEval, @@ -899,17 +792,17 @@ void NixRepl::loadFiles() void NixRepl::addAttrsToScope(Value & attrs) { state->forceAttrs(attrs, [&]() { return attrs.determinePos(noPos); }, "while evaluating an attribute set to be merged in the global scope"); - if (displ + attrs.attrs->size() >= envSize) + if (displ + attrs.attrs()->size() >= envSize) throw Error("environment full; cannot add more variables"); - for (auto & i : *attrs.attrs) { + for (auto & i : *attrs.attrs()) { staticEnv->vars.emplace_back(i.name, displ); env->values[displ++] = i.value; varNames.emplace(state->symbols[i.name]); } staticEnv->sort(); staticEnv->deduplicate(); - notice("Added %1% variables.", attrs.attrs->size()); + notice("Added %1% variables.", attrs.attrs()->size()); } @@ -941,11 +834,11 @@ void NixRepl::evalString(std::string s, Value & v) std::unique_ptr AbstractNixRepl::create( - const SearchPath & searchPath, nix::ref store, ref state, + const LookupPath & lookupPath, nix::ref store, ref state, std::function getValues) { return std::make_unique( - searchPath, + lookupPath, openStore(), state, getValues @@ -961,9 +854,9 @@ ReplExitStatus AbstractNixRepl::runSimple( NixRepl::AnnotatedValues values; return values; }; - SearchPath searchPath = {}; + LookupPath lookupPath = {}; auto repl = std::make_unique( - searchPath, + lookupPath, openStore(), evalState, getValues diff --git a/src/libcmd/repl.hh b/src/libcmd/repl.hh index 21aa8bfc7..3fd4b2c39 100644 --- a/src/libcmd/repl.hh +++ b/src/libcmd/repl.hh @@ -3,11 +3,6 @@ #include "eval.hh" -#if HAVE_BOEHMGC -#define GC_INCLUDE_NEW -#include -#endif - namespace nix { struct AbstractNixRepl @@ -25,7 +20,7 @@ struct AbstractNixRepl typedef std::vector> AnnotatedValues; static std::unique_ptr create( - const SearchPath & searchPath, nix::ref store, ref state, + const LookupPath & lookupPath, nix::ref store, ref state, std::function getValues); static ReplExitStatus runSimple( diff --git a/src/libexpr-c/.version b/src/libexpr-c/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libexpr-c/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libexpr-c/build-utils-meson b/src/libexpr-c/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libexpr-c/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libexpr-c/local.mk b/src/libexpr-c/local.mk new file mode 100644 index 000000000..227a4095b --- /dev/null +++ b/src/libexpr-c/local.mk @@ -0,0 +1,25 @@ +libraries += libexprc + +libexprc_NAME = libnixexprc + +libexprc_DIR := $(d) + +libexprc_SOURCES := \ + $(wildcard $(d)/*.cc) \ + +# Not just for this library itself, but also for downstream libraries using this library + +INCLUDE_libexprc := -I $(d) +libexprc_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libutilc) \ + $(INCLUDE_libfetchers) \ + $(INCLUDE_libstore) $(INCLUDE_libstorec) \ + $(INCLUDE_libexpr) $(INCLUDE_libexprc) + +libexprc_LIBS = libutil libutilc libstore libstorec libfetchers libexpr + +libexprc_LDFLAGS += $(THREAD_LDFLAGS) + +$(eval $(call install-file-in, $(d)/nix-expr-c.pc, $(libdir)/pkgconfig, 0644)) + +libexprc_FORCE_INSTALL := 1 + diff --git a/src/libexpr-c/meson.build b/src/libexpr-c/meson.build new file mode 100644 index 000000000..6db5b83b8 --- /dev/null +++ b/src/libexpr-c/meson.build @@ -0,0 +1,93 @@ +project('nix-expr-c', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), + dependency('nix-expr'), +] +deps_public_maybe_subproject = [ + dependency('nix-util-c'), + dependency('nix-store-c'), +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +# TODO rename, because it will conflict with downstream projects +configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) + +config_h = configure_file( + configuration : configdata, + output : 'config-expr.h', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + + # From C++ libraries, only for internals + '-include', 'config-util.hh', + '-include', 'config-store.hh', + '-include', 'config-expr.hh', + + # From C libraries, for our public, installed headers too + '-include', 'config-util.h', + '-include', 'config-store.h', + '-include', 'config-expr.h', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'nix_api_expr.cc', + 'nix_api_external.cc', + 'nix_api_value.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'nix_api_expr.h', + 'nix_api_external.h', + 'nix_api_value.h', +) + +# TODO move this header to libexpr, maybe don't use it in tests? +headers += files('nix_api_expr_internal.h') + +subdir('build-utils-meson/export-all-symbols') + +this_library = library( + 'nixexprc', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + link_args: linker_export_flags, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libexpr-c/nix-expr-c.pc.in b/src/libexpr-c/nix-expr-c.pc.in new file mode 100644 index 000000000..06897064d --- /dev/null +++ b/src/libexpr-c/nix-expr-c.pc.in @@ -0,0 +1,10 @@ +prefix=@prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: Nix +Description: Nix Language Evaluator - C API +Version: @PACKAGE_VERSION@ +Requires: nix-store-c +Libs: -L${libdir} -lnixexprc +Cflags: -I${includedir}/nix diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc new file mode 100644 index 000000000..8f21d7022 --- /dev/null +++ b/src/libexpr-c/nix_api_expr.cc @@ -0,0 +1,213 @@ +#include +#include +#include + +#include "eval.hh" +#include "eval-gc.hh" +#include "globals.hh" +#include "eval-settings.hh" + +#include "nix_api_expr.h" +#include "nix_api_expr_internal.h" +#include "nix_api_store.h" +#include "nix_api_store_internal.h" +#include "nix_api_util.h" +#include "nix_api_util_internal.h" + +#if HAVE_BOEHMGC +# include +# define GC_INCLUDE_NEW 1 +# include "gc_cpp.h" +#endif + +nix_err nix_libexpr_init(nix_c_context * context) +{ + if (context) + context->last_err_code = NIX_OK; + { + auto ret = nix_libutil_init(context); + if (ret != NIX_OK) + return ret; + } + { + auto ret = nix_libstore_init(context); + if (ret != NIX_OK) + return ret; + } + try { + nix::initGC(); + } + NIXC_CATCH_ERRS +} + +nix_err nix_expr_eval_from_string( + nix_c_context * context, EvalState * state, const char * expr, const char * path, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + nix::Expr * parsedExpr = state->state.parseExprFromString(expr, state->state.rootPath(nix::CanonPath(path))); + state->state.eval(parsedExpr, value->value); + state->state.forceValue(value->value, nix::noPos); + } + NIXC_CATCH_ERRS +} + +nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, nix_value * arg, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + state->state.callFunction(fn->value, arg->value, value->value, nix::noPos); + state->state.forceValue(value->value, nix::noPos); + } + NIXC_CATCH_ERRS +} + +nix_err nix_value_call_multi(nix_c_context * context, EvalState * state, nix_value * fn, size_t nargs, nix_value ** args, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + state->state.callFunction(fn->value, nargs, (nix::Value * *)args, value->value, nix::noPos); + state->state.forceValue(value->value, nix::noPos); + } + NIXC_CATCH_ERRS +} + +nix_err nix_value_force(nix_c_context * context, EvalState * state, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + state->state.forceValue(value->value, nix::noPos); + } + NIXC_CATCH_ERRS +} + +nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + state->state.forceValueDeep(value->value); + } + NIXC_CATCH_ERRS +} + +EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c, Store * store) +{ + if (context) + context->last_err_code = NIX_OK; + try { + nix::Strings lookupPath; + if (lookupPath_c != nullptr) + for (size_t i = 0; lookupPath_c[i] != nullptr; i++) + lookupPath.push_back(lookupPath_c[i]); + + void * p = ::operator new( + sizeof(EvalState), + static_cast(alignof(EvalState))); + auto * p2 = static_cast(p); + new (p) EvalState { + .fetchSettings = nix::fetchers::Settings{}, + .settings = nix::EvalSettings{ + nix::settings.readOnlyMode, + }, + .state = nix::EvalState( + nix::LookupPath::parse(lookupPath), + store->ptr, + p2->fetchSettings, + p2->settings), + }; + loadConfFile(p2->settings); + return p2; + } + NIXC_CATCH_ERRS_NULL +} + +void nix_state_free(EvalState * state) +{ + delete state; +} + +#if HAVE_BOEHMGC +std::unordered_map< + const void *, + unsigned int, + std::hash, + std::equal_to, + traceable_allocator
; } + > $ nix-instantiate --eval --expr '{ a = ; }' + > { a = ; } + > ``` + > + > For human-readable output, `nix eval` (experimental) is more informative: + > + > ```console + > $ nix-instantiate --eval --expr 'a: a' + > + > $ nix eval --expr 'a: a' + > «lambda @ «string»:1:1» + > ``` + > + > For machine-readable output, the `--xml` option produces unambiguous + > output: + > + > ```console + > $ nix-instantiate --eval --xml --expr '{ foo = ; }' + > + > + > + > + > + > + > + > + > ``` - - `--json`\ - When used with `--eval`, print the resulting value as an JSON - representation of the abstract syntax tree rather than as a Nix expression. +- `--find-file` - - `--xml`\ - When used with `--eval`, print the resulting value as an XML - representation of the abstract syntax tree rather than as a Nix expression. - The schema is the same as that used by the [`toXML` - built-in](../language/builtins.md). + Look up the given files in Nix’s search path (as specified by the + `NIX_PATH` environment variable). If found, print the corresponding + absolute paths on standard output. For instance, if `NIX_PATH` is + `nixpkgs=/home/alice/nixpkgs`, then `nix-instantiate --find-file + nixpkgs/default.nix` will print `/home/alice/nixpkgs/default.nix`. - - `--read-write-mode`\ - When used with `--eval`, perform evaluation in read/write mode so - nix language features that require it will still work (at the cost - of needing to do instantiation of every evaluated derivation). If - this option is not enabled, there may be uninstantiated store paths - in the final output. +- `--strict` + + When used with `--eval`, recursively evaluate list elements and + attributes. Normally, such sub-expressions are left unevaluated + (since the Nix language is lazy). + + > **Warning** + > + > This option can cause non-termination, because lazy data + > structures can be infinitely large. + +- `--json` + + When used with `--eval`, print the resulting value as an JSON + representation of the abstract syntax tree rather than as a Nix expression. + +- `--xml` + + When used with `--eval`, print the resulting value as an XML + representation of the abstract syntax tree rather than as a Nix expression. + The schema is the same as that used by the [`toXML` + built-in](../language/builtins.md). + +- `--read-write-mode` + + When used with `--eval`, perform evaluation in read/write mode so + nix language features that require it will still work (at the cost + of needing to do instantiation of every evaluated derivation). If + this option is not enabled, there may be uninstantiated store paths + in the final output. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-prefetch-url.md b/doc/manual/src/command-ref/nix-prefetch-url.md index 45ef01e02..ffab94b8a 100644 --- a/doc/manual/src/command-ref/nix-prefetch-url.md +++ b/doc/manual/src/command-ref/nix-prefetch-url.md @@ -39,27 +39,32 @@ the path of the downloaded file in the Nix store is also printed. # Options - - `--type` *hashAlgo*\ - Use the specified cryptographic hash algorithm, - which can be one of `md5`, `sha1`, `sha256`, and `sha512`. - The default is `sha256`. +- `--type` *hashAlgo* - - `--print-path`\ - Print the store path of the downloaded file on standard output. + Use the specified cryptographic hash algorithm, + which can be one of `md5`, `sha1`, `sha256`, and `sha512`. + The default is `sha256`. - - `--unpack`\ - Unpack the archive (which must be a tarball or zip file) and add the - result to the Nix store. The resulting hash can be used with - functions such as Nixpkgs’s `fetchzip` or `fetchFromGitHub`. +- `--print-path` - - `--executable`\ - Set the executable bit on the downloaded file. + Print the store path of the downloaded file on standard output. - - `--name` *name*\ - Override the name of the file in the Nix store. By default, this is - `hash-basename`, where *basename* is the last component of *url*. - Overriding the name is necessary when *basename* contains characters - that are not allowed in Nix store paths. +- `--unpack` + + Unpack the archive (which must be a tarball or zip file) and add the + result to the Nix store. The resulting hash can be used with + functions such as Nixpkgs’s `fetchzip` or `fetchFromGitHub`. + +- `--executable` + + Set the executable bit on the downloaded file. + +- `--name` *name* + + Override the name of the file in the Nix store. By default, this is + `hash-basename`, where *basename* is the last component of *url*. + Overriding the name is necessary when *basename* contains characters + that are not allowed in Nix store paths. # Examples diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index 1eaf3c36a..69a711bd5 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -60,55 +60,63 @@ All options not listed here are passed to `nix-store --realise`, except for `--arg` and `--attr` / `-A` which are passed to `nix-instantiate`. - - `--command` *cmd*\ - In the environment of the derivation, run the shell command *cmd*. - This command is executed in an interactive shell. (Use `--run` to - use a non-interactive shell instead.) However, a call to `exit` is - implicitly added to the command, so the shell will exit after - running the command. To prevent this, add `return` at the end; - e.g. `--command "echo Hello; return"` will print `Hello` and then - drop you into the interactive shell. This can be useful for doing - any additional initialisation. +- `--command` *cmd* - - `--run` *cmd*\ - Like `--command`, but executes the command in a non-interactive - shell. This means (among other things) that if you hit Ctrl-C while - the command is running, the shell exits. + In the environment of the derivation, run the shell command *cmd*. + This command is executed in an interactive shell. (Use `--run` to + use a non-interactive shell instead.) However, a call to `exit` is + implicitly added to the command, so the shell will exit after + running the command. To prevent this, add `return` at the end; + e.g. `--command "echo Hello; return"` will print `Hello` and then + drop you into the interactive shell. This can be useful for doing + any additional initialisation. - - `--exclude` *regexp*\ - Do not build any dependencies whose store path matches the regular - expression *regexp*. This option may be specified multiple times. +- `--run` *cmd* - - `--pure`\ - If this flag is specified, the environment is almost entirely - cleared before the interactive shell is started, so you get an - environment that more closely corresponds to the “real†Nix build. A - few variables, in particular `HOME`, `USER` and `DISPLAY`, are - retained. + Like `--command`, but executes the command in a non-interactive + shell. This means (among other things) that if you hit Ctrl-C while + the command is running, the shell exits. - - `--packages` / `-p` *packages*…\ - Set up an environment in which the specified packages are present. - The command line arguments are interpreted as attribute names inside - the Nix Packages collection. Thus, `nix-shell --packages libjpeg openjdk` - will start a shell in which the packages denoted by the attribute - names `libjpeg` and `openjdk` are present. +- `--exclude` *regexp* - - `-i` *interpreter*\ - The chained script interpreter to be invoked by `nix-shell`. Only - applicable in `#!`-scripts (described below). + Do not build any dependencies whose store path matches the regular + expression *regexp*. This option may be specified multiple times. - - `--keep` *name*\ - When a `--pure` shell is started, keep the listed environment - variables. +- `--pure` + + If this flag is specified, the environment is almost entirely + cleared before the interactive shell is started, so you get an + environment that more closely corresponds to the “real†Nix build. A + few variables, in particular `HOME`, `USER` and `DISPLAY`, are + retained. + +- `--packages` / `-p` *packages*… + + Set up an environment in which the specified packages are present. + The command line arguments are interpreted as attribute names inside + the Nix Packages collection. Thus, `nix-shell --packages libjpeg openjdk` + will start a shell in which the packages denoted by the attribute + names `libjpeg` and `openjdk` are present. + +- `-i` *interpreter* + + The chained script interpreter to be invoked by `nix-shell`. Only + applicable in `#!`-scripts (described below). + +- `--keep` *name* + + When a `--pure` shell is started, keep the listed environment + variables. {{#include ./opt-common.md}} # Environment variables - - `NIX_BUILD_SHELL`\ - Shell used to start the interactive environment. Defaults to the - `bash` found in ``, falling back to the `bash` found in - `PATH` if not found. +- `NIX_BUILD_SHELL` + + Shell used to start the interactive environment. Defaults to the + `bash` found in ``, falling back to the `bash` found in + `PATH` if not found. {{#include ./env-common.md}} @@ -202,14 +210,14 @@ For example, here is a Python script that depends on Python and the ```python #! /usr/bin/env nix-shell -#! nix-shell -i python --packages python pythonPackages.prettytable +#! nix-shell -i python3 --packages python3 python3Packages.prettytable import prettytable # Print a simple table. t = prettytable.PrettyTable(["N", "N^2"]) for n in range(1, 10): t.add_row([n, n * n]) -print t +print(t) ``` Similarly, the following is a Perl script that specifies that it @@ -289,3 +297,8 @@ with import {}; runCommand "dummy" { buildInputs = [ python pythonPackages.prettytable ]; } "" ``` + +The script's file name is passed as the first argument to the interpreter specified by the `-i` flag. + +Aside from the very first line, which is a directive to the operating system, the additional `#! nix-shell` lines do not need to be at the beginning of the file. +This allows wrapping them in block comments for languages where `#` does not start a comment, such as ECMAScript, Erlang, PHP, or Ruby. diff --git a/doc/manual/src/command-ref/nix-store/add-fixed.md b/doc/manual/src/command-ref/nix-store/add-fixed.md index d25db091c..bebf15026 100644 --- a/doc/manual/src/command-ref/nix-store/add-fixed.md +++ b/doc/manual/src/command-ref/nix-store/add-fixed.md @@ -16,9 +16,10 @@ public url or broke since the download expression was written. This operation has the following options: - - `--recursive`\ - Use recursive instead of flat hashing mode, used when adding - directories to the store. +- `--recursive` + + Use recursive instead of flat hashing mode, used when adding + directories to the store. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/dump.md b/doc/manual/src/command-ref/nix-store/dump.md index c2f3c42ef..3de0e27b0 100644 --- a/doc/manual/src/command-ref/nix-store/dump.md +++ b/doc/manual/src/command-ref/nix-store/dump.md @@ -1,6 +1,6 @@ # Name -`nix-store --dump` - write a single path to a Nix Archive +`nix-store --dump` - write a single path to a [Nix Archive] ## Synopsis @@ -8,7 +8,7 @@ ## Description -The operation `--dump` produces a NAR (Nix ARchive) file containing the +The operation `--dump` produces a [Nix archive](@docroot@/glossary.md#gloss-nar) (NAR) file containing the contents of the file system tree rooted at *path*. The archive is written to standard output. @@ -30,8 +30,9 @@ NAR archives support filenames of unlimited length and 64-bit file sizes. They can contain regular files, directories, and symbolic links, but not other types of files (such as device nodes). -A Nix archive can be unpacked using `nix-store ---restore`. +A Nix archive can be unpacked using [`nix-store --restore`](@docroot@/command-ref/nix-store/restore.md). + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/export.md b/doc/manual/src/command-ref/nix-store/export.md index 1bc46f53b..ba772eb43 100644 --- a/doc/manual/src/command-ref/nix-store/export.md +++ b/doc/manual/src/command-ref/nix-store/export.md @@ -1,6 +1,6 @@ # Name -`nix-store --export` - export store paths to a Nix Archive +`nix-store --export` - export store paths to a [Nix Archive] ## Synopsis @@ -8,16 +8,22 @@ ## Description -The operation `--export` writes a serialisation of the specified store -paths to standard output in a format that can be imported into another -Nix store with `nix-store --import`. This is like `nix-store ---dump`, except that the NAR archive produced by that command doesn’t -contain the necessary meta-information to allow it to be imported into -another Nix store (namely, the set of references of the path). +The operation `--export` writes a serialisation of the given [store objects](@docroot@/glossary.md#gloss-store-object) to standard output in a format that can be imported into another [Nix store](@docroot@/store/index.md) with [`nix-store --import`](./import.md). -This command does not produce a *closure* of the specified paths, so if -a store path references other store paths that are missing in the target -Nix store, the import will fail. +> **Warning** +> +> This command *does not* produce a [closure](@docroot@/glossary.md#gloss-closure) of the specified store paths. +> Trying to import a store object that refers to store paths not available in the target Nix store will fail. +> +> Use [`nix-store --query`](@docroot@/command-ref/nix-store/query.md) to obtain the closure of a store path. + +This command is different from [`nix-store --dump`](./dump.md), which produces a [Nix archive](@docroot@/glossary.md#gloss-nar) that *does not* contain the set of [references](@docroot@/glossary.md#gloss-reference) of a given store path. + +> **Note** +> +> For efficient transfer of closures to remote machines over SSH, use [`nix-copy-closure`](@docroot@/command-ref/nix-copy-closure.md). + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive {{#include ./opt-common.md}} @@ -27,15 +33,21 @@ Nix store, the import will fail. # Examples -To copy a whole closure, do something -like: - -```console -$ nix-store --export $(nix-store --query --requisites paths) > out -``` - -To import the whole closure again, run: - -```console -$ nix-store --import < out -``` +> **Example** +> +> Deploy GNU Hello to an airgapped machine via USB stick. +> +> Write the closure to the block device on a machine with internet connection: +> +> ```shell-session +> [alice@itchy]$ storePath=$(nix-build '' -I nixpkgs=channel:nixpkgs-unstable -A hello --no-out-link) +> [alice@itchy]$ nix-store --export $(nix-store --query --requisites $storePath) | sudo dd of=/dev/usb +> ``` +> +> Read the closure from the block device on the machine without internet connection: +> +> ```shell-session +> [bob@scratchy]$ hello=$(sudo dd if=/dev/usb | nix-store --import | tail -1) +> [bob@scratchy]$ $hello/bin/hello +> Hello, world! +> ``` diff --git a/doc/manual/src/command-ref/nix-store/gc.md b/doc/manual/src/command-ref/nix-store/gc.md index 7be0d559a..f432e00eb 100644 --- a/doc/manual/src/command-ref/nix-store/gc.md +++ b/doc/manual/src/command-ref/nix-store/gc.md @@ -14,30 +14,34 @@ reachable via file system references from a set of “rootsâ€, are deleted. The following suboperations may be specified: - - `--print-roots`\ - This operation prints on standard output the set of roots used by - the garbage collector. +- `--print-roots` - - `--print-live`\ - This operation prints on standard output the set of “live†store - paths, which are all the store paths reachable from the roots. Live - paths should never be deleted, since that would break consistency — - it would become possible that applications are installed that - reference things that are no longer present in the store. + This operation prints on standard output the set of roots used by + the garbage collector. - - `--print-dead`\ - This operation prints out on standard output the set of “dead†store - paths, which is just the opposite of the set of live paths: any path - in the store that is not live (with respect to the roots) is dead. +- `--print-live` + + This operation prints on standard output the set of “live†store + paths, which are all the store paths reachable from the roots. Live + paths should never be deleted, since that would break consistency — + it would become possible that applications are installed that + reference things that are no longer present in the store. + +- `--print-dead` + + This operation prints out on standard output the set of “dead†store + paths, which is just the opposite of the set of live paths: any path + in the store that is not live (with respect to the roots) is dead. By default, all unreachable paths are deleted. The following options control what gets deleted and in what order: - - `--max-freed` *bytes*\ - Keep deleting paths until at least *bytes* bytes have been deleted, - then stop. The argument *bytes* can be followed by the - multiplicative suffix `K`, `M`, `G` or `T`, denoting KiB, MiB, GiB - or TiB units. +- `--max-freed` *bytes* + + Keep deleting paths until at least *bytes* bytes have been deleted, + then stop. The argument *bytes* can be followed by the + multiplicative suffix `K`, `M`, `G` or `T`, denoting KiB, MiB, GiB + or TiB units. The behaviour of the collector is also influenced by the `keep-outputs` and `keep-derivations` settings in the Nix diff --git a/doc/manual/src/command-ref/nix-store/import.md b/doc/manual/src/command-ref/nix-store/import.md index 2711316a7..3f6b3d076 100644 --- a/doc/manual/src/command-ref/nix-store/import.md +++ b/doc/manual/src/command-ref/nix-store/import.md @@ -1,6 +1,8 @@ # Name -`nix-store --import` - import Nix Archive into the store +`nix-store --import` - import [Nix Archive] into the store + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive # Synopsis @@ -8,14 +10,34 @@ # Description -The operation `--import` reads a serialisation of a set of store paths -produced by `nix-store --export` from standard input and adds those -store paths to the Nix store. Paths that already exist in the Nix store -are ignored. If a path refers to another path that doesn’t exist in the -Nix store, the import fails. +The operation `--import` reads a serialisation of a set of [store objects](@docroot@/glossary.md#gloss-store-object) produced by [`nix-store --export`](./export.md) from standard input, and adds those store objects to the specified [Nix store](@docroot@/store/index.md). +Paths that already exist in the target Nix store are ignored. +If a path [refers](@docroot@/glossary.md#gloss-reference) to another path that doesn’t exist in the target Nix store, the import fails. + +> **Note** +> +> For efficient transfer of closures to remote machines over SSH, use [`nix-copy-closure`](@docroot@/command-ref/nix-copy-closure.md). {{#include ./opt-common.md}} {{#include ../opt-common.md}} {{#include ../env-common.md}} + +# Examples + +> **Example** +> +> Given a closure of GNU Hello as a file: +> +> ```shell-session +> $ storePath="$(nix-build '' -I nixpkgs=channel:nixpkgs-unstable -A hello --no-out-link)" +> $ nix-store --export $(nix-store --query --requisites $storePath) > hello.closure +> ``` +> +> Import the closure into a [remote SSH store](@docroot@/store/types/ssh-store.md) using the [`--store`](@docroot@/command-ref/conf-file.md#conf-store) option: +> +> ```console +> $ nix-store --import --store ssh://alice@itchy.example.org < hello.closure +> ``` + diff --git a/doc/manual/src/command-ref/nix-store/optimise.md b/doc/manual/src/command-ref/nix-store/optimise.md index dc392aeb8..b257466b2 100644 --- a/doc/manual/src/command-ref/nix-store/optimise.md +++ b/doc/manual/src/command-ref/nix-store/optimise.md @@ -12,7 +12,7 @@ The operation `--optimise` reduces Nix store disk space usage by finding identical files in the store and hard-linking them to each other. It typically reduces the size of the store by something like 25-35%. Only regular files and symlinks are hard-linked in this manner. Files are -considered identical when they have the same NAR archive serialisation: +considered identical when they have the same [Nix Archive (NAR)][Nix Archive] serialisation: that is, regular files must have the same contents and permission (executable or non-executable), and symlinks must have the same contents. @@ -38,3 +38,4 @@ hashing files in `/nix/store/qhqx7l2f1kmwihc9bnxs7rc159hsxnf3-gcc-4.1.1' there are 114486 files with equal contents out of 215894 files in total ``` +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive diff --git a/doc/manual/src/command-ref/nix-store/query.md b/doc/manual/src/command-ref/nix-store/query.md index a158c76aa..b4efa734e 100644 --- a/doc/manual/src/command-ref/nix-store/query.md +++ b/doc/manual/src/command-ref/nix-store/query.md @@ -24,122 +24,138 @@ symlink. # Common query options - - `--use-output`; `-u`\ - For each argument to the query that is a [store derivation], apply the - query to the output path of the derivation instead. +- `--use-output` / `-u` - - `--force-realise`; `-f`\ - Realise each argument to the query first (see [`nix-store --realise`](./realise.md)). + For each argument to the query that is a [store derivation], apply the + query to the output path of the derivation instead. + +- `--force-realise` / `-f` + + Realise each argument to the query first (see [`nix-store --realise`](./realise.md)). [store derivation]: @docroot@/glossary.md#gloss-store-derivation # Queries - - `--outputs`\ - Prints out the [output paths] of the store - derivations *paths*. These are the paths that will be produced when - the derivation is built. +- `--outputs` - [output paths]: ../../glossary.md#gloss-output-path + Prints out the [output paths] of the store + derivations *paths*. These are the paths that will be produced when + the derivation is built. - - `--requisites`; `-R`\ - Prints out the [closure] of the store path *paths*. + [output paths]: @docroot@/glossary.md#gloss-output-path - [closure]: ../../glossary.md#gloss-closure +- `--requisites` / `-R` - This query has one option: + Prints out the [closure] of the store path *paths*. - - `--include-outputs` - Also include the existing output paths of [store derivation]s, - and their closures. + [closure]: @docroot@/glossary.md#gloss-closure - This query can be used to implement various kinds of deployment. A - *source deployment* is obtained by distributing the closure of a - store derivation. A *binary deployment* is obtained by distributing - the closure of an output path. A *cache deployment* (combined - source/binary deployment, including binaries of build-time-only - dependencies) is obtained by distributing the closure of a store - derivation and specifying the option `--include-outputs`. + This query has one option: - - `--references`\ - Prints the set of [references] of the store paths - *paths*, that is, their immediate dependencies. (For *all* - dependencies, use `--requisites`.) + - `--include-outputs` + Also include the existing output paths of [store derivation]s, + and their closures. - [references]: ../../glossary.md#gloss-reference + This query can be used to implement various kinds of deployment. A + *source deployment* is obtained by distributing the closure of a + store derivation. A *binary deployment* is obtained by distributing + the closure of an output path. A *cache deployment* (combined + source/binary deployment, including binaries of build-time-only + dependencies) is obtained by distributing the closure of a store + derivation and specifying the option `--include-outputs`. - - `--referrers`\ - Prints the set of *referrers* of the store paths *paths*, that is, - the store paths currently existing in the Nix store that refer to - one of *paths*. Note that contrary to the references, the set of - referrers is not constant; it can change as store paths are added or - removed. +- `--references` - - `--referrers-closure`\ - Prints the closure of the set of store paths *paths* under the - referrers relation; that is, all store paths that directly or - indirectly refer to one of *paths*. These are all the path currently - in the Nix store that are dependent on *paths*. + Prints the set of [references] of the store paths + *paths*, that is, their immediate dependencies. (For *all* + dependencies, use `--requisites`.) - - `--deriver`; `-d`\ - Prints the [deriver] that was used to build the store paths *paths*. If - the path has no deriver (e.g., if it is a source file), or if the - deriver is not known (e.g., in the case of a binary-only - deployment), the string `unknown-deriver` is printed. - The returned deriver is not guaranteed to exist in the local store, for - example when *paths* were substituted from a binary cache. - Use `--valid-derivers` instead to obtain valid paths only. + [references]: @docroot@/glossary.md#gloss-reference - [deriver]: ../../glossary.md#gloss-deriver +- `--referrers` - - `--valid-derivers`\ - Prints a set of derivation files (`.drv`) which are supposed produce - said paths when realized. Might print nothing, for example for source paths - or paths subsituted from a binary cache. + Prints the set of *referrers* of the store paths *paths*, that is, + the store paths currently existing in the Nix store that refer to + one of *paths*. Note that contrary to the references, the set of + referrers is not constant; it can change as store paths are added or + removed. - - `--graph`\ - Prints the references graph of the store paths *paths* in the format - of the `dot` tool of AT\&T's [Graphviz - package](http://www.graphviz.org/). This can be used to visualise - dependency graphs. To obtain a build-time dependency graph, apply - this to a store derivation. To obtain a runtime dependency graph, - apply it to an output path. +- `--referrers-closure` - - `--tree`\ - Prints the references graph of the store paths *paths* as a nested - ASCII tree. References are ordered by descending closure size; this - tends to flatten the tree, making it more readable. The query only - recurses into a store path when it is first encountered; this - prevents a blowup of the tree representation of the graph. + Prints the closure of the set of store paths *paths* under the + referrers relation; that is, all store paths that directly or + indirectly refer to one of *paths*. These are all the path currently + in the Nix store that are dependent on *paths*. - - `--graphml`\ - Prints the references graph of the store paths *paths* in the - [GraphML](http://graphml.graphdrawing.org/) file format. This can be - used to visualise dependency graphs. To obtain a build-time - dependency graph, apply this to a [store derivation]. To obtain a - runtime dependency graph, apply it to an output path. +- `--deriver` / `-d` - - `--binding` *name*; `-b` *name*\ - Prints the value of the attribute *name* (i.e., environment - variable) of the [store derivation]s *paths*. It is an error for a - derivation to not have the specified attribute. + Prints the [deriver] that was used to build the store paths *paths*. If + the path has no deriver (e.g., if it is a source file), or if the + deriver is not known (e.g., in the case of a binary-only + deployment), the string `unknown-deriver` is printed. + The returned deriver is not guaranteed to exist in the local store, for + example when *paths* were substituted from a binary cache. + Use `--valid-derivers` instead to obtain valid paths only. - - `--hash`\ - Prints the SHA-256 hash of the contents of the store paths *paths* - (that is, the hash of the output of `nix-store --dump` on the given - paths). Since the hash is stored in the Nix database, this is a fast - operation. + [deriver]: @docroot@/glossary.md#gloss-deriver - - `--size`\ - Prints the size in bytes of the contents of the store paths *paths* - — to be precise, the size of the output of `nix-store --dump` on - the given paths. Note that the actual disk space required by the - store paths may be higher, especially on filesystems with large - cluster sizes. +- `--valid-derivers` - - `--roots`\ - Prints the garbage collector roots that point, directly or - indirectly, at the store paths *paths*. + Prints a set of derivation files (`.drv`) which are supposed produce + said paths when realized. Might print nothing, for example for source paths + or paths subsituted from a binary cache. + +- `--graph` + + Prints the references graph of the store paths *paths* in the format + of the `dot` tool of AT\&T's [Graphviz + package](http://www.graphviz.org/). This can be used to visualise + dependency graphs. To obtain a build-time dependency graph, apply + this to a store derivation. To obtain a runtime dependency graph, + apply it to an output path. + +- `--tree` + + Prints the references graph of the store paths *paths* as a nested + ASCII tree. References are ordered by descending closure size; this + tends to flatten the tree, making it more readable. The query only + recurses into a store path when it is first encountered; this + prevents a blowup of the tree representation of the graph. + +- `--graphml` + + Prints the references graph of the store paths *paths* in the + [GraphML](http://graphml.graphdrawing.org/) file format. This can be + used to visualise dependency graphs. To obtain a build-time + dependency graph, apply this to a [store derivation]. To obtain a + runtime dependency graph, apply it to an output path. + +- `--binding` *name* / `-b` *name* + + Prints the value of the attribute *name* (i.e., environment + variable) of the [store derivation]s *paths*. It is an error for a + derivation to not have the specified attribute. + +- `--hash` + + Prints the SHA-256 hash of the contents of the store paths *paths* + (that is, the hash of the output of `nix-store --dump` on the given + paths). Since the hash is stored in the Nix database, this is a fast + operation. + +- `--size` + + Prints the size in bytes of the contents of the store paths *paths* + — to be precise, the size of the output of `nix-store --dump` on + the given paths. Note that the actual disk space required by the + store paths may be higher, especially on filesystems with large + cluster sizes. + +- `--roots` + + Prints the garbage collector roots that point, directly or + indirectly, at the store paths *paths*. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index 5428d57fa..a899758df 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -25,14 +25,14 @@ Each of *paths* is processed as follows: If no substitutes are available and no store derivation is given, realisation fails. -[store paths]: @docroot@/glossary.md#gloss-store-path +[store paths]: @docroot@/store/store-path.md [valid]: @docroot@/glossary.md#gloss-validity [store derivation]: @docroot@/glossary.md#gloss-store-derivation [output paths]: @docroot@/glossary.md#gloss-output-path -[store objects]: @docroot@/glossary.md#gloss-store-object +[store objects]: @docroot@/store/store-object.md [closure]: @docroot@/glossary.md#gloss-closure [substituters]: @docroot@/command-ref/conf-file.md#conf-substituters -[content-addressed derivations]: @docroot@/contributing/experimental-features.md#xp-feature-ca-derivations +[content-addressed derivations]: @docroot@/development/experimental-features.md#xp-feature-ca-derivations [Nix database]: @docroot@/glossary.md#gloss-nix-database The resulting paths are printed on standard output. @@ -42,23 +42,26 @@ For non-derivation arguments, the argument itself is printed. # Options - - `--dry-run`\ - Print on standard error a description of what packages would be - built or downloaded, without actually performing the operation. +- `--dry-run` - - `--ignore-unknown`\ - If a non-derivation path does not have a substitute, then silently - ignore it. + Print on standard error a description of what packages would be + built or downloaded, without actually performing the operation. - - `--check`\ - This option allows you to check whether a derivation is - deterministic. It rebuilds the specified derivation and checks - whether the result is bitwise-identical with the existing outputs, - printing an error if that’s not the case. The outputs of the - specified derivation must already exist. When used with `-K`, if an - output path is not identical to the corresponding output from the - previous build, the new output path is left in - `/nix/store/name.check.` +- `--ignore-unknown` + + If a non-derivation path does not have a substitute, then silently + ignore it. + +- `--check` + + This option allows you to check whether a derivation is + deterministic. It rebuilds the specified derivation and checks + whether the result is bitwise-identical with the existing outputs, + printing an error if that’s not the case. The outputs of the + specified derivation must already exist. When used with `-K`, if an + output path is not identical to the corresponding output from the + previous build, the new output path is left in + `/nix/store/name.check.` {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/restore.md b/doc/manual/src/command-ref/nix-store/restore.md index fcba43df4..2d0aa3127 100644 --- a/doc/manual/src/command-ref/nix-store/restore.md +++ b/doc/manual/src/command-ref/nix-store/restore.md @@ -8,9 +8,11 @@ ## Description -The operation `--restore` unpacks a NAR archive to *path*, which must +The operation `--restore` unpacks a [Nix Archive (NAR)][Nix Archive] to *path*, which must not already exist. The archive is read from standard input. +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive + {{#include ./opt-common.md}} {{#include ../opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/serve.md b/doc/manual/src/command-ref/nix-store/serve.md index 0f90f65ae..9a4cf5216 100644 --- a/doc/manual/src/command-ref/nix-store/serve.md +++ b/doc/manual/src/command-ref/nix-store/serve.md @@ -14,10 +14,11 @@ access to a restricted ssh user. The following flags are available: - - `--write`\ - Allow the connected client to request the realization of - derivations. In effect, this can be used to make the host act as a - remote builder. +- `--write` + + Allow the connected client to request the realization of + derivations. In effect, this can be used to make the host act as a + remote builder. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/verify.md b/doc/manual/src/command-ref/nix-store/verify.md index 2695b3361..40c9180db 100644 --- a/doc/manual/src/command-ref/nix-store/verify.md +++ b/doc/manual/src/command-ref/nix-store/verify.md @@ -16,18 +16,20 @@ being modified by non-Nix tools, or of bugs in Nix itself. This operation has the following options: - - `--check-contents`\ - Checks that the contents of every valid store path has not been - altered by computing a SHA-256 hash of the contents and comparing it - with the hash stored in the Nix database at build time. Paths that - have been modified are printed out. For large stores, - `--check-contents` is obviously quite slow. +- `--check-contents` - - `--repair`\ - If any valid path is missing from the store, or (if - `--check-contents` is given) the contents of a valid path has been - modified, then try to repair the path by redownloading it. See - `nix-store --repair-path` for details. + Checks that the contents of every valid store path has not been + altered by computing a SHA-256 hash of the contents and comparing it + with the hash stored in the Nix database at build time. Paths that + have been modified are printed out. For large stores, + `--check-contents` is obviously quite slow. + +- `--repair` + + If any valid path is missing from the store, or (if + `--check-contents` is given) the contents of a valid path has been + modified, then try to repair the path by redownloading it. See + `nix-store --repair-path` for details. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index 114b292f9..69a700207 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -37,7 +37,7 @@ Most Nix commands accept the following command-line options: Print even more informational messages. - `4` “Debug†- + Print debug information. - `5` “Vomit†@@ -143,7 +143,7 @@ Most Nix commands accept the following command-line options: This option is accepted by `nix-env`, `nix-instantiate`, `nix-shell` and `nix-build`. When evaluating Nix expressions, the expression evaluator will automatically try to call functions that it encounters. - It can automatically call functions for which every argument has a [default value](@docroot@/language/constructs.md#functions) (e.g., `{ argName ? defaultValue }: ...`). + It can automatically call functions for which every argument has a [default value](@docroot@/language/syntax.md#functions) (e.g., `{ argName ? defaultValue }: ...`). With `--arg`, you can also call functions that have arguments without a default value (or override a default value). That is, if the evaluator encounters a function with an argument named *name*, it will call it with value *value*. @@ -187,11 +187,12 @@ Most Nix commands accept the following command-line options: For `nix-shell`, this option is commonly used to give you a shell in which you can build the packages returned by the expression. If you want to get a shell which contain the *built* packages ready for use, give your expression to the `nix-shell --packages ` convenience flag instead. -- [`-I`](#opt-I) *path* +- [`-I` / `--include`](#opt-I) *path* - Add an entry to the [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path). + Add an entry to the list of search paths used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). This option may be given multiple times. - Paths added through `-I` take precedence over [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH). + + Paths added through `-I` take precedence over the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path) and the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH). - [`--option`](#opt-option) *name* *value* diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/development/building.md similarity index 67% rename from doc/manual/src/contributing/hacking.md rename to doc/manual/src/development/building.md index 916ec3077..5a5fb3368 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/development/building.md @@ -1,24 +1,67 @@ -# Hacking +# Building Nix -This section provides some notes on how to hack on Nix. To get the -latest version of Nix from GitHub: +This section provides some notes on how to start hacking on Nix. +To get the latest version of Nix from GitHub: ```console $ git clone https://github.com/NixOS/nix.git $ cd nix ``` -The following instructions assume you already have some version of Nix installed locally, so that you can use it to set up the development environment. If you don't have it installed, follow the [installation instructions]. +> **Note** +> +> The following instructions assume you already have some version of Nix installed locally, so that you can use it to set up the development environment. +> If you don't have it installed, follow the [installation instructions](../installation/index.md). -[installation instructions]: ../installation/index.md + +To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: + +```console +$ nix-shell +``` + +To get a shell with one of the other [supported compilation environments](#compilation-environments): + +```console +$ nix-shell --attr devShells.x86_64-linux.native-clangStdenvPackages +``` + +> **Note** +> +> You can use `native-ccacheStdenvPackages` to drastically improve rebuild time. +> By default, [ccache](https://ccache.dev) keeps artifacts in `~/.cache/ccache/`. + +To build Nix itself in this shell: + +```console +[nix-shell]$ autoreconfPhase +[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out +[nix-shell]$ make -j $NIX_BUILD_CORES +``` + +To install it in `$(pwd)/outputs` and test it: + +```console +[nix-shell]$ make install +[nix-shell]$ make installcheck -j $NIX_BUILD_CORES +[nix-shell]$ ./outputs/out/bin/nix --version +nix (Nix) 2.12 +``` + +To build a release version of Nix for the current operating system and CPU architecture: + +```console +$ nix-build +``` + +You can also build Nix for one of the [supported platforms](#platforms). ## Building Nix with flakes This section assumes you are using Nix with the [`flakes`] and [`nix-command`] experimental features enabled. -See the [Building Nix](#building-nix) section for equivalent instructions using stable Nix interfaces. -[`flakes`]: @docroot@/contributing/experimental-features.md#xp-feature-flakes -[`nix-command`]: @docroot@/contributing/experimental-features.md#xp-nix-command +[`flakes`]: @docroot@/development/experimental-features.md#xp-feature-flakes +[`nix-command`]: @docroot@/development/experimental-features.md#xp-nix-command To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: @@ -67,50 +110,6 @@ $ nix build You can also build Nix for one of the [supported platforms](#platforms). -## Building Nix - -To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: - -```console -$ nix-shell -``` - -To get a shell with one of the other [supported compilation environments](#compilation-environments): - -```console -$ nix-shell --attr devShells.x86_64-linux.native-clangStdenvPackages -``` - -> **Note** -> -> You can use `native-ccacheStdenvPackages` to drastically improve rebuild time. -> By default, [ccache](https://ccache.dev) keeps artifacts in `~/.cache/ccache/`. - -To build Nix itself in this shell: - -```console -[nix-shell]$ autoreconfPhase -[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out -[nix-shell]$ make -j $NIX_BUILD_CORES -``` - -To install it in `$(pwd)/outputs` and test it: - -```console -[nix-shell]$ make install -[nix-shell]$ make installcheck -j $NIX_BUILD_CORES -[nix-shell]$ ./outputs/out/bin/nix --version -nix (Nix) 2.12 -``` - -To build a release version of Nix for the current operating system and CPU architecture: - -```console -$ nix-build -``` - -You can also build Nix for one of the [supported platforms](#platforms). - ## Makefile variables You may need `profiledir=$out/etc/profile.d` and `sysconfdir=$out/etc` to run `make install`. @@ -122,7 +121,6 @@ Run `make` with [`-e` / `--environment-overrides`](https://www.gnu.org/software/ The docs can take a while to build, so you may want to disable this for local development. - `ENABLE_FUNCTIONAL_TESTS=yes` to enable building the functional tests. -- `ENABLE_UNIT_TESTS=yes` to enable building the unit tests. - `OPTIMIZE=1` to enable optimizations. - `libraries=libutil programs=` to only build a specific library. @@ -144,6 +142,7 @@ Nix can be built for various platforms, as specified in [`flake.nix`]: - `aarch64-darwin` - `armv6l-linux` - `armv7l-linux` +- `riscv64-linux` In order to build Nix for a different platform than the one you're currently on, you need a way for your current Nix installation to build code for that @@ -166,7 +165,10 @@ or for Nix with the [`flakes`] and [`nix-command`] experimental features enabled $ nix build .#packages.aarch64-linux.default ``` -Cross-compiled builds are available for ARMv6 (`armv6l-linux`) and ARMv7 (`armv7l-linux`). +Cross-compiled builds are available for: +- `armv6l-linux` +- `armv7l-linux` +- `riscv64-linux` Add more [system types](#system-type) to `crossSystems` in `flake.nix` to bootstrap Nix on unsupported platforms. ### Building for multiple platforms at once @@ -196,7 +198,7 @@ In order to facilitate this, Nix has some support for being built out of tree ## System type -Nix uses a string with he following format to identify the *system type* or *platform* it runs on: +Nix uses a string with the following format to identify the *system type* or *platform* it runs on: ``` -[-] @@ -258,10 +260,10 @@ See [supported compilation environments](#compilation-environments) and instruct To use the LSP with your editor, you first need to [set up `clangd`](https://clangd.llvm.org/installation#project-setup) by running: ```console -make clean && bear -- make -j$NIX_BUILD_CORES default check install +make compile_commands.json ``` -Configure your editor to use the `clangd` from the shell, either by running it inside the development shell, or by using [nix-direnv](https://github.com/nix-community/nix-direnv) and [the appropriate editor plugin](https://github.com/direnv/direnv/wiki#editor-integration). +Configure your editor to use the `clangd` from the `.#native-clangStdenvPackages` shell. You can do that either by running it inside the development shell, or by using [nix-direnv](https://github.com/nix-community/nix-direnv) and [the appropriate editor plugin](https://github.com/direnv/direnv/wiki#editor-integration). > **Note** > @@ -269,80 +271,25 @@ Configure your editor to use the `clangd` from the shell, either by running it i > Some other editors (e.g. Emacs, Vim) need a plugin to support LSP servers in general (e.g. [lsp-mode](https://github.com/emacs-lsp/lsp-mode) for Emacs and [vim-lsp](https://github.com/prabirshrestha/vim-lsp) for vim). > Editor-specific setup is typically opinionated, so we will not cover it here in more detail. -## Add a release note +## Formatting and pre-commit hooks -`doc/manual/rl-next` contains release notes entries for all unreleased changes. +You may run the formatters as a one-off using: -User-visible changes should come with a release note. - -### Add an entry - -Here's what a complete entry looks like. The file name is not incorporated in the document. - -``` ---- -synopsis: Basically a title -issues: 1234 -prs: 1238 ---- - -Here's one or more paragraphs that describe the change. - -- It's markdown -- Add references to the manual using @docroot@ +```console +make format ``` -Significant changes should add the following header, which moves them to the top. +If you'd like to run the formatters before every commit, install the hooks: ``` -significance: significant +pre-commit-hooks-install ``` - -See also the [format documentation](https://github.com/haskell/cabal/blob/master/CONTRIBUTING.md#changelog). +This installs [pre-commit](https://pre-commit.com) using [cachix/git-hooks.nix](https://github.com/cachix/git-hooks.nix). -### Build process +When making a commit, pay attention to the console output. +If it fails, run `git add --patch` to approve the suggestions _and commit again_. -Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`. - -## Branches - -- [`master`](https://github.com/NixOS/nix/commits/master) - - The main development branch. All changes are approved and merged here. - When developing a change, create a branch based on the latest `master`. - - Maintainers try to [keep it in a release-worthy state](#reverting). - -- [`maintenance-*.*`](https://github.com/NixOS/nix/branches/all?query=maintenance) - - These branches are the subject of backports only, and are - also [kept](#reverting) in a release-worthy state. - - See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) - -- [`latest-release`](https://github.com/NixOS/nix/tree/latest-release) - - The latest patch release of the latest minor version. - - See [`maintainers/release-process.md`](https://github.com/NixOS/nix/blob/master/maintainers/release-process.md) - -- [`backport-*-to-*`](https://github.com/NixOS/nix/branches/all?query=backport) - - Generally branches created by the backport action. - - See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) - -- [_other_](https://github.com/NixOS/nix/branches/all) - - Branches that do not conform to the above patterns should be feature branches. - -## Reverting - -If a change turns out to be merged by mistake, or contain a regression, it may be reverted. -A revert is not a rejection of the contribution, but merely part of an effective development process. -It makes sure that development keeps running smoothly, with minimal uncertainty, and less overhead. -If maintainers have to worry too much about avoiding reverts, they would not be able to merge as much. -By embracing reverts as a good part of the development process, everyone wins. - -However, taking a step back may be frustrating, so maintainers will be extra supportive on the next try. +To refresh pre-commit hook's config file, do the following: +1. Exit the development shell and start it again by running `nix develop`. +2. If you also use the pre-commit hook, also run `pre-commit-hooks-install` again. diff --git a/doc/manual/src/contributing/cli-guideline.md b/doc/manual/src/development/cli-guideline.md similarity index 88% rename from doc/manual/src/contributing/cli-guideline.md rename to doc/manual/src/development/cli-guideline.md index e90d6de8d..23df844ec 100644 --- a/doc/manual/src/contributing/cli-guideline.md +++ b/doc/manual/src/development/cli-guideline.md @@ -389,88 +389,6 @@ colors, no emojis and using ASCII instead of Unicode symbols). The same should happen when TTY is not detected on STDERR. We should not display progress / status section, but only print warnings and errors. -## Returning future proof JSON - -The schema of JSON output should allow for backwards compatible extension. This section explains how to achieve this. - -Two definitions are helpful here, because while JSON only defines one "key-value" -object type, we use it to cover two use cases: - - - **dictionary**: a map from names to value that all have the same type. In - C++ this would be a `std::map` with string keys. - - **record**: a fixed set of attributes each with their own type. In C++, this - would be represented by a `struct`. - -It is best not to mix these use cases, as that may lead to incompatibilities when the schema changes. For example, adding a record field to a dictionary breaks consumers that assume all JSON object fields to have the same meaning and type. - -This leads to the following guidelines: - - - The top-level (root) value must be a record. - - Otherwise, one can not change the structure of a command's output. - - - The value of a dictionary item must be a record. - - Otherwise, the item type can not be extended. - - - List items should be records. - - Otherwise, one can not change the structure of the list items. - - If the order of the items does not matter, and each item has a unique key that is a string, consider representing the list as a dictionary instead. If the order of the items needs to be preserved, return a list of records. - - - Streaming JSON should return records. - - An example of a streaming JSON format is [JSON lines](https://jsonlines.org/), where each line represents a JSON value. These JSON values can be considered top-level values or list items, and they must be records. - -### Examples - - -This is bad, because all keys must be assumed to be store types: - -```json -{ - "local": { ... }, - "remote": { ... }, - "http": { ... } -} -``` - -This is good, because the it is extensible at the root, and is somewhat self-documenting: - -```json -{ - "storeTypes": { "local": { ... }, ... }, - "pluginSupport": true -} -``` - -While the dictionary of store types seems like a very complete response at first, a use case may arise that warrants returning additional information. -For example, the presence of plugin support may be crucial information for a client to proceed when their desired store type is missing. - - - -The following representation is bad because it is not extensible: - -```json -{ "outputs": [ "out" "bin" ] } -``` - -However, simply converting everything to records is not enough, because the order of outputs must be preserved: - -```json -{ "outputs": { "bin": {}, "out": {} } } -``` - -The first item is the default output. Deriving this information from the outputs ordering is not great, but this is how Nix currently happens to work. -While it is possible for a JSON parser to preserve the order of fields, we can not rely on this capability to be present in all JSON libraries. - -This representation is extensible and preserves the ordering: - -```json -{ "outputs": [ { "outputName": "out" }, { "outputName": "bin" } ] } -``` - ## Dialog with the user CLIs don't always make it clear when an action has taken place. For every diff --git a/doc/manual/src/development/contributing.md b/doc/manual/src/development/contributing.md new file mode 100644 index 000000000..7de7489dc --- /dev/null +++ b/doc/manual/src/development/contributing.md @@ -0,0 +1,79 @@ +# Contributing + +## Add a release note + +`doc/manual/rl-next` contains release notes entries for all unreleased changes. + +User-visible changes should come with a release note. + +### Add an entry + +Here's what a complete entry looks like. The file name is not incorporated in the document. + +``` +--- +synopsis: Basically a title +issues: 1234 +prs: 1238 +--- + +Here's one or more paragraphs that describe the change. + +- It's markdown +- Add references to the manual using @docroot@ +``` + +Significant changes should add the following header, which moves them to the top. + +``` +significance: significant +``` + + +See also the [format documentation](https://github.com/haskell/cabal/blob/master/CONTRIBUTING.md#changelog). + +### Build process + +Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`. + +## Branches + +- [`master`](https://github.com/NixOS/nix/commits/master) + + The main development branch. All changes are approved and merged here. + When developing a change, create a branch based on the latest `master`. + + Maintainers try to [keep it in a release-worthy state](#reverting). + +- [`maintenance-*.*`](https://github.com/NixOS/nix/branches/all?query=maintenance) + + These branches are the subject of backports only, and are + also [kept](#reverting) in a release-worthy state. + + See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) + +- [`latest-release`](https://github.com/NixOS/nix/tree/latest-release) + + The latest patch release of the latest minor version. + + See [`maintainers/release-process.md`](https://github.com/NixOS/nix/blob/master/maintainers/release-process.md) + +- [`backport-*-to-*`](https://github.com/NixOS/nix/branches/all?query=backport) + + Generally branches created by the backport action. + + See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) + +- [_other_](https://github.com/NixOS/nix/branches/all) + + Branches that do not conform to the above patterns should be feature branches. + +## Reverting + +If a change turns out to be merged by mistake, or contain a regression, it may be reverted. +A revert is not a rejection of the contribution, but merely part of an effective development process. +It makes sure that development keeps running smoothly, with minimal uncertainty, and less overhead. +If maintainers have to worry too much about avoiding reverts, they would not be able to merge as much. +By embracing reverts as a good part of the development process, everyone wins. + +However, taking a step back may be frustrating, so maintainers will be extra supportive on the next try. diff --git a/doc/manual/src/contributing/cxx.md b/doc/manual/src/development/cxx.md similarity index 100% rename from doc/manual/src/contributing/cxx.md rename to doc/manual/src/development/cxx.md diff --git a/doc/manual/src/contributing/documentation.md b/doc/manual/src/development/documentation.md similarity index 89% rename from doc/manual/src/contributing/documentation.md rename to doc/manual/src/development/documentation.md index 1dddb207c..63f574ab7 100644 --- a/doc/manual/src/contributing/documentation.md +++ b/doc/manual/src/development/documentation.md @@ -24,14 +24,12 @@ nix build .#^doc and open `./result-doc/share/doc/nix/manual/index.html`. -To build the manual incrementally, [enter the development shell](./hacking.md) and run: +To build the manual incrementally, [enter the development shell](./building.md) and run: ```console -make manual-html -j $NIX_BUILD_CORES +make manual-html-open -j $NIX_BUILD_CORES ``` -and open `./outputs/out/share/doc/nix/manual/language/index.html`. - In order to reflect changes to the [Makefile for the manual], clear all generated files before re-building: [Makefile for the manual]: https://github.com/NixOS/nix/blob/master/doc/manual/local.mk @@ -149,7 +147,7 @@ Please observe these guidelines to ease reviews: ``` A [store object] contains a [file system object] and [references] to other store objects. - [store object]: @docroot@/glossary.md#gloss-store-object + [store object]: @docroot@/store/store-object.md [file system object]: @docroot@/architecture/file-system-object.md [references]: @docroot@/glossary.md#gloss-reference ``` @@ -198,13 +196,35 @@ You can also build and view it yourself: [Doxygen API documentation]: https://hydra.nixos.org/job/nix/master/internal-api-docs/latest/download-by-type/doc/internal-api-docs ```console -# nix build .#hydraJobs.internal-api-docs -# xdg-open ./result/share/doc/nix/internal-api/html/index.html +$ nix build .#hydraJobs.internal-api-docs +$ xdg-open ./result/share/doc/nix/internal-api/html/index.html +``` + +or inside `nix-shell` or `nix develop`: + +```console +$ mesonConfigurePhase +$ ninja src/internal-api-docs/html +$ xdg-open src/internal-api-docs/html/index.html +``` + +## C API documentation + +Note that the C API is not yet stable. +[C API documentation] is available online. +You can also build and view it yourself: + +[C API documentation]: https://hydra.nixos.org/job/nix/master/external-api-docs/latest/download-by-type/doc/external-api-docs + +```console +$ nix build .#hydraJobs.external-api-docs +$ xdg-open ./result/share/doc/nix/external-api/html/index.html ``` or inside `nix-shell` or `nix develop`: ``` -# make internal-api-html -# xdg-open ./outputs/doc/share/doc/nix/internal-api/html/index.html +$ mesonConfigurePhase +$ ninja src/external-api-docs/html +$ xdg-open src/external-api-docs/html/index.html ``` diff --git a/doc/manual/src/contributing/experimental-features.md b/doc/manual/src/development/experimental-features.md similarity index 100% rename from doc/manual/src/contributing/experimental-features.md rename to doc/manual/src/development/experimental-features.md diff --git a/doc/manual/src/contributing/index.md b/doc/manual/src/development/index.md similarity index 77% rename from doc/manual/src/contributing/index.md rename to doc/manual/src/development/index.md index 4d55c17a4..6403c3e66 100644 --- a/doc/manual/src/contributing/index.md +++ b/doc/manual/src/development/index.md @@ -5,4 +5,4 @@ Check the [contributing guide](https://github.com/NixOS/nix/blob/master/CONTRIBU This chapter is a collection of guides for making changes to the code and documentation. -If you're not sure where to start, try to [compile Nix from source](./hacking.md) and consider [making improvements to documentation](./documentation.md). +If you're not sure where to start, try to [compile Nix from source](./building.md) and consider [making improvements to documentation](./documentation.md). diff --git a/doc/manual/src/development/json-guideline.md b/doc/manual/src/development/json-guideline.md new file mode 100644 index 000000000..b4bc92af9 --- /dev/null +++ b/doc/manual/src/development/json-guideline.md @@ -0,0 +1,128 @@ +# JSON guideline + +Nix consumes and produces JSON in a variety of contexts. +These guidelines ensure consistent practices for all our JSON interfaces, for ease of use, and so that experience in one part carries over to another. + +## Extensibility + +The schema of JSON input and output should allow for backwards compatible extension. +This section explains how to achieve this. + +Two definitions are helpful here, because while JSON only defines one "key-value" object type, we use it to cover two use cases: + + - **dictionary**: a map from names to value that all have the same type. + In C++ this would be a `std::map` with string keys. + + - **record**: a fixed set of attributes each with their own type. + In C++, this would be represented by a `struct`. + +It is best not to mix these use cases, as that may lead to incompatibilities when the schema changes. +For example, adding a record field to a dictionary breaks consumers that assume all JSON object fields to have the same meaning and type, and dictionary items with a colliding name can not be represented anymore. + +This leads to the following guidelines: + + - The top-level (root) value must be a record. + + Otherwise, one can not change the structure of a command's output. + + - The value of a dictionary item must be a record. + + Otherwise, the item type can not be extended. + + - List items should be records. + + Otherwise, one can not change the structure of the list items. + + If the order of the items does not matter, and each item has a unique key that is a string, consider representing the list as a dictionary instead. + If the order of the items needs to be preserved, return a list of records. + + - Streaming JSON should return records. + + An example of a streaming JSON format is [JSON lines](https://jsonlines.org/), where each line represents a JSON value. + These JSON values can be considered top-level values or list items, and they must be records. + +### Examples + +This is bad, because all keys must be assumed to be store types: + +```json +{ + "local": { ... }, + "remote": { ... }, + "http": { ... } +} +``` + +This is good, because the it is extensible at the root, and is somewhat self-documenting: + +```json +{ + "storeTypes": { "local": { ... }, ... }, + "pluginSupport": true +} +``` + +While the dictionary of store types seems like a very complete response at first, a use case may arise that warrants returning additional information. +For example, the presence of plugin support may be crucial information for a client to proceed when their desired store type is missing. + + + +The following representation is bad because it is not extensible: + +```json +{ "outputs": [ "out" "bin" ] } +``` + +However, simply converting everything to records is not enough, because the order of outputs must be preserved: + +```json +{ "outputs": { "bin": {}, "out": {} } } +``` + +The first item is the default output. Deriving this information from the outputs ordering is not great, but this is how Nix currently happens to work. +While it is possible for a JSON parser to preserve the order of fields, we can not rely on this capability to be present in all JSON libraries. + +This representation is extensible and preserves the ordering: + +```json +{ "outputs": [ { "outputName": "out" }, { "outputName": "bin" } ] } +``` + +## Self-describing values + +As described in the previous section, it's crucial that schemas can be extended with with new fields without breaking compatibility. +However, that should *not* mean we use the presence/absence of fields to indicate optional information *within* a version of the schema. +Instead, always include the field, and use `null` to indicate the "nothing" case. + +### Examples + +Here are two JSON objects: + +```json +{ + "foo": {} +} +``` +```json +{ + "foo": {}, + "bar": {} +} +``` + +Since they differ in which fields they contain, they should *not* both be valid values of the same schema. +At most, they can match two different schemas where the second (with `foo` and `bar`) is considered a newer version of the first (with just `foo`). +Within each version, all fields are mandatory (always `foo`, and always `foo` and `bar`). +Only *between* each version, `bar` gets added as a new mandatory field. + +Here are another two JSON objects: + +```json +{ "foo": null } +``` +```json +{ "foo": { "bar": 1 } } +``` + +Since they both contain a `foo` field, they could be valid values of the same schema. +The schema would have `foo` has an optional field, which is either `null` or an object where `bar` is an integer. diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/development/testing.md similarity index 81% rename from doc/manual/src/contributing/testing.md rename to doc/manual/src/development/testing.md index 31c39c16c..3949164d5 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/development/testing.md @@ -59,15 +59,15 @@ The unit tests are defined using the [googletest] and [rapidcheck] frameworks. > … > ``` -The tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `tests/unit/${library_name_without-nix}`. -Given a interface (header) and implementation pair in the original library, say, `src/libexpr/value/context.{hh,cc}`, we write tests for it in `tests/unit/libexpr/tests/value/context.cc`, and (possibly) declare/define additional interfaces for testing purposes in `tests/unit/libexpr-support/tests/value/context.{hh,cc}`. +The tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `src/${library_name_without-nix}-test`. +Given an interface (header) and implementation pair in the original library, say, `src/libexpr/value/context.{hh,cc}`, we write tests for it in `src/nix-expr-tests/value/context.cc`, and (possibly) declare/define additional interfaces for testing purposes in `src/nix-expr-test-support/tests/value/context.{hh,cc}`. Data for unit tests is stored in a `data` subdir of the directory for each unit test executable. -For example, `libnixstore` code is in `src/libstore`, and its test data is in `tests/unit/libstore/data`. -The path to the `tests/unit/data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`. +For example, `libnixstore` code is in `src/libstore`, and its test data is in `src/nix-store-tests/data`. +The path to the `src/${library_name_without-nix}-test/data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`. Note that each executable only gets the data for its tests. -The unit test libraries are in `tests/unit/${library_name_without-nix}-lib`. +The unit test libraries are in `src/${library_name_without-nix}-test-support`. All headers are in a `tests` subdirectory so they are included with `#include "tests/"`. The use of all these separate directories for the unit tests might seem inconvenient, as for example the tests are not "right next to" the part of the code they are testing. @@ -76,8 +76,25 @@ there is no risk of any build-system wildcards for the library accidentally pick ### Running tests -You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. -Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option, or the `GTEST_FILTER` environment variable, e.g. `GTEST_FILTER='ErrorTraceTest.*' make check`. +You can run the whole testsuite with `meson test` from the Meson build directory, or the tests for a specific component with `meson test nix-store-tests`. +A environment variables that Google Test accepts are also worth knowing: + +1. [`GTEST_FILTER`](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) + + This is used for finer-grained filtering of which tests to run. + + +2. [`GTEST_BRIEF`](https://google.github.io/googletest/advanced.html#suppressing-test-passes) + + This is used to avoid logging passing tests. + +Putting the two together, one might run + +```bash +GTEST_BRIEF=1 GTEST_FILTER='ErrorTraceTest.*' meson test nix-expr-tests -v +``` + +for short but comprensive output. ### Characterisation testing { #characaterisation-testing-unit } @@ -86,7 +103,7 @@ See [functional characterisation testing](#characterisation-testing-functional) Like with the functional characterisation, `_NIX_TEST_ACCEPT=1` is also used. For example: ```shell-session -$ _NIX_TEST_ACCEPT=1 make libstore-tests_RUN +$ _NIX_TEST_ACCEPT=1 meson test nix-store-tests -v ... [ SKIPPED ] WorkerProtoTest.string_read [ SKIPPED ] WorkerProtoTest.string_write @@ -114,6 +131,8 @@ On other platforms they wouldn't be run at all. The functional tests reside under the `tests/functional` directory and are listed in `tests/functional/local.mk`. Each test is a bash script. +Functional tests are run during `installCheck` in the `nix` package build, as well as separately from the build, in VM tests. + ### Running the whole test suite The whole test suite can be run with: @@ -162,14 +181,14 @@ ran test tests/functional/${testName}.sh... [PASS] or without `make`: ```shell-session -$ ./mk/run-test.sh tests/functional/${testName}.sh tests/functional/init.sh +$ ./mk/run-test.sh tests/functional/${testName}.sh ran test tests/functional/${testName}.sh... [PASS] ``` To see the complete output, one can also run: ```shell-session -$ ./mk/debug-test.sh tests/functional/${testName}.sh tests/functional/init.sh +$ ./mk/debug-test.sh tests/functional/${testName}.sh +(${testName}.sh:1) foo output from foo +(${testName}.sh:2) bar @@ -204,7 +223,7 @@ edit it like so: Then, running the test with `./mk/debug-test.sh` will drop you into GDB once the script reaches that point: ```shell-session -$ ./mk/debug-test.sh tests/functional/${testName}.sh tests/functional/init.sh +$ ./mk/debug-test.sh tests/functional/${testName}.sh ... + gdb blash blub GNU gdb (GDB) 12.1 @@ -252,13 +271,30 @@ Regressions are caught, and improvements always show up in code review. To ensure that characterisation testing doesn't make it harder to intentionally change these interfaces, there always must be an easy way to regenerate the expected output, as we do with `_NIX_TEST_ACCEPT=1`. +### Running functional tests on NixOS + +We run the functional tests not just in the build, but also in VM tests. +This helps us ensure that Nix works correctly on NixOS, and environments that have similar characteristics that are hard to reproduce in a build environment. + +The recommended way to run these tests during development is: + +```shell +nix build .#hydraJobs.tests.functional_user.quickBuild +``` + +The `quickBuild` attribute configures the test to use a `nix` package that's built without integration tests, so that you can iterate on the tests without performing recompilations due to the changed sources for `installCheck`. + +Generally, this build is sufficient, but in nightly or CI we also test the attributes `functional_root` and `functional_trusted`, in which the test suite is run with different levels of authorization. + ## Integration tests The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute. These tests include everything that needs to interact with external services or run Nix in a non-trivial distributed setup. Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on ). -You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}` +You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}`. + +If you are testing a build of `nix` that you haven't compiled yet, you may iterate faster by appending the `quickBuild` attribute: `nix build .#hydraJobs.tests.{testName}.quickBuild`. ## Installer tests diff --git a/doc/manual/src/favicon.png b/doc/manual/src/favicon.png new file mode 100644 index 000000000..1ed2b5fe0 Binary files /dev/null and b/doc/manual/src/favicon.png differ diff --git a/doc/manual/src/favicon.svg b/doc/manual/src/favicon.svg new file mode 100644 index 000000000..1d2a6e835 --- /dev/null +++ b/doc/manual/src/favicon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index a71b2e2b3..877c4668b 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -1,5 +1,24 @@ # Glossary +- [content address]{#gloss-content-address} + + A + [*content address*](https://en.wikipedia.org/wiki/Content-addressable_storage) + is a secure way to reference immutable data. + The reference is calculated directly from the content of the data being referenced, which means the reference is + [*tamper proof*](https://en.wikipedia.org/wiki/Tamperproofing) + --- variations of the data should always calculate to distinct content addresses. + + For how Nix uses content addresses, see: + + - [Content-Addressing File System Objects](@docroot@/store/file-system-object/content-address.md) + - [Content-Addressing Store Objects](@docroot@/store/store-object/content-address.md) + - [content-addressed derivation](#gloss-content-addressed-derivation) + + Software Heritage's writing on [*Intrinsic and Extrinsic identifiers*](https://www.softwareheritage.org/2020/07/09/intrinsic-vs-extrinsic-identifiers) is also a good introduction to the value of content-addressing over other referencing schemes. + + Besides content addressing, the Nix store also uses [input addressing](#gloss-input-addressed-store-object). + - [derivation]{#gloss-derivation} A description of a build task. The result of a derivation is a @@ -52,10 +71,9 @@ [`__contentAddressed`](./language/advanced-attributes.md#adv-attr-__contentAddressed) attribute set to `true`. -- [fixed-output derivation]{#gloss-fixed-output-derivation} +- [fixed-output derivation]{#gloss-fixed-output-derivation} (FOD) - A derivation which includes the - [`outputHash`](./language/advanced-attributes.md#adv-attr-outputHash) attribute. + A [derivation] where a cryptographic hash of the [output] is determined in advance using the [`outputHash`](./language/advanced-attributes.md#adv-attr-outputHash) attribute, and where the [`builder`](@docroot@/language/derivations.md#attr-builder) executable has access to the network. - [store]{#gloss-store} @@ -86,7 +104,7 @@ [store path]: #gloss-store-path -- [file system object]{#gloss-store-object} +- [file system object]{#gloss-file-system-object} The Nix data model for representing simplified file system data. @@ -118,9 +136,12 @@ - [content-addressed store object]{#gloss-content-addressed-store-object} - A [store object] whose [store path] is determined by its contents. + A [store object] which is [content-addressed](#gloss-content-address), + i.e. whose [store path] is determined by its contents. This includes derivations, the outputs of [content-addressed derivations](#gloss-content-addressed-derivation), and the outputs of [fixed-output derivations](#gloss-fixed-output-derivation). + See [Content-Addressing Store Objects](@docroot@/store/store-object/content-address.md) for details. + - [substitute]{#gloss-substitute} A substitute is a command invocation stored in the [Nix database] that @@ -147,7 +168,7 @@ - [impure derivation]{#gloss-impure-derivation} - [An experimental feature](#@docroot@/contributing/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, + [An experimental feature](#@docroot@/development/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, so that they are always rebuilt, and their outputs not reused by subsequent calls to realise them. - [Nix database]{#gloss-nix-database} @@ -215,6 +236,20 @@ [output path]: #gloss-output-path +- [output closure]{#gloss-output-closure}\ + The [closure] of an [output path]. It only contains what is [reachable] from the output. + +- [deriving path]{#gloss-deriving-path} + + Deriving paths are a way to refer to [store objects][store object] that ar not yet [realised][realise]. + This is necessary because, in general and particularly for [content-addressed derivations][content-addressed derivation], the [output path] of an [output] is not known in advance. + There are two forms: + + - *constant*: just a [store path] + It can be made [valid][validity] by copying it into the store: from the evaluator, command line interface or another store. + + - *output*: a pair of a [store path] to a [derivation] and an [output] name. + - [deriver]{#gloss-deriver} The [store derivation] that produced an [output path]. @@ -252,13 +287,15 @@ See [installables](./command-ref/new-cli/nix.md#installables) for [`nix` commands](./command-ref/new-cli/nix.md) (experimental) for details. -- [NAR]{#gloss-nar} +- [Nix Archive (NAR)]{#gloss-nar} A *N*ix *AR*chive. This is a serialisation of a path in the Nix store. It can contain regular files, directories and symbolic links. NARs are generated and unpacked using `nix-store --dump` and `nix-store --restore`. + See [Nix Archive](store/file-system-object/content-address.html#serial-nix-archive) for details. + - [`∅`]{#gloss-emtpy-set} The empty set symbol. In the context of profile history, this denotes a package is not present in a particular version of the profile. @@ -275,7 +312,7 @@ - [package attribute set]{#package-attribute-set} - An [attribute set](@docroot@/language/values.md#attribute-set) containing the attribute `type = "derivation";` (derivation for historical reasons), as well as other attributes, such as + An [attribute set](@docroot@/language/types.md#attribute-set) containing the attribute `type = "derivation";` (derivation for historical reasons), as well as other attributes, such as - attributes that refer to the files of a [package], typically in the form of [derivation outputs](#output), - attributes that declare something about how the package is supposed to be installed or used, - other metadata or arbitrary attributes. @@ -288,16 +325,35 @@ See [String interpolation](./language/string-interpolation.md) for details. - [string]: ./language/values.md#type-string - [path]: ./language/values.md#type-path - [attribute name]: ./language/values.md#attribute-set + [string]: ./language/types.md#type-string + [path]: ./language/types.md#type-path + [attribute name]: ./language/types.md#attribute-set + +- [base directory]{#gloss-base-directory} + + The location from which relative paths are resolved. + + - For expressions in a file, the base directory is the directory containing that file. + This is analogous to the directory of a [base URL](https://datatracker.ietf.org/doc/html/rfc1808#section-3.3). + + + + - For expressions written in command line arguments with [`--expr`](@docroot@/command-ref/opt-common.html#opt-expr), the base directory is the current working directory. + + [base directory]: #gloss-base-directory - [experimental feature]{#gloss-experimental-feature} Not yet stabilized functionality guarded by named experimental feature flags. These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting. - See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/contributing/experimental-features.md). + See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/development/experimental-features.md). [Nix language]: ./language/index.md diff --git a/doc/manual/src/installation/env-variables.md b/doc/manual/src/installation/env-variables.md index db98f52ff..035090421 100644 --- a/doc/manual/src/installation/env-variables.md +++ b/doc/manual/src/installation/env-variables.md @@ -53,7 +53,8 @@ ssl-cert-file = /etc/ssl/my-certificate-bundle.crt The Nix installer has special handling for these proxy-related environment variables: `http_proxy`, `https_proxy`, `ftp_proxy`, -`no_proxy`, `HTTP_PROXY`, `HTTPS_PROXY`, `FTP_PROXY`, `NO_PROXY`. +`all_proxy`, `no_proxy`, `HTTP_PROXY`, `HTTPS_PROXY`, `FTP_PROXY`, +`ALL_PROXY`, `NO_PROXY`. If any of these variables are set when running the Nix installer, then the installer will create an override file at diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index 0dc989159..6a168ff3d 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -50,7 +50,7 @@ Supported systems: To explicitly instruct the installer to perform a multi-user installation on your system: ```console -$ curl -L https://nixos.org/nix/install | sh -s -- --daemon +$ bash <(curl -L https://nixos.org/nix/install) --daemon ``` You can run this under your usual user account or `root`. @@ -61,7 +61,7 @@ The script will invoke `sudo` as needed. To explicitly select a single-user installation on your system: ```console -$ curl -L https://nixos.org/nix/install | sh -s -- --no-daemon +$ bash <(curl -L https://nixos.org/nix/install) --no-daemon ``` In a single-user installation, `/nix` is owned by the invoking user. @@ -77,7 +77,7 @@ $ su root # Installing from a binary tarball You can also download a binary tarball that contains Nix and all its dependencies: -- Choose a [version](https://releases.nixos.org/?prefix=nix/) and [system type](../contributing/hacking.md#platforms) +- Choose a [version](https://releases.nixos.org/?prefix=nix/) and [system type](../development/building.md#platforms) - Download and unpack the tarball - Run the installer diff --git a/doc/manual/src/installation/uninstall.md b/doc/manual/src/installation/uninstall.md index 9ead5e53c..590327fea 100644 --- a/doc/manual/src/installation/uninstall.md +++ b/doc/manual/src/installation/uninstall.md @@ -1,16 +1,8 @@ # Uninstalling Nix -## Single User - -If you have a [single-user installation](./installing-binary.md#single-user-installation) of Nix, uninstall it by running: - -```console -$ rm -rf /nix -``` - ## Multi User -Removing a [multi-user installation](./installing-binary.md#multi-user-installation) of Nix is more involved, and depends on the operating system. +Removing a [multi-user installation](./installing-binary.md#multi-user-installation) depends on the operating system. ### Linux @@ -51,7 +43,15 @@ which you may remove. ### macOS -1. Edit `/etc/zshrc`, `/etc/bashrc`, and `/etc/bash.bashrc` to remove the lines sourcing `nix-daemon.sh`, which should look like this: +1. If system-wide shell initialisation files haven't been altered since installing Nix, use the backups made by the installer: + + ```console + sudo mv /etc/zshrc.backup-before-nix /etc/zshrc + sudo mv /etc/bashrc.backup-before-nix /etc/bashrc + sudo mv /etc/bash.bashrc.backup-before-nix /etc/bash.bashrc + ``` + + Otherwise, edit `/etc/zshrc`, `/etc/bashrc`, and `/etc/bash.bashrc` to remove the lines sourcing `nix-daemon.sh`, which should look like this: ```bash # Nix @@ -61,18 +61,6 @@ which you may remove. # End Nix ``` - If these files haven't been altered since installing Nix you can simply put - the backups back in place: - - ```console - sudo mv /etc/zshrc.backup-before-nix /etc/zshrc - sudo mv /etc/bashrc.backup-before-nix /etc/bashrc - sudo mv /etc/bash.bashrc.backup-before-nix /etc/bash.bashrc - ``` - - This will stop shells from sourcing the file and bringing everything you - installed using Nix in scope. - 2. Stop and remove the Nix daemon services: ```console @@ -82,8 +70,7 @@ which you may remove. sudo rm /Library/LaunchDaemons/org.nixos.darwin-store.plist ``` - This stops the Nix daemon and prevents it from being started next time you - boot the system. + This stops the Nix daemon and prevents it from being started next time you boot the system. 3. Remove the `nixbld` group and the `_nixbuildN` users: @@ -94,25 +81,42 @@ which you may remove. This will remove all the build users that no longer serve a purpose. -4. Edit fstab using `sudo vifs` to remove the line mounting the Nix Store - volume on `/nix`, which looks like - `UUID= /nix apfs rw,noauto,nobrowse,suid,owners` or - `LABEL=Nix\040Store /nix apfs rw,nobrowse`. This will prevent automatic - mounting of the Nix Store volume. +4. Edit fstab using `sudo vifs` to remove the line mounting the Nix Store volume on `/nix`, which looks like -5. Edit `/etc/synthetic.conf` to remove the `nix` line. If this is the only - line in the file you can remove it entirely, `sudo rm /etc/synthetic.conf`. - This will prevent the creation of the empty `/nix` directory to provide a - mountpoint for the Nix Store volume. + ``` + UUID= /nix apfs rw,noauto,nobrowse,suid,owners + ``` + or -6. Remove the files Nix added to your system: + ``` + LABEL=Nix\040Store /nix apfs rw,nobrowse + ``` + + by setting the cursor on the respective line using the error keys, and pressing `dd`, and then `:wq` to save the file. + + This will prevent automatic mounting of the Nix Store volume. + +5. Edit `/etc/synthetic.conf` to remove the `nix` line. + If this is the only line in the file you can remove it entirely: + + ```bash + if [ -f /etc/synthetic.conf ]; then + if [ "$(cat /etc/synthetic.conf)" = "nix" ]; then + sudo rm /etc/synthetic.conf + else + sudo vi /etc/synthetic.conf + fi + fi + ``` + + This will prevent the creation of the empty `/nix` directory. + +6. Remove the files Nix added to your system, except for the store: ```console sudo rm -rf /etc/nix /var/root/.nix-profile /var/root/.nix-defexpr /var/root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels ``` - This gets rid of any data Nix may have created except for the store which is - removed next. 7. Remove the Nix Store volume: @@ -120,29 +124,32 @@ which you may remove. sudo diskutil apfs deleteVolume /nix ``` - This will remove the Nix Store volume and everything that was added to the - store. + This will remove the Nix Store volume and everything that was added to the store. - If the output indicates that the command couldn't remove the volume, you should - make sure you don't have an _unmounted_ Nix Store volume. Look for a - "Nix Store" volume in the output of the following command: + If the output indicates that the command couldn't remove the volume, you should make sure you don't have an _unmounted_ Nix Store volume. + Look for a "Nix Store" volume in the output of the following command: ```console diskutil list ``` - If you _do_ see a "Nix Store" volume, delete it by re-running the diskutil - deleteVolume command, but replace `/nix` with the store volume's `diskXsY` - identifier. + If you _do_ find a "Nix Store" volume, delete it by running `diskutil deleteVolume` with the store volume's `diskXsY` identifier. > **Note** > -> After you complete the steps here, you will still have an empty `/nix` -> directory. This is an expected sign of a successful uninstall. The empty -> `/nix` directory will disappear the next time you reboot. +> After you complete the steps here, you will still have an empty `/nix` directory. +> This is an expected sign of a successful uninstall. +> The empty `/nix` directory will disappear the next time you reboot. > -> You do not have to reboot to finish uninstalling Nix. The uninstall is -> complete. macOS (Catalina+) directly controls root directories and its -> read-only root will prevent you from manually deleting the empty `/nix` -> mountpoint. +> You do not have to reboot to finish uninstalling Nix. +> The uninstall is complete. +> macOS (Catalina+) directly controls root directories, and its read-only root will prevent you from manually deleting the empty `/nix` mountpoint. +## Single User + +To remove a [single-user installation](./installing-binary.md#single-user-installation) of Nix, run: + +```console +$ rm -rf /nix ~/.nix-channels ~/.nix-defexpr ~/.nix-profile +``` +You might also want to manually remove references to Nix from your `~/.profile`. diff --git a/doc/manual/src/installation/upgrading.md b/doc/manual/src/installation/upgrading.md index 38edcdbc5..a433f1d30 100644 --- a/doc/manual/src/installation/upgrading.md +++ b/doc/manual/src/installation/upgrading.md @@ -28,7 +28,7 @@ $ sudo su ## macOS multi-user ```console -$ sudo nix-env --install --file '' --attr nix -I nixpkgs=channel:nixpkgs-unstable +$ sudo nix-env --install --file '' --attr nix cacert -I nixpkgs=channel:nixpkgs-unstable $ sudo launchctl remove org.nixos.nix-daemon $ sudo launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist ``` diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 7306fc182..51b83fc8a 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -113,19 +113,18 @@ Derivations can declare some infrequently used optional attributes. > `nix-build`. If the [`configurable-impure-env` experimental - feature](@docroot@/contributing/experimental-features.md#xp-feature-configurable-impure-env) + feature](@docroot@/development/experimental-features.md#xp-feature-configurable-impure-env) is enabled, these environment variables can also be controlled through the [`impure-env`](@docroot@/command-ref/conf-file.md#conf-impure-env) configuration setting. - [`outputHash`]{#adv-attr-outputHash}; [`outputHashAlgo`]{#adv-attr-outputHashAlgo}; [`outputHashMode`]{#adv-attr-outputHashMode}\ - These attributes declare that the derivation is a so-called - *fixed-output derivation*, which means that a cryptographic hash of - the output is already known in advance. When the build of a - fixed-output derivation finishes, Nix computes the cryptographic - hash of the output and compares it to the hash declared with these - attributes. If there is a mismatch, the build fails. + These attributes declare that the derivation is a so-called *fixed-output derivation* (FOD), which means that a cryptographic hash of the output is already known in advance. + + As opposed to regular derivations, the [`builder`] executable of a fixed-output derivation has access to the network. + Nix computes a cryptographic hash of its output and compares that to the hash declared with these attributes. + If there is a mismatch, the derivation fails. The rationale for fixed-output derivations is derivations such as those produced by the `fetchurl` function. This function downloads a @@ -188,38 +187,49 @@ Derivations can declare some infrequently used optional attributes. } ``` - The `outputHashAlgo` attribute specifies the hash algorithm used to - compute the hash. It can currently be `"sha1"`, `"sha256"` or - `"sha512"`. + The `outputHash` attribute must be a string containing the hash in either hexadecimal or "nix32" encoding, or following the format for integrity metadata as defined by [SRI](https://www.w3.org/TR/SRI/). + The "nix32" encoding is an adaptation of base-32 encoding. + The [`convertHash`](@docroot@/language/builtins.md#builtins-convertHash) function shows how to convert between different encodings, and the [`nix-hash` command](../command-ref/nix-hash.md) has information about obtaining the hash for some contents, as well as converting to and from encodings. + + The `outputHashAlgo` attribute specifies the hash algorithm used to compute the hash. + It can currently be `"sha1"`, `"sha256"`, `"sha512"`, or `null`. + `outputHashAlgo` can only be `null` when `outputHash` follows the SRI format. The `outputHashMode` attribute determines how the hash is computed. - It must be one of the following two values: + It must be one of the following values: - - `"flat"`\ - The output must be a non-executable regular file. If it isn’t, - the build fails. The hash is simply computed over the contents - of that file (so it’s equal to what Unix commands like - `sha256sum` or `sha1sum` produce). + - [`"flat"`](@docroot@/store/store-object/content-address.md#method-flat) This is the default. - - `"recursive"`\ - The hash is computed over the NAR archive dump of the output - (i.e., the result of [`nix-store --dump`](@docroot@/command-ref/nix-store/dump.md)). In - this case, the output can be anything, including a directory - tree. + - [`"recursive"` or `"nar"`](@docroot@/store/store-object/content-address.md#method-nix-archive) - The `outputHash` attribute, finally, must be a string containing - the hash in either hexadecimal or base-32 notation. (See the - [`nix-hash` command](../command-ref/nix-hash.md) for information - about converting to and from base-32 notation.) + > **Compatibility** + > + > `"recursive"` is the traditional way of indicating this, + > and is supported since 2005 (virtually the entire history of Nix). + > `"nar"` is more clear, and consistent with other parts of Nix (such as the CLI), + > however support for it is only added in Nix version 2.21. + + - [`"text"`](@docroot@/store/store-object/content-address.md#method-text) + + > **Warning** + > + > The use of this method for derivation outputs is part of the [`dynamic-derivations`][xp-feature-dynamic-derivations] experimental feature. + + - [`"git"`](@docroot@/store/store-object/content-address.md#method-git) + + > **Warning** + > + > This method is part of the [`git-hashing`][xp-feature-git-hashing] experimental feature. - [`__contentAddressed`]{#adv-attr-__contentAddressed} + > **Warning** - > This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md). + > This attribute is part of an [experimental feature](@docroot@/development/experimental-features.md). > > To use this attribute, you must enable the - > [`ca-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-ca-derivations) experimental feature. + > [`ca-derivations`][xp-feature-ca-derivations] experimental feature. > For example, in [nix.conf](../command-ref/conf-file.md) you could add: > > ``` @@ -268,7 +278,9 @@ Derivations can declare some infrequently used optional attributes. > **Note** > - > If set to `false`, the [`builder`](./derivations.md#attr-builder) should be able to run on the system type specified in the [`system` attribute](./derivations.md#attr-system), since the derivation cannot be substituted. + > If set to `false`, the [`builder`] should be able to run on the system type specified in the [`system` attribute](./derivations.md#attr-system), since the derivation cannot be substituted. + + [`builder`]: ./derivations.md#attr-builder - [`__structuredAttrs`]{#adv-attr-structuredAttrs}\ If the special attribute `__structuredAttrs` is set to `true`, the other derivation @@ -290,6 +302,12 @@ Derivations can declare some infrequently used optional attributes. (associative) arrays. For example, the attribute `hardening.format = true` ends up as the Bash associative array element `${hardening[format]}`. + > **Warning** + > + > If set to `true`, other advanced attributes such as [`allowedReferences`](#adv-attr-allowedReferences), [`allowedReferences`](#adv-attr-allowedReferences), [`allowedRequisites`](#adv-attr-allowedRequisites), + [`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites), maxSize, and maxClosureSize. + will have no effect. + - [`outputChecks`]{#adv-attr-outputChecks}\ When using [structured attributes](#adv-attr-structuredAttrs), the `outputChecks` attribute allows defining checks per-output. @@ -299,7 +317,7 @@ Derivations can declare some infrequently used optional attributes. [`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites), the following attributes are available: - - `maxSize` defines the maximum size of the resulting [store object](../glossary.md#gloss-store-object). + - `maxSize` defines the maximum size of the resulting [store object](@docroot@/store/store-object.md). - `maxClosureSize` defines the maximum size of the output's closure. - `ignoreSelfRefs` controls whether self-references should be considered when checking for allowed references/requisites. @@ -351,3 +369,7 @@ Derivations can declare some infrequently used optional attributes. ``` ensures that the derivation can only be built on a machine with the `kvm` feature. + +[xp-feature-ca-derivations]: @docroot@/development/experimental-features.md#xp-feature-ca-derivations +[xp-feature-dynamic-derivations]: @docroot@/development/experimental-features.md#xp-feature-dynamic-derivations +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/doc/manual/src/language/builtin-constants-prefix.md b/doc/manual/src/language/builtin-constants-prefix.md deleted file mode 100644 index 50f43006d..000000000 --- a/doc/manual/src/language/builtin-constants-prefix.md +++ /dev/null @@ -1,5 +0,0 @@ -# Built-in Constants - -These constants are built into the Nix language evaluator: - - diff --git a/doc/manual/src/language/builtin-constants-suffix.md b/doc/manual/src/language/builtin-constants-suffix.md deleted file mode 100644 index a74db2857..000000000 --- a/doc/manual/src/language/builtin-constants-suffix.md +++ /dev/null @@ -1 +0,0 @@ - diff --git a/doc/manual/src/language/builtins-prefix.md b/doc/manual/src/language/builtins-prefix.md index 7b2321466..fb983bb7f 100644 --- a/doc/manual/src/language/builtins-prefix.md +++ b/doc/manual/src/language/builtins-prefix.md @@ -1,9 +1,11 @@ -# Built-in Functions +# Built-ins -This section lists the functions built into the Nix language evaluator. -All built-in functions are available through the global [`builtins`](./builtin-constants.md#builtins-builtins) constant. +This section lists the values and functions built into the Nix language evaluator. +All built-ins are available through the global [`builtins`](#builtins-builtins) constant. -For convenience, some built-ins can be accessed directly: +Some built-ins are also exposed directly in the global scope: + + - [`derivation`](#builtins-derivation) - [`import`](#builtins-import) diff --git a/doc/manual/src/language/constructs.md b/doc/manual/src/language/constructs.md deleted file mode 100644 index a82ec5960..000000000 --- a/doc/manual/src/language/constructs.md +++ /dev/null @@ -1,408 +0,0 @@ -# Language Constructs - -## Recursive sets - -Recursive sets are like normal [attribute sets](./values.md#attribute-set), but the attributes can refer to each other. - -> *rec-attrset* = `rec {` [ *name* `=` *expr* `;` `]`... `}` - -Example: - -```nix -rec { - x = y; - y = 123; -}.x -``` - -This evaluates to `123`. - -Note that without `rec` the binding `x = y;` would -refer to the variable `y` in the surrounding scope, if one exists, and -would be invalid if no such variable exists. That is, in a normal -(non-recursive) set, attributes are not added to the lexical scope; in a -recursive set, they are. - -Recursive sets of course introduce the danger of infinite recursion. For -example, the expression - -```nix -rec { - x = y; - y = x; -}.x -``` - -will crash with an `infinite recursion encountered` error message. - -## Let-expressions - -A let-expression allows you to define local variables for an expression. - -> *let-in* = `let` [ *identifier* = *expr* ]... `in` *expr* - -Example: - -```nix -let - x = "foo"; - y = "bar"; -in x + y -``` - -This evaluates to `"foobar"`. - -## Inheriting attributes - -When defining an [attribute set](./values.md#attribute-set) or in a [let-expression](#let-expressions) it is often convenient to copy variables from the surrounding lexical scope (e.g., when you want to propagate attributes). -This can be shortened using the `inherit` keyword. - -Example: - -```nix -let x = 123; in -{ - inherit x; - y = 456; -} -``` - -is equivalent to - -```nix -let x = 123; in -{ - x = x; - y = 456; -} -``` - -and both evaluate to `{ x = 123; y = 456; }`. - -> **Note** -> -> This works because `x` is added to the lexical scope by the `let` construct. - -It is also possible to inherit attributes from another attribute set. - -Example: - -In this fragment from `all-packages.nix`, - -```nix -graphviz = (import ../tools/graphics/graphviz) { - inherit fetchurl stdenv libpng libjpeg expat x11 yacc; - inherit (xorg) libXaw; -}; - -xorg = { - libX11 = ...; - libXaw = ...; - ... -} - -libpng = ...; -libjpg = ...; -... -``` - -the set used in the function call to the function defined in -`../tools/graphics/graphviz` inherits a number of variables from the -surrounding scope (`fetchurl` ... `yacc`), but also inherits `libXaw` -(the X Athena Widgets) from the `xorg` set. - -Summarizing the fragment - -```nix -... -inherit x y z; -inherit (src-set) a b c; -... -``` - -is equivalent to - -```nix -... -x = x; y = y; z = z; -a = src-set.a; b = src-set.b; c = src-set.c; -... -``` - -when used while defining local variables in a let-expression or while -defining a set. - -In a `let` expression, `inherit` can be used to selectively bring specific attributes of a set into scope. For example - - -```nix -let - x = { a = 1; b = 2; }; - inherit (builtins) attrNames; -in -{ - names = attrNames x; -} -``` - -is equivalent to - -```nix -let - x = { a = 1; b = 2; }; -in -{ - names = builtins.attrNames x; -} -``` - -both evaluate to `{ names = [ "a" "b" ]; }`. - -## Functions - -Functions have the following form: - -```nix -pattern: body -``` - -The pattern specifies what the argument of the function must look like, -and binds variables in the body to (parts of) the argument. There are -three kinds of patterns: - - - If a pattern is a single identifier, then the function matches any - argument. Example: - - ```nix - let negate = x: !x; - concat = x: y: x + y; - in if negate true then concat "foo" "bar" else "" - ``` - - Note that `concat` is a function that takes one argument and returns - a function that takes another argument. This allows partial - parameterisation (i.e., only filling some of the arguments of a - function); e.g., - - ```nix - map (concat "foo") [ "bar" "bla" "abc" ] - ``` - - evaluates to `[ "foobar" "foobla" "fooabc" ]`. - - - A *set pattern* of the form `{ name1, name2, …, nameN }` matches a - set containing the listed attributes, and binds the values of those - attributes to variables in the function body. For example, the - function - - ```nix - { x, y, z }: z + y + x - ``` - - can only be called with a set containing exactly the attributes `x`, - `y` and `z`. No other attributes are allowed. If you want to allow - additional arguments, you can use an ellipsis (`...`): - - ```nix - { x, y, z, ... }: z + y + x - ``` - - This works on any set that contains at least the three named - attributes. - - It is possible to provide *default values* for attributes, in - which case they are allowed to be missing. A default value is - specified by writing `name ? e`, where *e* is an arbitrary - expression. For example, - - ```nix - { x, y ? "foo", z ? "bar" }: z + y + x - ``` - - specifies a function that only requires an attribute named `x`, but - optionally accepts `y` and `z`. - - - An `@`-pattern provides a means of referring to the whole value - being matched: - - ```nix - args@{ x, y, z, ... }: z + y + x + args.a - ``` - - but can also be written as: - - ```nix - { x, y, z, ... } @ args: z + y + x + args.a - ``` - - Here `args` is bound to the argument *as passed*, which is further - matched against the pattern `{ x, y, z, ... }`. - The `@`-pattern makes mainly sense with an ellipsis(`...`) as - you can access attribute names as `a`, using `args.a`, which was - given as an additional attribute to the function. - - > **Warning** - > - > `args@` binds the name `args` to the attribute set that is passed to the function. - > In particular, `args` does *not* include any default values specified with `?` in the function's set pattern. - > - > For instance - > - > ```nix - > let - > f = args@{ a ? 23, ... }: [ a args ]; - > in - > f {} - > ``` - > - > is equivalent to - > - > ```nix - > let - > f = args @ { ... }: [ (args.a or 23) args ]; - > in - > f {} - > ``` - > - > and both expressions will evaluate to: - > - > ```nix - > [ 23 {} ] - > ``` - -Note that functions do not have names. If you want to give them a name, -you can bind them to an attribute, e.g., - -```nix -let concat = { x, y }: x + y; -in concat { x = "foo"; y = "bar"; } -``` - -## Conditionals - -Conditionals look like this: - -```nix -if e1 then e2 else e3 -``` - -where *e1* is an expression that should evaluate to a Boolean value -(`true` or `false`). - -## Assertions - -Assertions are generally used to check that certain requirements on or -between features and dependencies hold. They look like this: - -```nix -assert e1; e2 -``` - -where *e1* is an expression that should evaluate to a Boolean value. If -it evaluates to `true`, *e2* is returned; otherwise expression -evaluation is aborted and a backtrace is printed. - -Here is a Nix expression for the Subversion package that shows how -assertions can be used:. - -```nix -{ localServer ? false -, httpServer ? false -, sslSupport ? false -, pythonBindings ? false -, javaSwigBindings ? false -, javahlBindings ? false -, stdenv, fetchurl -, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null -}: - -assert localServer -> db4 != null; â‘ -assert httpServer -> httpd != null && httpd.expat == expat; â‘¡ -assert sslSupport -> openssl != null && (httpServer -> httpd.openssl == openssl); â‘¢ -assert pythonBindings -> swig != null && swig.pythonSupport; -assert javaSwigBindings -> swig != null && swig.javaSupport; -assert javahlBindings -> j2sdk != null; - -stdenv.mkDerivation { - name = "subversion-1.1.1"; - ... - openssl = if sslSupport then openssl else null; â‘£ - ... -} -``` - -The points of interest are: - -1. This assertion states that if Subversion is to have support for - local repositories, then Berkeley DB is needed. So if the Subversion - function is called with the `localServer` argument set to `true` but - the `db4` argument set to `null`, then the evaluation fails. - - Note that `->` is the [logical - implication](https://en.wikipedia.org/wiki/Truth_table#Logical_implication) - Boolean operation. - -2. This is a more subtle condition: if Subversion is built with Apache - (`httpServer`) support, then the Expat library (an XML library) used - by Subversion should be same as the one used by Apache. This is - because in this configuration Subversion code ends up being linked - with Apache code, and if the Expat libraries do not match, a build- - or runtime link error or incompatibility might occur. - -3. This assertion says that in order for Subversion to have SSL support - (so that it can access `https` URLs), an OpenSSL library must be - passed. Additionally, it says that *if* Apache support is enabled, - then Apache's OpenSSL should match Subversion's. (Note that if - Apache support is not enabled, we don't care about Apache's - OpenSSL.) - -4. The conditional here is not really related to assertions, but is - worth pointing out: it ensures that if SSL support is disabled, then - the Subversion derivation is not dependent on OpenSSL, even if a - non-`null` value was passed. This prevents an unnecessary rebuild of - Subversion if OpenSSL changes. - -## With-expressions - -A *with-expression*, - -```nix -with e1; e2 -``` - -introduces the set *e1* into the lexical scope of the expression *e2*. -For instance, - -```nix -let as = { x = "foo"; y = "bar"; }; -in with as; x + y -``` - -evaluates to `"foobar"` since the `with` adds the `x` and `y` attributes -of `as` to the lexical scope in the expression `x + y`. The most common -use of `with` is in conjunction with the `import` function. E.g., - -```nix -with (import ./definitions.nix); ... -``` - -makes all attributes defined in the file `definitions.nix` available as -if they were defined locally in a `let`-expression. - -The bindings introduced by `with` do not shadow bindings introduced by -other means, e.g. - -```nix -let a = 3; in with { a = 1; }; let a = 4; in with { a = 2; }; ... -``` - -establishes the same scope as - -```nix -let a = 1; in let a = 2; in let a = 3; in let a = 4; in ... -``` - -## Comments - -Comments can be single-line, started with a `#` character, or -inline/multi-line, enclosed within `/* ... */`. diff --git a/doc/manual/src/language/constructs/lookup-path.md b/doc/manual/src/language/constructs/lookup-path.md index e87d2922b..11b9fe88c 100644 --- a/doc/manual/src/language/constructs/lookup-path.md +++ b/doc/manual/src/language/constructs/lookup-path.md @@ -4,9 +4,9 @@ > > *lookup-path* = `<` *identifier* [ `/` *identifier* ]... `>` -A lookup path is an identifier with an optional path suffix that resolves to a [path value](@docroot@/language/values.md#type-path) if the identifier matches a search path entry. +A lookup path is an identifier with an optional path suffix that resolves to a [path value](@docroot@/language/types.md#type-path) if the identifier matches a search path entry. -The value of a lookup path is determined by [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath). +The value of a lookup path is determined by [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath). See [`builtins.findFile`](@docroot@/language/builtins.md#builtins-findFile) for details on lookup path resolution. diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index 75f824a34..8e3f0f791 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -12,12 +12,12 @@ It outputs an attribute set, and produces a [store derivation] as a side effect ### Required -- [`name`]{#attr-name} ([String](@docroot@/language/values.md#type-string)) +- [`name`]{#attr-name} ([String](@docroot@/language/types.md#type-string)) A symbolic name for the derivation. It is added to the [store path] of the corresponding [store derivation] as well as to its [output paths](@docroot@/glossary.md#gloss-output-path). - [store path]: @docroot@/glossary.md#gloss-store-path + [store path]: @docroot@/store/store-path.md > **Example** > @@ -31,7 +31,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect > The store derivation's path will be `/nix/store/-hello.drv`. > The [output](#attr-outputs) paths will be of the form `/nix/store/-hello[-]` -- [`system`]{#attr-system} ([String](@docroot@/language/values.md#type-string)) +- [`system`]{#attr-system} ([String](@docroot@/language/types.md#type-string)) The system type on which the [`builder`](#attr-builder) executable is meant to be run. @@ -64,9 +64,9 @@ It outputs an attribute set, and produces a [store derivation] as a side effect > } > ``` > - > [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) has the value of the [`system` configuration option], and defaults to the system type of the current Nix installation. + > [`builtins.currentSystem`](@docroot@/language/builtins.md#builtins-currentSystem) has the value of the [`system` configuration option], and defaults to the system type of the current Nix installation. -- [`builder`]{#attr-builder} ([Path](@docroot@/language/values.md#type-path) | [String](@docroot@/language/values.md#type-string)) +- [`builder`]{#attr-builder} ([Path](@docroot@/language/types.md#type-path) | [String](@docroot@/language/types.md#type-string)) Path to an executable that will perform the build. @@ -113,7 +113,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect ### Optional -- [`args`]{#attr-args} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) +- [`args`]{#attr-args} ([List](@docroot@/language/types.md#list) of [String](@docroot@/language/types.md#type-string)) Default: `[ ]` @@ -132,7 +132,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect > }; > ``` -- [`outputs`]{#attr-outputs} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) +- [`outputs`]{#attr-outputs} ([List](@docroot@/language/types.md#list) of [String](@docroot@/language/types.md#type-string)) Default: `[ "out" ]` @@ -141,7 +141,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect By default, a derivation produces a single output called `out`. However, derivations can produce multiple outputs. - This allows the associated [store objects](@docroot@/glossary.md#gloss-store-object) and their [closures](@docroot@/glossary.md#gloss-closure) to be copied or garbage-collected separately. + This allows the associated [store objects](@docroot@/store/store-object.md) and their [closures](@docroot@/glossary.md#gloss-closure) to be copied or garbage-collected separately. > **Example** > diff --git a/doc/manual/src/language/identifiers.md b/doc/manual/src/language/identifiers.md new file mode 100644 index 000000000..861ee3e20 --- /dev/null +++ b/doc/manual/src/language/identifiers.md @@ -0,0 +1,51 @@ +# Identifiers + +An *identifier* is an [ASCII](https://en.wikipedia.org/wiki/ASCII) character sequence that: +- Starts with a letter (`a-z`, `A-Z`) or underscore (`_`) +- Can contain any number of: + - Letters (`a-z`, `A-Z`) + - Digits (`0-9`) + - Underscores (`_`) + - Apostrophes (`'`) + - Hyphens (`-`) +- Is not one of the [keywords](#keywords) + +> **Syntax** +> +> *identifier* ~ `[A-Za-z_][A-Za-z0-9_'-]*` + +# Names + +A *name* can be written as an [identifier](#identifier) or a [string literal](./syntax.md#string-literal). + +> **Syntax** +> +> *name* → *identifier* | *string* + +Names are used in [attribute sets](./syntax.md#attrs-literal), [`let` bindings](./syntax.md#let-expressions), and [`inherit`](./syntax.md#inheriting-attributes). +Two names are the same if they represent the same sequence of characters, regardless of whether they are written as identifiers or strings. + +# Keywords + +These keywords are reserved and cannot be used as [identifiers](#identifiers): + +- [`assert`](./syntax.md#assertions) +- [`else`][if] +- [`if`][if] +- [`in`][let] +- [`inherit`](./syntax.md#inheriting-attributes) +- [`let`][let] +- [`or`](./operators.md#attribute-selection) (see note) +- [`rec`](./syntax.md#recursive-sets) +- [`then`][if] +- [`with`](./syntax.md#with-expressions) + +[if]: ./syntax.md#conditionals +[let]: ./syntax.md#let-expressions + +> **Note** +> +> The Nix language evaluator currently allows `or` to be used as a name in some contexts, for backwards compatibility reasons. +> Users are advised not to rely on this. +> +> There are long-standing issues with how `or` is parsed as a name, which can't be resolved without making a breaking change to the language. diff --git a/doc/manual/src/language/import-from-derivation.md b/doc/manual/src/language/import-from-derivation.md index fb12ba51a..e901f5bcf 100644 --- a/doc/manual/src/language/import-from-derivation.md +++ b/doc/manual/src/language/import-from-derivation.md @@ -2,9 +2,9 @@ The value of a Nix expression can depend on the contents of a [store object]. -[store object]: @docroot@/glossary.md#gloss-store-object +[store object]: @docroot@/store/store-object.md -Passing an expression `expr` that evaluates to a [store path](@docroot@/glossary.md#gloss-store-path) to any built-in function which reads from the filesystem constitutes Import From Derivation (IFD): +Passing an expression `expr` that evaluates to a [store path](@docroot@/store/store-path.md) to any built-in function which reads from the filesystem constitutes Import From Derivation (IFD): - [`import`](./builtins.md#builtins-import)` expr` - [`builtins.readFile`](./builtins.md#builtins-readFile)` expr` diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md index a26e43a05..2bfdbb8a0 100644 --- a/doc/manual/src/language/index.md +++ b/doc/manual/src/language/index.md @@ -1,7 +1,13 @@ # Nix Language The Nix language is designed for conveniently creating and composing *derivations* – precise descriptions of how contents of existing files are used to derive new files. -It is: + +> **Tip** +> +> These pages are written as a reference. +> If you are learning Nix, nix.dev has a good [introduction to the Nix language](https://nix.dev/tutorials/nix-language). + +The language is: - *domain-specific* @@ -47,7 +53,7 @@ This is an incomplete overview of language features, by example. - *Basic values* + *Basic values ([primitives](@docroot@/language/types.md#primitives))* @@ -65,7 +71,7 @@ This is an incomplete overview of language features, by example. - A string + A [string](@docroot@/language/types.md#type-string) @@ -88,6 +94,18 @@ This is an incomplete overview of language features, by example. + + + + `# Explanation` + + + + + A [comment](@docroot@/language/syntax.md#comments). + + + @@ -100,7 +118,7 @@ This is an incomplete overview of language features, by example. - String interpolation (expands to `"hello world"`, `"1 2 3"`, `"/nix/store/-bash-/bin/sh"`) + [String interpolation](@docroot@/language/string-interpolation.md) (expands to `"hello world"`, `"1 2 3"`, `"/nix/store/-bash-/bin/sh"`) @@ -112,7 +130,7 @@ This is an incomplete overview of language features, by example. - Booleans + [Booleans](@docroot@/language/types.md#type-boolean) @@ -124,7 +142,7 @@ This is an incomplete overview of language features, by example. - Null value + [Null](@docroot@/language/types.md#type-null) value @@ -136,7 +154,7 @@ This is an incomplete overview of language features, by example. - An integer + An [integer](@docroot@/language/types.md#type-int) @@ -148,7 +166,7 @@ This is an incomplete overview of language features, by example. - A floating point number + A [floating point number](@docroot@/language/types.md#type-float) @@ -160,7 +178,7 @@ This is an incomplete overview of language features, by example. - An absolute path + An absolute [path](@docroot@/language/types.md#type-path) @@ -172,7 +190,7 @@ This is an incomplete overview of language features, by example. - A path relative to the file containing this Nix expression + A [path](@docroot@/language/types.md#type-path) relative to the file containing this Nix expression @@ -184,7 +202,7 @@ This is an incomplete overview of language features, by example. - A home path. Evaluates to the `"/.config"`. + A home [path](@docroot@/language/types.md#type-path). Evaluates to the `"/.config"`. @@ -196,7 +214,7 @@ This is an incomplete overview of language features, by example. - Search path for Nix files. Value determined by [`$NIX_PATH` environment variable](../command-ref/env-common.md#env-NIX_PATH). + A [lookup path](@docroot@/language/constructs/lookup-path.md) for Nix files. Value determined by [`$NIX_PATH` environment variable](../command-ref/env-common.md#env-NIX_PATH). @@ -220,7 +238,7 @@ This is an incomplete overview of language features, by example. - A set with attributes named `x` and `y` + An [attribute set](@docroot@/language/types.md#attribute-set) with attributes named `x` and `y` @@ -244,7 +262,7 @@ This is an incomplete overview of language features, by example. - A recursive set, equivalent to `{ x = "foo"; y = "foobar"; }` + A [recursive set](@docroot@/language/syntax.md#recursive-sets), equivalent to `{ x = "foo"; y = "foobar"; }`. @@ -260,7 +278,7 @@ This is an incomplete overview of language features, by example. - Lists with three elements. + [Lists](@docroot@/language/types.md#list) with three elements. @@ -344,7 +362,7 @@ This is an incomplete overview of language features, by example. - Attribute selection (evaluates to `1`) + [Attribute selection](@docroot@/language/types.md#attribute-set) (evaluates to `1`) @@ -356,7 +374,7 @@ This is an incomplete overview of language features, by example. - Attribute selection with default (evaluates to `3`) + [Attribute selection](@docroot@/language/types.md#attribute-set) with default (evaluates to `3`) @@ -392,7 +410,7 @@ This is an incomplete overview of language features, by example. - Conditional expression + [Conditional expression](@docroot@/language/syntax.md#conditionals). @@ -404,7 +422,7 @@ This is an incomplete overview of language features, by example. - Assertion check (evaluates to `"yes!"`). + [Assertion](@docroot@/language/syntax.md#assertions) check (evaluates to `"yes!"`). @@ -416,7 +434,7 @@ This is an incomplete overview of language features, by example. - Variable definition + Variable definition. See [`let`-expressions](@docroot@/language/syntax.md#let-expressions). @@ -428,14 +446,44 @@ This is an incomplete overview of language features, by example. - Add all attributes from the given set to the scope (evaluates to `1`) + Add all attributes from the given set to the scope (evaluates to `1`). + + See [`with`-expressions](@docroot@/language/syntax.md#with-expressions) for details and shadowing caveats. - *Functions (lambdas)* + `inherit pkgs src;` + + + + + Adds the variables to the current scope (attribute set or `let` binding). + Desugars to `pkgs = pkgs; src = src;`. + See [Inheriting attributes](@docroot@/language/syntax.md#inheriting-attributes). + + + + + + + `inherit (pkgs) lib stdenv;` + + + + + Adds the attributes, from the attribute set in parentheses, to the current scope (attribute set or `let` binding). + Desugars to `lib = pkgs.lib; stdenv = pkgs.stdenv;`. + See [Inheriting attributes](@docroot@/language/syntax.md#inheriting-attributes). + + + + + + + *[Functions](@docroot@/language/syntax.md#functions) (lambdas)* @@ -452,7 +500,7 @@ This is an incomplete overview of language features, by example. - A function that expects an integer and returns it increased by 1 + A [function](@docroot@/language/syntax.md#functions) that expects an integer and returns it increased by 1. @@ -464,7 +512,7 @@ This is an incomplete overview of language features, by example. - Curried function, equivalent to `x: (y: x + y)`. Can be used like a function that takes two arguments and returns their sum. + Curried [function](@docroot@/language/syntax.md#functions), equivalent to `x: (y: x + y)`. Can be used like a function that takes two arguments and returns their sum. @@ -476,7 +524,7 @@ This is an incomplete overview of language features, by example. - A function call (evaluates to 101) + A [function](@docroot@/language/syntax.md#functions) call (evaluates to 101) @@ -488,7 +536,7 @@ This is an incomplete overview of language features, by example. - A function bound to a variable and subsequently called by name (evaluates to 103) + A [function](@docroot@/language/syntax.md#functions) bound to a variable and subsequently called by name (evaluates to 103) @@ -500,7 +548,7 @@ This is an incomplete overview of language features, by example. - A function that expects a set with required attributes `x` and `y` and concatenates them + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attributes `x` and `y` and concatenates them @@ -512,7 +560,7 @@ This is an incomplete overview of language features, by example. - A function that expects a set with required attribute `x` and optional `y`, using `"bar"` as default value for `y` + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attribute `x` and optional `y`, using `"bar"` as default value for `y` @@ -524,7 +572,7 @@ This is an incomplete overview of language features, by example. - A function that expects a set with required attributes `x` and `y` and ignores any other attributes + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attributes `x` and `y` and ignores any other attributes @@ -538,7 +586,7 @@ This is an incomplete overview of language features, by example. - A function that expects a set with required attributes `x` and `y`, and binds the whole set to `args` + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attributes `x` and `y`, and binds the whole set to `args` @@ -562,7 +610,8 @@ This is an incomplete overview of language features, by example. - Load and return Nix expression in given file + Load and return Nix expression in given file. + See [import](@docroot@/language/builtins.md#builtins-import). @@ -574,7 +623,8 @@ This is an incomplete overview of language features, by example. - Apply a function to every element of a list (evaluates to `[ 2 4 6 ]`) + Apply a function to every element of a list (evaluates to `[ 2 4 6 ]`). + See [`map`](@docroot@/language/builtins.md#builtins-map). diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index 6fd66864b..e1c020781 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -26,12 +26,16 @@ | Logical conjunction (`AND`) | *bool* `&&` *bool* | left | 12 | | Logical disjunction (`OR`) | *bool* \|\| *bool* | left | 13 | | [Logical implication] | *bool* `->` *bool* | right | 14 | +| [Pipe operator] (experimental) | *expr* `\|>` *func* | left | 15 | +| [Pipe operator] (experimental) | *func* `<\|` *expr* | right | 15 | -[string]: ./values.md#type-string -[path]: ./values.md#type-path -[number]: ./values.md#type-number -[list]: ./values.md#list -[attribute set]: ./values.md#attribute-set +[string]: ./types.md#type-string +[path]: ./types.md#type-path +[number]: ./types.md#type-float +[list]: ./types.md#list +[attribute set]: ./types.md#attribute-set + + ## Attribute selection @@ -42,12 +46,6 @@ Select the attribute denoted by attribute path *attrpath* from [attribute set] *attrset*. If the attribute doesn’t exist, return the *expr* after `or` if provided, otherwise abort evaluation. -An attribute path is a dot-separated list of [attribute names](./values.md#attribute-set). - -> **Syntax** -> -> *attrpath* = *name* [ `.` *name* ]... - [Attribute selection]: #attribute-selection ## Has attribute @@ -61,7 +59,7 @@ The result is a [Boolean] value. See also: [`builtins.hasAttr`](@docroot@/language/builtins.md#builtins-hasAttr) -[Boolean]: ./values.md#type-boolean +[Boolean]: ./types.md#type-boolean [Has attribute]: #has-attribute @@ -128,8 +126,8 @@ The result is a string. > The file or directory at *path* must exist and is copied to the [store]. > The path appears in the result as the corresponding [store path]. -[store path]: ../glossary.md#gloss-store-path -[store]: ../glossary.md#gloss-store +[store path]: @docroot@/store/store-path.md +[store]: @docroot@/glossary.md#gloss-store [String and path concatenation]: #string-and-path-concatenation @@ -141,7 +139,7 @@ The result is a string. Update [attribute set] *attrset1* with names and values from *attrset2*. -The returned attribute set will have of all the attributes in *attrset1* and *attrset2*. +The returned attribute set will have all of the attributes in *attrset1* and *attrset2*. If an attribute name is present in both, the attribute value from the latter is taken. [Update]: #update @@ -172,7 +170,7 @@ All comparison operators are implemented in terms of `<`, and the following equi - Numbers are type-compatible, see [arithmetic] operators. - Floating point numbers only differ up to a limited precision. -[function]: ./constructs.md#functions +[function]: ./syntax.md#functions [Equality]: #equality @@ -182,3 +180,34 @@ Equivalent to `!`*b1* `||` *b2*. [Logical implication]: #logical-implication +## Pipe operators + +- *a* `|>` *b* is equivalent to *b* *a* +- *a* `<|` *b* is equivalent to *a* *b* + +> **Example** +> +> ``` +> nix-repl> 1 |> builtins.add 2 |> builtins.mul 3 +> 9 +> +> nix-repl> builtins.add 1 <| builtins.mul 2 <| 3 +> 7 +> ``` + +> **Warning** +> +> This syntax is part of an +> [experimental feature](@docroot@/development/experimental-features.md) +> and may change in future releases. +> +> To use this syntax, make sure the +> [`pipe-operators` experimental feature](@docroot@/development/experimental-features.md#xp-feature-pipe-operators) +> is enabled. +> For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): +> +> ``` +> extra-experimental-features = pipe-operators +> ``` + +[Pipe operator]: #pipe-operators diff --git a/doc/manual/src/language/scope.md b/doc/manual/src/language/scope.md new file mode 100644 index 000000000..9373324e2 --- /dev/null +++ b/doc/manual/src/language/scope.md @@ -0,0 +1,28 @@ +# Scoping rules + +A *scope* in the Nix language is a dictionary keyed by [name](./identifiers.md#names), mapping each name to an expression and a *definition type*. +The definition type is either *explicit* or *implicit*. +Each entry in this dictionary is a *definition*. + +Explicit definitions are created by the following expressions: +- [let-expressions](syntax.md#let-expressions) +- [recursive attribute set literals](syntax.md#recursive-sets) (`rec`) +- [function literals](syntax.md#functions) + +Implicit definitions are only created by [with-expressions](./syntax.md#with-expressions). + +Every expression is *enclosed* by a scope. +The outermost expression is enclosed by the [built-in, global scope](./builtins.md), which contains only explicit definitions. +The expressions listed above *extend* their enclosing scope by adding new definitions, or replacing existing ones with the same name. +An explicit definition can replace a definition of any type; an implicit definition can only replace another implicit definition. + +Each of the above expressions defines which of its subexpressions are enclosed by the extended scope. +In all other cases, the same scope that encloses an expression is the enclosing scope for its subexpressions. + +The Nix language is [statically scoped](https://en.wikipedia.org/wiki/Scope_(computer_science)#Lexical_scope); +the value of a variable is determined only by the variable's enclosing scope, and not by the dynamic context in which the variable is evaluated. + +> **Note** +> +> Expressions entered into the [Nix REPL](@docroot@/command-ref/new-cli/nix3-repl.md) are enclosed by a scope that can be extended by command line arguments or previous REPL commands. +> These ways of extending scope are not, strictly speaking, part of the Nix language. diff --git a/doc/manual/src/language/string-context.md b/doc/manual/src/language/string-context.md new file mode 100644 index 000000000..6a3482cfd --- /dev/null +++ b/doc/manual/src/language/string-context.md @@ -0,0 +1,134 @@ +# String context + +> **Note** +> +> This is an advanced topic. +> The Nix language is designed to be used without the programmer consciously dealing with string contexts or even knowing what they are. + +A string in the Nix language is not just a sequence of characters like strings in other languages. +It is actually a pair of a sequence of characters and a *string context*. +The string context is an (unordered) set of *string context elements*. + +The purpose of string contexts is to collect non-string values attached to strings via +[string concatenation](./operators.md#string-concatenation), +[string interpolation](./string-interpolation.md), +and similar operations. +The idea is that a user can combine together values to create a build instructions for derivations without manually keeping track of where they come from. +Then the Nix language implicitly does that bookkeeping to efficiently obtain the closure of derivation inputs. + +> **Note** +> +> String contexts are *not* explicitly manipulated in idiomatic Nix language code. + +String context elements come in different forms: + +- [deriving path]{#string-context-element-derived-path} + + A string context element of this type is a [deriving path](@docroot@/glossary.md#gloss-deriving-path). + They can be either of type [constant](#string-context-constant) or [output](#string-context-output), which correspond to the types of deriving paths. + + - [Constant string context elements]{#string-context-constant} + + > **Example** + > + > [`builtins.storePath`] creates a string with a single constant string context element: + > + > ```nix + > builtins.getContext (builtins.storePath "/nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10") + > ``` + > evaluates to + > ```nix + > { + > "/nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10" = { + > path = true; + > }; + > } + > ``` + + [deriving path]: @docroot@/glossary.md#gloss-deriving-path + [store path]: @docroot@/glossary.md#gloss-store-path + [`builtins.storePath`]: ./builtins.md#builtins-storePath + + - [Output string context elements]{#string-context-output} + + > **Example** + > + > The behavior of string contexts are best demonstrated with a built-in function that is still experimental: [`builtins.outputOf`]. + > This example will *not* work with stable Nix! + > + > ```nix + > builtins.getContext + > (builtins.outputOf + > (builtins.storePath "/nix/store/fvchh9cvcr7kdla6n860hshchsba305w-hello-2.12.drv") + > "out") + > ``` + > evaluates to + > ```nix + > { + > "/nix/store/fvchh9cvcr7kdla6n860hshchsba305w-hello-2.12.drv" = { + > outputs = [ "out" ]; + > }; + > } + > ``` + + [`builtins.outputOf`]: ./builtins.md#builtins-outputOf + +- [*derivation deep*]{#string-context-element-derivation-deep} + + *derivation deep* is an advanced feature intended to be used with the + [`exportReferencesGraph` derivation attribute](./advanced-attributes.html#adv-attr-exportReferencesGraph). + A *derivation deep* string context element is a derivation path, and refers to both its outputs and the entire build closure of that derivation: + all its outputs, all the other derivations the given derivation depends on, and all the outputs of those. + + > **Example** + > + > The best way to illustrate *derivation deep* string contexts is with [`builtins.addDrvOutputDependencies`]. + > Take a regular constant string context element pointing to a derivation, and transform it into a "Derivation deep" string context element. + > + > ```nix + > builtins.getContext + > (builtins.addDrvOutputDependencies + > (builtins.storePath "/nix/store/fvchh9cvcr7kdla6n860hshchsba305w-hello-2.12.drv")) + > ``` + > evaluates to + > ```nix + > { + > "/nix/store/fvchh9cvcr7kdla6n860hshchsba305w-hello-2.12.drv" = { + > allOutputs = true; + > }; + > } + > ``` + + [`builtins.addDrvOutputDependencies`]: ./builtins.md#builtins-addDrvOutputDependencies + [`builtins.unsafeDiscardOutputDependency`]: ./builtins.md#builtins-unsafeDiscardOutputDependency + +## Inspecting string contexts + +Most basically, [`builtins.hasContext`] will tell whether a string has a non-empty context. + +When more granular information is needed, [`builtins.getContext`] can be used. +It creates an [attribute set] representing the string context, which can be inspected as usual. + +[`builtins.hasContext`]: ./builtins.md#builtins-hasContext +[`builtins.getContext`]: ./builtins.md#builtins-getContext +[attribute set]: ./types.md#attribute-set + +## Clearing string contexts + +[`buitins.unsafeDiscardStringContext`](./builtins.md#builtins-unsafeDiscardStringContext) will make a copy of a string, but with an empty string context. +The returned string can be used in more ways, e.g. by operators that require the string context to be empty. +The requirement to explicitly discard the string context in such use cases helps ensure that string context elements are not lost by mistake. +The "unsafe" marker is only there to remind that Nix normally guarantees that dependencies are tracked, whereas the returned string has lost them. + +## Constructing string contexts + +[`builtins.appendContext`] will create a copy of a string, but with additional string context elements. +The context is specified explicitly by an [attribute set] in the format that [`builtins.hasContext`] produces. +A string with arbitrary contexts can be made like this: + +1. Create a string with the desired string context elements. + (The contents of the string do not matter.) +2. Dump its context with [`builtins.getContext`]. +3. Combine it with a base string and repeated [`builtins.appendContext`] calls. + +[`builtins.appendContext`]: ./builtins.md#builtins-appendContext diff --git a/doc/manual/src/language/string-interpolation.md b/doc/manual/src/language/string-interpolation.md index 7d81c2020..1778bdfa0 100644 --- a/doc/manual/src/language/string-interpolation.md +++ b/doc/manual/src/language/string-interpolation.md @@ -4,9 +4,9 @@ String interpolation is a language feature where a [string], [path], or [attribu Such a construct is called *interpolated string*, and the expression inside is an [interpolated expression](#interpolated-expression). -[string]: ./values.md#type-string -[path]: ./values.md#type-path -[attribute set]: ./values.md#attribute-set +[string]: ./types.md#type-string +[path]: ./types.md#type-path +[attribute set]: ./types.md#attribute-set ## Examples @@ -20,7 +20,7 @@ Rather than writing (where `freetype` is a [derivation]), you can instead write -[derivation]: ../glossary.md#gloss-derivation +[derivation]: @docroot@/glossary.md#gloss-derivation ```nix "--with-freetype2-library=${freetype}/lib" @@ -43,6 +43,47 @@ configureFlags = " Note that Nix expressions and strings can be arbitrarily nested; in this case the outer string contains various interpolated expressions that themselves contain strings (e.g., `"-thread"`), some of which in turn contain interpolated expressions (e.g., `${mesa}`). +To write a literal `${` in an regular string, escape it with a backslash (`\`). + +> **Example** +> +> ```nix +> "echo \${PATH}" +> ``` +> +> "echo ${PATH}" + +To write a literal `${` in an indented string, escape it with two single quotes (`''`). + +> **Example** +> +> ```nix +> '' +> echo ''${PATH} +> '' +> ``` +> +> "echo ${PATH}\n" + +`$${` can be written literally in any string. + +> **Example** +> +> In Make, `$` in file names or recipes is represented as `$$`, see [GNU `make`: Basics of Variable Reference](https://www.gnu.org/software/make/manual/html_node/Reference.html#Basics-of-Variable-References). +> This can be expressed directly in the Nix language strings: +> +> ```nix +> '' +> MAKEVAR = Hello +> all: +> @export BASHVAR=world; echo $(MAKEVAR) $${BASHVAR} +> '' +> ``` +> +> "MAKEVAR = Hello\nall:\n\t@export BASHVAR=world; echo $(MAKEVAR) $\${BASHVAR}\n" + +See the [documentation on strings][string] for details. + ### Path Rather than writing @@ -107,9 +148,9 @@ An expression that is interpolated must evaluate to one of the following: A string interpolates to itself. -A path in an interpolated expression is first copied into the Nix store, and the resulting string is the [store path] of the newly created [store object](../glossary.md#gloss-store-object). +A path in an interpolated expression is first copied into the Nix store, and the resulting string is the [store path] of the newly created [store object](@docroot@/store/store-object.md). -[store path]: ../glossary.md#gloss-store-path +[store path]: @docroot@/store/store-path.md > **Example** > diff --git a/doc/manual/src/language/syntax.md b/doc/manual/src/language/syntax.md new file mode 100644 index 000000000..6108bacd6 --- /dev/null +++ b/doc/manual/src/language/syntax.md @@ -0,0 +1,866 @@ +# Language Constructs + +This section covers syntax and semantics of the Nix language. + +## Basic Literals + +### String {#string-literal} + + *Strings* can be written in three ways. + + The most common way is to enclose the string between double quotes, e.g., `"foo bar"`. + Strings can span multiple lines. + The results of other expressions can be included into a string by enclosing them in `${ }`, a feature known as [string interpolation]. + + [string interpolation]: ./string-interpolation.md + + The following must be escaped to represent them within a string, by prefixing with a backslash (`\`): + + - Double quote (`"`) + + > **Example** + > + > ```nix + > "\"" + > ``` + > + > "\"" + + - Backslash (`\`) + + > **Example** + > + > ```nix + > "\\" + > ``` + > + > "\\" + + - Dollar sign followed by an opening curly bracket (`${`) – "dollar-curly" + + > **Example** + > + > ```nix + > "\${" + > ``` + > + > "\${" + + The newline, carriage return, and tab characters can be written as `\n`, `\r` and `\t`, respectively. + + A "double-dollar-curly" (`$${`) can be written literally. + + > **Example** + > + > ```nix + > "$${" + > ``` + > + > "$\${" + + String values are output on the terminal with Nix-specific escaping. + Strings written to files will contain the characters encoded by the escaping. + + The second way to write string literals is as an *indented string*, which is enclosed between pairs of *double single-quotes* (`''`), like so: + + ```nix + '' + This is the first line. + This is the second line. + This is the third line. + '' + ``` + + This kind of string literal intelligently strips indentation from + the start of each line. To be precise, it strips from each line a + number of spaces equal to the minimal indentation of the string as a + whole (disregarding the indentation of empty lines). For instance, + the first and second line are indented two spaces, while the third + line is indented four spaces. Thus, two spaces are stripped from + each line, so the resulting string is + + ```nix + "This is the first line.\nThis is the second line.\n This is the third line.\n" + ``` + + > **Note** + > + > Whitespace and newline following the opening `''` is ignored if there is no non-whitespace text on the initial line. + + > **Warning** + > + > Prefixed tab characters are not stripped. + > + > > **Example** + > > + > > The following indented string is prefixed with tabs: + > > + > > '' + > > all: + > > @echo hello + > > '' + > > + > > "\tall:\n\t\t@echo hello\n" + + Indented strings support [string interpolation]. + + The following must be escaped to represent them in an indented string: + + - `$` is escaped by prefixing it with two single quotes (`''`) + + > **Example** + > + > ```nix + > '' + > ''$ + > '' + > ``` + > + > "$\n" + + - `''` is escaped by prefixing it with one single quote (`'`) + + > **Example** + > + > ```nix + > '' + > ''' + > '' + > ``` + > + > "''\n" + + These special characters are escaped as follows: + - Linefeed (`\n`): `''\n` + - Carriage return (`\r`): `''\r` + - Tab (`\t`): `''\t` + + `''\` escapes any other character. + + A "double-dollar-curly" (`$${`) can be written literally. + + > **Example** + > + > ```nix + > '' + > $${ + > '' + > ``` + > + > "$\${\n" + + Indented strings are primarily useful in that they allow multi-line + string literals to follow the indentation of the enclosing Nix + expression, and that less escaping is typically necessary for + strings representing languages such as shell scripts and + configuration files because `''` is much less common than `"`. + Example: + + ```nix + stdenv.mkDerivation { + ... + postInstall = + '' + mkdir $out/bin $out/etc + cp foo $out/bin + echo "Hello World" > $out/etc/foo.conf + ${if enableBar then "cp bar $out/bin" else ""} + ''; + ... + } + ``` + + Finally, as a convenience, *URIs* as defined in appendix B of + [RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as + is*, without quotes. For instance, the string + `"http://example.org/foo.tar.bz2"` can also be written as + `http://example.org/foo.tar.bz2`. + +### Number {#number-literal} + + + + Numbers, which can be *integers* (like `123`) or *floating point* + (like `123.43` or `.27e13`). + + See [arithmetic] and [comparison] operators for semantics. + + [arithmetic]: ./operators.md#arithmetic + [comparison]: ./operators.md#comparison + +### Path {#path-literal} + + *Paths* can be expressed by path literals such as `./builder.sh`. + + A path literal must contain at least one slash to be recognised as such. + For instance, `builder.sh` is not a path: + it's parsed as an expression that selects the attribute `sh` from the variable `builder`. + + Path literals are resolved relative to their [base directory](@docroot@/glossary.md#gloss-base-directory). + Path literals may also refer to absolute paths by starting with a slash. + + > **Note** + > + > Absolute paths make expressions less portable. + > In the case where a function translates a path literal into an absolute path string for a configuration file, it is recommended to write a string literal instead. + > This avoids some confusion about whether files at that location will be used during evaluation. + > It also avoids unintentional situations where some function might try to copy everything at the location into the store. + + If the first component of a path is a `~`, it is interpreted such that the rest of the path were relative to the user's home directory. + For example, `~/foo` would be equivalent to `/home/edolstra/foo` for a user whose home directory is `/home/edolstra`. + Path literals that start with `~` are not allowed in [pure](@docroot@/command-ref/conf-file.md#conf-pure-eval) evaluation. + + Path literals can also include [string interpolation], besides being [interpolated into other expressions]. + + [interpolated into other expressions]: ./string-interpolation.md#interpolated-expressions + + At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path. + + `a.${foo}/b.${bar}` is a syntactically valid number division operation. + `./a.${foo}/b.${bar}` is a path. + + [Lookup path](./constructs/lookup-path.md) literals such as `` also resolve to path values. + +## List {#list-literal} + +Lists are formed by enclosing a whitespace-separated list of values +between square brackets. For example, + +```nix +[ 123 ./foo.nix "abc" (f { x = y; }) ] +``` + +defines a list of four elements, the last being the result of a call to +the function `f`. Note that function calls have to be enclosed in +parentheses. If they had been omitted, e.g., + +```nix +[ 123 ./foo.nix "abc" f { x = y; } ] +``` + +the result would be a list of five elements, the fourth one being a +function and the fifth being a set. + +Note that lists are only lazy in values, and they are strict in length. + +Elements in a list can be accessed using [`builtins.elemAt`](./builtins.md#builtins-elemAt). + +## Attribute Set {#attrs-literal} + +An attribute set is a collection of name-value-pairs called *attributes*. + +Attribute sets are written enclosed in curly brackets (`{ }`). +Attribute names and attribute values are separated by an equal sign (`=`). +Each value can be an arbitrary expression, terminated by a semicolon (`;`) + +An attribute name is a string without context, and is denoted by a [name] (an [identifier](./identifiers.md#identifiers) or [string literal](#string-literal)). + +[name]: ./identifiers.md#names + +> **Syntax** +> +> *attrset* → `{` { *name* `=` *expr* `;` } `}` + +Attributes can appear in any order. +An attribute name may only occur once in each attribute set. + +> **Example** +> +> This defines an attribute set with attributes named: +> - `x` with the value `123`, an integer +> - `text` with the value `"Hello"`, a string +> - `y` where the value is the result of applying the function `f` to the attribute set `{ bla = 456; }` +> +> ```nix +> { +> x = 123; +> text = "Hello"; +> y = f { bla = 456; }; +> } +> ``` + +Attributes in nested attribute sets can be written using *attribute paths*. + +> **Syntax** +> +> *attrset* → `{` { *attrpath* `=` *expr* `;` } `}` + +An attribute path is a dot-separated list of [names][name]. + +> **Syntax** +> +> *attrpath* = *name* { `.` *name* } + + + +> **Example** +> +> ```nix +> { a.b.c = 1; a.b.d = 2; } +> ``` +> +> { +> a = { +> b = { +> c = 1; +> d = 2; +> }; +> }; +> } + +Attribute names can also be set implicitly by using the [`inherit` keyword](#inheriting-attributes). + +> **Example** +> +> ```nix +> { inherit (builtins) true; } +> ``` +> +> { true = true; } + +Attributes can be accessed with the [`.` operator](./operators.md#attribute-selection). + +Example: + +```nix +{ a = "Foo"; b = "Bar"; }.a +``` + +This evaluates to `"Foo"`. + +It is possible to provide a default value in an attribute selection using the `or` keyword. + +Example: + +```nix +{ a = "Foo"; b = "Bar"; }.c or "Xyzzy" +``` + +```nix +{ a = "Foo"; b = "Bar"; }.c.d.e.f.g or "Xyzzy" +``` + +will both evaluate to `"Xyzzy"` because there is no `c` attribute in the set. + +You can use arbitrary double-quoted strings as attribute names: + +```nix +{ "$!@#?" = 123; }."$!@#?" +``` + +```nix +let bar = "bar"; in +{ "foo ${bar}" = 123; }."foo ${bar}" +``` + +Both will evaluate to `123`. + +Attribute names support [string interpolation]: + +```nix +let bar = "foo"; in +{ foo = 123; }.${bar} +``` + +```nix +let bar = "foo"; in +{ ${bar} = 123; }.foo +``` + +Both will evaluate to `123`. + +In the special case where an attribute name inside of a set declaration +evaluates to `null` (which is normally an error, as `null` cannot be coerced to +a string), that attribute is simply not added to the set: + +```nix +{ ${if foo then "bar" else null} = true; } +``` + +This will evaluate to `{}` if `foo` evaluates to `false`. + +A set that has a `__functor` attribute whose value is callable (i.e. is +itself a function or a set with a `__functor` attribute whose value is +callable) can be applied as if it were a function, with the set itself +passed in first , e.g., + +```nix +let add = { __functor = self: x: x + self.x; }; + inc = add // { x = 1; }; +in inc 1 +``` + +evaluates to `2`. This can be used to attach metadata to a function +without the caller needing to treat it specially, or to implement a form +of object-oriented programming, for example. + +## Recursive sets + +Recursive sets are like normal [attribute sets](./types.md#attribute-set), but the attributes can refer to each other. + +> *rec-attrset* = `rec {` [ *name* `=` *expr* `;` `]`... `}` + +Example: + +```nix +rec { + x = y; + y = 123; +}.x +``` + +This evaluates to `123`. + +Note that without `rec` the binding `x = y;` would +refer to the variable `y` in the surrounding scope, if one exists, and +would be invalid if no such variable exists. That is, in a normal +(non-recursive) set, attributes are not added to the lexical scope; in a +recursive set, they are. + +Recursive sets of course introduce the danger of infinite recursion. For +example, the expression + +```nix +rec { + x = y; + y = x; +}.x +``` + +will crash with an `infinite recursion encountered` error message. + +## Let-expressions + +A let-expression allows you to define local variables for an expression. + +> *let-in* = `let` [ *identifier* = *expr* ]... `in` *expr* + +Example: + +```nix +let + x = "foo"; + y = "bar"; +in x + y +``` + +This evaluates to `"foobar"`. + +## Inheriting attributes + +When defining an [attribute set](./types.md#attribute-set) or in a [let-expression](#let-expressions) it is often convenient to copy variables from the surrounding lexical scope (e.g., when you want to propagate attributes). +This can be shortened using the `inherit` keyword. + +Example: + +```nix +let x = 123; in +{ + inherit x; + y = 456; +} +``` + +is equivalent to + +```nix +let x = 123; in +{ + x = x; + y = 456; +} +``` + +and both evaluate to `{ x = 123; y = 456; }`. + +> **Note** +> +> This works because `x` is added to the lexical scope by the `let` construct. + +It is also possible to inherit attributes from another attribute set. + +Example: + +In this fragment from `all-packages.nix`, + +```nix +graphviz = (import ../tools/graphics/graphviz) { + inherit fetchurl stdenv libpng libjpeg expat x11 yacc; + inherit (xorg) libXaw; +}; + +xorg = { + libX11 = ...; + libXaw = ...; + ... +} + +libpng = ...; +libjpg = ...; +... +``` + +the set used in the function call to the function defined in +`../tools/graphics/graphviz` inherits a number of variables from the +surrounding scope (`fetchurl` ... `yacc`), but also inherits `libXaw` +(the X Athena Widgets) from the `xorg` set. + +Summarizing the fragment + +```nix +... +inherit x y z; +inherit (src-set) a b c; +... +``` + +is equivalent to + +```nix +... +x = x; y = y; z = z; +a = src-set.a; b = src-set.b; c = src-set.c; +... +``` + +when used while defining local variables in a let-expression or while +defining a set. + +In a `let` expression, `inherit` can be used to selectively bring specific attributes of a set into scope. For example + + +```nix +let + x = { a = 1; b = 2; }; + inherit (builtins) attrNames; +in +{ + names = attrNames x; +} +``` + +is equivalent to + +```nix +let + x = { a = 1; b = 2; }; +in +{ + names = builtins.attrNames x; +} +``` + +both evaluate to `{ names = [ "a" "b" ]; }`. + +## Functions + +Functions have the following form: + +```nix +pattern: body +``` + +The pattern specifies what the argument of the function must look like, +and binds variables in the body to (parts of) the argument. There are +three kinds of patterns: + + - If a pattern is a single identifier, then the function matches any + argument. Example: + + ```nix + let negate = x: !x; + concat = x: y: x + y; + in if negate true then concat "foo" "bar" else "" + ``` + + Note that `concat` is a function that takes one argument and returns + a function that takes another argument. This allows partial + parameterisation (i.e., only filling some of the arguments of a + function); e.g., + + ```nix + map (concat "foo") [ "bar" "bla" "abc" ] + ``` + + evaluates to `[ "foobar" "foobla" "fooabc" ]`. + + - A *set pattern* of the form `{ name1, name2, …, nameN }` matches a + set containing the listed attributes, and binds the values of those + attributes to variables in the function body. For example, the + function + + ```nix + { x, y, z }: z + y + x + ``` + + can only be called with a set containing exactly the attributes `x`, + `y` and `z`. No other attributes are allowed. If you want to allow + additional arguments, you can use an ellipsis (`...`): + + ```nix + { x, y, z, ... }: z + y + x + ``` + + This works on any set that contains at least the three named + attributes. + + It is possible to provide *default values* for attributes, in + which case they are allowed to be missing. A default value is + specified by writing `name ? e`, where *e* is an arbitrary + expression. For example, + + ```nix + { x, y ? "foo", z ? "bar" }: z + y + x + ``` + + specifies a function that only requires an attribute named `x`, but + optionally accepts `y` and `z`. + + - An `@`-pattern provides a means of referring to the whole value + being matched: + + ```nix + args@{ x, y, z, ... }: z + y + x + args.a + ``` + + but can also be written as: + + ```nix + { x, y, z, ... } @ args: z + y + x + args.a + ``` + + Here `args` is bound to the argument *as passed*, which is further + matched against the pattern `{ x, y, z, ... }`. + The `@`-pattern makes mainly sense with an ellipsis(`...`) as + you can access attribute names as `a`, using `args.a`, which was + given as an additional attribute to the function. + + > **Warning** + > + > `args@` binds the name `args` to the attribute set that is passed to the function. + > In particular, `args` does *not* include any default values specified with `?` in the function's set pattern. + > + > For instance + > + > ```nix + > let + > f = args@{ a ? 23, ... }: [ a args ]; + > in + > f {} + > ``` + > + > is equivalent to + > + > ```nix + > let + > f = args @ { ... }: [ (args.a or 23) args ]; + > in + > f {} + > ``` + > + > and both expressions will evaluate to: + > + > ```nix + > [ 23 {} ] + > ``` + +Note that functions do not have names. If you want to give them a name, +you can bind them to an attribute, e.g., + +```nix +let concat = { x, y }: x + y; +in concat { x = "foo"; y = "bar"; } +``` + +## Conditionals + +Conditionals look like this: + +```nix +if e1 then e2 else e3 +``` + +where *e1* is an expression that should evaluate to a Boolean value +(`true` or `false`). + +## Assertions + +Assertions are generally used to check that certain requirements on or +between features and dependencies hold. They look like this: + +```nix +assert e1; e2 +``` + +where *e1* is an expression that should evaluate to a Boolean value. If +it evaluates to `true`, *e2* is returned; otherwise expression +evaluation is aborted and a backtrace is printed. + +Here is a Nix expression for the Subversion package that shows how +assertions can be used:. + +```nix +{ localServer ? false +, httpServer ? false +, sslSupport ? false +, pythonBindings ? false +, javaSwigBindings ? false +, javahlBindings ? false +, stdenv, fetchurl +, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null +}: + +assert localServer -> db4 != null; â‘ +assert httpServer -> httpd != null && httpd.expat == expat; â‘¡ +assert sslSupport -> openssl != null && (httpServer -> httpd.openssl == openssl); â‘¢ +assert pythonBindings -> swig != null && swig.pythonSupport; +assert javaSwigBindings -> swig != null && swig.javaSupport; +assert javahlBindings -> j2sdk != null; + +stdenv.mkDerivation { + name = "subversion-1.1.1"; + ... + openssl = if sslSupport then openssl else null; â‘£ + ... +} +``` + +The points of interest are: + +1. This assertion states that if Subversion is to have support for + local repositories, then Berkeley DB is needed. So if the Subversion + function is called with the `localServer` argument set to `true` but + the `db4` argument set to `null`, then the evaluation fails. + + Note that `->` is the [logical + implication](https://en.wikipedia.org/wiki/Truth_table#Logical_implication) + Boolean operation. + +2. This is a more subtle condition: if Subversion is built with Apache + (`httpServer`) support, then the Expat library (an XML library) used + by Subversion should be same as the one used by Apache. This is + because in this configuration Subversion code ends up being linked + with Apache code, and if the Expat libraries do not match, a build- + or runtime link error or incompatibility might occur. + +3. This assertion says that in order for Subversion to have SSL support + (so that it can access `https` URLs), an OpenSSL library must be + passed. Additionally, it says that *if* Apache support is enabled, + then Apache's OpenSSL should match Subversion's. (Note that if + Apache support is not enabled, we don't care about Apache's + OpenSSL.) + +4. The conditional here is not really related to assertions, but is + worth pointing out: it ensures that if SSL support is disabled, then + the Subversion derivation is not dependent on OpenSSL, even if a + non-`null` value was passed. This prevents an unnecessary rebuild of + Subversion if OpenSSL changes. + +## With-expressions + +A *with-expression*, + +```nix +with e1; e2 +``` + +introduces the set *e1* into the lexical scope of the expression *e2*. +For instance, + +```nix +let as = { x = "foo"; y = "bar"; }; +in with as; x + y +``` + +evaluates to `"foobar"` since the `with` adds the `x` and `y` attributes +of `as` to the lexical scope in the expression `x + y`. The most common +use of `with` is in conjunction with the `import` function. E.g., + +```nix +with (import ./definitions.nix); ... +``` + +makes all attributes defined in the file `definitions.nix` available as +if they were defined locally in a `let`-expression. + +The bindings introduced by `with` do not shadow bindings introduced by +other means, e.g. + +```nix +let a = 3; in with { a = 1; }; let a = 4; in with { a = 2; }; ... +``` + +establishes the same scope as + +```nix +let a = 1; in let a = 2; in let a = 3; in let a = 4; in ... +``` + +Variables coming from outer `with` expressions *are* shadowed: + +```nix +with { a = "outer"; }; +with { a = "inner"; }; +a +``` + +Does evaluate to `"inner"`. + +## Comments + +- Inline comments start with `#` and run until the end of the line. + + > **Example** + > + > ```nix + > # A number + > 2 # Equals 1 + 1 + > ``` + > + > ```console + > 2 + > ``` + +- Block comments start with `/*` and run until the next occurrence of `*/`. + + > **Example** + > + > ```nix + > /* + > Block comments + > can span multiple lines. + > */ "hello" + > ``` + > + > ```console + > "hello" + > ``` + + This means that block comments cannot be nested. + + > **Example** + > + > ```nix + > /* /* nope */ */ 1 + > ``` + > + > ```console + > error: syntax error, unexpected '*' + > + > at «string»:1:15: + > + > 1| /* /* nope */ * + > | ^ + > ``` + + Consider escaping nested comments and unescaping them in post-processing. + + > **Example** + > + > ```nix + > /* /* nested *\/ */ 1 + > ``` + > + > ```console + > 1 + > ``` diff --git a/doc/manual/src/language/types.md b/doc/manual/src/language/types.md new file mode 100644 index 000000000..229756e6b --- /dev/null +++ b/doc/manual/src/language/types.md @@ -0,0 +1,120 @@ +# Data Types + +Every value in the Nix language has one of the following types: + +* [Integer](#type-int) +* [Float](#type-float) +* [Boolean](#type-bool) +* [String](#type-string) +* [Path](#type-path) +* [Null](#type-null) +* [Attribute set](#type-attrs) +* [List](#type-list) +* [Function](#type-function) +* [External](#type-external) + +## Primitives + +### Integer {#type-int} + +An _integer_ in the Nix language is a signed 64-bit integer. + +Non-negative integers can be expressed as [integer literals](syntax.md#number-literal). +Negative integers are created with the [arithmetic negation operator](./operators.md#arithmetic). +The function [`builtins.isInt`](builtins.md#builtins-isInt) can be used to determine if a value is an integer. + +### Float {#type-float} + +A _float_ in the Nix language is a 64-bit [IEEE 754](https://en.wikipedia.org/wiki/IEEE_754) floating-point number. + +Most non-negative floats can be expressed as [float literals](syntax.md#number-literal). +Negative floats are created with the [arithmetic negation operator](./operators.md#arithmetic). +The function [`builtins.isFloat`](builtins.md#builtins-isFloat) can be used to determine if a value is a float. + +### Boolean {#type-bool} + +A _boolean_ in the Nix language is one of _true_ or _false_. + + + +These values are available as attributes of [`builtins`](builtins.md#builtins-builtins) as [`builtins.true`](builtins.md#builtins-true) and [`builtins.false`](builtins.md#builtins-false). +The function [`builtins.isBool`](builtins.md#builtins-isBool) can be used to determine if a value is a boolean. + +### String {#type-string} + +A _string_ in the Nix language is an immutable, finite-length sequence of bytes, along with a [string context](string-context.md). +Nix does not assume or support working natively with character encodings. + +String values without string context can be expressed as [string literals](syntax.md#string-literal). +The function [`builtins.isString`](builtins.md#builtins-isString) can be used to determine if a value is a string. + +### Path {#type-path} + +A _path_ in the Nix language is an immutable, finite-length sequence of bytes starting with `/`, representing a POSIX-style, canonical file system path. +Path values are distinct from string values, even if they contain the same sequence of bytes. +Operations that produce paths will simplify the result as the standard C function [`realpath`] would, except that there is no symbolic link resolution. + +[`realpath`]: https://pubs.opengroup.org/onlinepubs/9699919799/functions/realpath.html + +Paths are suitable for referring to local files, and are often preferable over strings. +- Path values do not contain trailing or duplicate slashes, `.`, or `..`. +- Relative path literals are automatically resolved relative to their [base directory]. +- Tooling can recognize path literals and provide additional features, such as autocompletion, refactoring automation and jump-to-file. + +[base directory]: @docroot@/glossary.md#gloss-base-directory + +A file is not required to exist at a given path in order for that path value to be valid, but a path that is converted to a string with [string interpolation] or [string-and-path concatenation] must resolve to a readable file or directory which will be copied into the Nix store. +For instance, evaluating `"${./foo.txt}"` will cause `foo.txt` from the same directory to be copied into the Nix store and result in the string `"/nix/store/-foo.txt"`. +Operations such as [`import`] can also expect a path to resolve to a readable file or directory. + +[string interpolation]: string-interpolation.md#interpolated-expression +[string-and-path concatenation]: operators.md#string-and-path-concatenation +[`import`]: builtins.md#builtins-import + +> **Note** +> +> The Nix language assumes that all input files will remain _unchanged_ while evaluating a Nix expression. +> For example, assume you used a file path in an interpolated string during a `nix repl` session. +> Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new [store path], since Nix might not re-read the file contents. +> Use `:r` to reset the repl as needed. + +[store path]: @docroot@/store/store-path.md + +Path values can be expressed as [path literals](syntax.md#path-literal). +The function [`builtins.isPath`](builtins.md#builtins-isPath) can be used to determine if a value is a path. + +### Null {#type-null} + +There is a single value of type _null_ in the Nix language. + + + +This value is available as an attribute on the [`builtins`](builtins.md#builtins-builtins) attribute set as [`builtins.null`](builtins.md#builtins-null). + +## Compound values + +### Attribute set {#type-attrs} + + + +An attribute set can be constructed with an [attribute set literal](syntax.md#attrs-literal). +The function [`builtins.isAttrs`](builtins.md#builtins-isAttrs) can be used to determine if a value is an attribute set. + +### List {#type-list} + + + +A list can be constructed with a [list literal](syntax.md#list-literal). +The function [`builtins.isList`](builtins.md#builtins-isList) can be used to determine if a value is a list. + +## Function {#type-function} + + + +A function can be constructed with a [function expression](syntax.md#functions). +The function [`builtins.isFunction`](builtins.md#builtins-isFunction) can be used to determine if a value is a function. + +## External {#type-external} + +An _external_ value is an opaque value created by a Nix [plugin](../command-ref/conf-file.md#conf-plugin-files). +Such a value can be substituted in Nix expressions but only created and used by plugin code. diff --git a/doc/manual/src/language/values.md b/doc/manual/src/language/values.md deleted file mode 100644 index 74ffc7070..000000000 --- a/doc/manual/src/language/values.md +++ /dev/null @@ -1,269 +0,0 @@ -# Data Types - -## Primitives - -- String - - *Strings* can be written in three ways. - - The most common way is to enclose the string between double quotes, - e.g., `"foo bar"`. Strings can span multiple lines. The special - characters `"` and `\` and the character sequence `${` must be - escaped by prefixing them with a backslash (`\`). Newlines, carriage - returns and tabs can be written as `\n`, `\r` and `\t`, - respectively. - - You can include the results of other expressions into a string by enclosing them in `${ }`, a feature known as [string interpolation]. - - [string interpolation]: ./string-interpolation.md - - The second way to write string literals is as an *indented string*, - which is enclosed between pairs of *double single-quotes*, like so: - - ```nix - '' - This is the first line. - This is the second line. - This is the third line. - '' - ``` - - This kind of string literal intelligently strips indentation from - the start of each line. To be precise, it strips from each line a - number of spaces equal to the minimal indentation of the string as a - whole (disregarding the indentation of empty lines). For instance, - the first and second line are indented two spaces, while the third - line is indented four spaces. Thus, two spaces are stripped from - each line, so the resulting string is - - ```nix - "This is the first line.\nThis is the second line.\n This is the third line.\n" - ``` - - Note that the whitespace and newline following the opening `''` is - ignored if there is no non-whitespace text on the initial line. - - Indented strings support [string interpolation]. - - Since `${` and `''` have special meaning in indented strings, you - need a way to quote them. `$` can be escaped by prefixing it with - `''` (that is, two single quotes), i.e., `''$`. `''` can be escaped - by prefixing it with `'`, i.e., `'''`. `$` removes any special - meaning from the following `$`. Linefeed, carriage-return and tab - characters can be written as `''\n`, `''\r`, `''\t`, and `''\` - escapes any other character. - - Indented strings are primarily useful in that they allow multi-line - string literals to follow the indentation of the enclosing Nix - expression, and that less escaping is typically necessary for - strings representing languages such as shell scripts and - configuration files because `''` is much less common than `"`. - Example: - - ```nix - stdenv.mkDerivation { - ... - postInstall = - '' - mkdir $out/bin $out/etc - cp foo $out/bin - echo "Hello World" > $out/etc/foo.conf - ${if enableBar then "cp bar $out/bin" else ""} - ''; - ... - } - ``` - - Finally, as a convenience, *URIs* as defined in appendix B of - [RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as - is*, without quotes. For instance, the string - `"http://example.org/foo.tar.bz2"` can also be written as - `http://example.org/foo.tar.bz2`. - -- Number - - Numbers, which can be *integers* (like `123`) or *floating point* - (like `123.43` or `.27e13`). - - See [arithmetic] and [comparison] operators for semantics. - - [arithmetic]: ./operators.md#arithmetic - [comparison]: ./operators.md#comparison - -- Path - - *Paths*, e.g., `/bin/sh` or `./builder.sh`. A path must contain at - least one slash to be recognised as such. For instance, `builder.sh` - is not a path: it's parsed as an expression that selects the - attribute `sh` from the variable `builder`. If the file name is - relative, i.e., if it does not begin with a slash, it is made - absolute at parse time relative to the directory of the Nix - expression that contained it. For instance, if a Nix expression in - `/foo/bar/bla.nix` refers to `../xyzzy/fnord.nix`, the absolute path - is `/foo/xyzzy/fnord.nix`. - - If the first component of a path is a `~`, it is interpreted as if - the rest of the path were relative to the user's home directory. - e.g. `~/foo` would be equivalent to `/home/edolstra/foo` for a user - whose home directory is `/home/edolstra`. - - For instance, evaluating `"${./foo.txt}"` will cause `foo.txt` in the current directory to be copied into the Nix store and result in the string `"/nix/store/-foo.txt"`. - - Note that the Nix language assumes that all input files will remain _unchanged_ while evaluating a Nix expression. - For example, assume you used a file path in an interpolated string during a `nix repl` session. - Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new [store path], since Nix might not re-read the file contents. - - [store path]: ../glossary.md#gloss-store-path - - Paths can include [string interpolation] and can themselves be [interpolated in other expressions]. - - [interpolated in other expressions]: ./string-interpolation.md#interpolated-expressions - - At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path. - - `a.${foo}/b.${bar}` is a syntactically valid division operation. - `./a.${foo}/b.${bar}` is a path. - - [Lookup paths](./constructs/lookup-path.md) such as `` resolve to path values. - -- Boolean - - *Booleans* with values `true` and `false`. - -- Null - - The null value, denoted as `null`. - -## List - -Lists are formed by enclosing a whitespace-separated list of values -between square brackets. For example, - -```nix -[ 123 ./foo.nix "abc" (f { x = y; }) ] -``` - -defines a list of four elements, the last being the result of a call to -the function `f`. Note that function calls have to be enclosed in -parentheses. If they had been omitted, e.g., - -```nix -[ 123 ./foo.nix "abc" f { x = y; } ] -``` - -the result would be a list of five elements, the fourth one being a -function and the fifth being a set. - -Note that lists are only lazy in values, and they are strict in length. - -Elements in a list can be accessed using [`builtins.elemAt`](./builtins.md#builtins-elemAt). - -## Attribute Set - -An attribute set is a collection of name-value-pairs (called *attributes*) enclosed in curly brackets (`{ }`). - -An attribute name can be an identifier or a [string](#string). -An identifier must start with a letter (`a-z`, `A-Z`) or underscore (`_`), and can otherwise contain letters (`a-z`, `A-Z`), numbers (`0-9`), underscores (`_`), apostrophes (`'`), or dashes (`-`). - -> **Syntax** -> -> *name* = *identifier* | *string* \ -> *identifier* ~ `[a-zA-Z_][a-zA-Z0-9_'-]*` - -Names and values are separated by an equal sign (`=`). -Each value is an arbitrary expression terminated by a semicolon (`;`). - -> **Syntax** -> -> *attrset* = `{` [ *name* `=` *expr* `;` ]... `}` - -Attributes can appear in any order. -An attribute name may only occur once. - -Example: - -```nix -{ - x = 123; - text = "Hello"; - y = f { bla = 456; }; -} -``` - -This defines a set with attributes named `x`, `text`, `y`. - -Attributes can be accessed with the [`.` operator](./operators.md#attribute-selection). - -Example: - -```nix -{ a = "Foo"; b = "Bar"; }.a -``` - -This evaluates to `"Foo"`. - -It is possible to provide a default value in an attribute selection using the `or` keyword. - -Example: - -```nix -{ a = "Foo"; b = "Bar"; }.c or "Xyzzy" -``` - -```nix -{ a = "Foo"; b = "Bar"; }.c.d.e.f.g or "Xyzzy" -``` - -will both evaluate to `"Xyzzy"` because there is no `c` attribute in the set. - -You can use arbitrary double-quoted strings as attribute names: - -```nix -{ "$!@#?" = 123; }."$!@#?" -``` - -```nix -let bar = "bar"; in -{ "foo ${bar}" = 123; }."foo ${bar}" -``` - -Both will evaluate to `123`. - -Attribute names support [string interpolation]: - -```nix -let bar = "foo"; in -{ foo = 123; }.${bar} -``` - -```nix -let bar = "foo"; in -{ ${bar} = 123; }.foo -``` - -Both will evaluate to `123`. - -In the special case where an attribute name inside of a set declaration -evaluates to `null` (which is normally an error, as `null` cannot be coerced to -a string), that attribute is simply not added to the set: - -```nix -{ ${if foo then "bar" else null} = true; } -``` - -This will evaluate to `{}` if `foo` evaluates to `false`. - -A set that has a `__functor` attribute whose value is callable (i.e. is -itself a function or a set with a `__functor` attribute whose value is -callable) can be applied as if it were a function, with the set itself -passed in first , e.g., - -```nix -let add = { __functor = self: x: x + self.x; }; - inc = add // { x = 1; }; -in inc 1 -``` - -evaluates to `2`. This can be used to attach metadata to a function -without the caller needing to treat it specially, or to implement a form -of object-oriented programming, for example. diff --git a/doc/manual/src/language/variables.md b/doc/manual/src/language/variables.md new file mode 100644 index 000000000..af6aff8a2 --- /dev/null +++ b/doc/manual/src/language/variables.md @@ -0,0 +1,10 @@ +# Variables + +A *variable* is an [identifier](identifiers.md) used as an expression. + +> **Syntax** +> +> *expression* → *identifier* + +A variable must have the same name as a definition in the [scope](./scope.md) that encloses it. +The value of a variable is the value of the corresponding expression in the enclosing scope. diff --git a/doc/manual/src/package-management/copy-closure.md b/doc/manual/src/package-management/copy-closure.md deleted file mode 100644 index 14326298b..000000000 --- a/doc/manual/src/package-management/copy-closure.md +++ /dev/null @@ -1,34 +0,0 @@ -# Copying Closures via SSH - -The command `nix-copy-closure` copies a Nix store path along with all -its dependencies to or from another machine via the SSH protocol. It -doesn’t copy store paths that are already present on the target machine. -For example, the following command copies Firefox with all its -dependencies: - - $ nix-copy-closure --to alice@itchy.example.org $(type -p firefox) - -See the [manpage for `nix-copy-closure`](../command-ref/nix-copy-closure.md) for details. - -With `nix-store ---export` and `nix-store --import` you can write the closure of a store -path (that is, the path and all its dependencies) to a file, and then -unpack that file into another Nix store. For example, - - $ nix-store --export $(nix-store --query --requisites $(type -p firefox)) > firefox.closure - -writes the closure of Firefox to a file. You can then copy this file to -another machine and install the closure: - - $ nix-store --import < firefox.closure - -Any store paths in the closure that are already present in the target -store are ignored. It is also possible to pipe the export into another -command, e.g. to copy and install a closure directly to/on another -machine: - - $ nix-store --export $(nix-store --query --requisites $(type -p firefox)) | bzip2 | \ - ssh alice@itchy.example.org "bunzip2 | nix-store --import" - -However, `nix-copy-closure` is generally more efficient because it only -copies paths that are not already present in the target Nix store. diff --git a/doc/manual/src/protocols/derivation-aterm.md b/doc/manual/src/protocols/derivation-aterm.md index e58b602a3..1ba757ae0 100644 --- a/doc/manual/src/protocols/derivation-aterm.md +++ b/doc/manual/src/protocols/derivation-aterm.md @@ -14,6 +14,6 @@ Derivations are serialised in one of the following formats: DrvWithVersion(, ...) ``` - The only `version-string`s that are in use today are for [experimental features](@docroot@/contributing/experimental-features.md): + The only `version-string`s that are in use today are for [experimental features](@docroot@/development/experimental-features.md): - - `"xp-dyn-drv"` for the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. + - `"xp-dyn-drv"` for the [`dynamic-derivations`](@docroot@/development/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. diff --git a/doc/manual/src/protocols/json/derivation.md b/doc/manual/src/protocols/json/derivation.md index 649d543cc..2f85340d6 100644 --- a/doc/manual/src/protocols/json/derivation.md +++ b/doc/manual/src/protocols/json/derivation.md @@ -3,7 +3,7 @@ > **Warning** > > This JSON format is currently -> [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) +> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) > and subject to change. The JSON serialization of a @@ -18,10 +18,30 @@ is a JSON object with the following fields: Information about the output paths of the derivation. This is a JSON object with one member per output, where the key is the output name and the value is a JSON object with these fields: - * `path`: The output path. + * `path`: + The output path, if it is known in advanced. + Otherwise, `null`. + + + * `method`: + For an output which will be [content addresed], a string representing the [method](@docroot@/store/store-object/content-address.md) of content addressing that is chosen. + Valid method strings are: + + - [`flat`](@docroot@/store/store-object/content-address.md#method-flat) + - [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive) + - [`text`](@docroot@/store/store-object/content-address.md#method-text) + - [`git`](@docroot@/store/store-object/content-address.md#method-git) + + Otherwise, `null`. * `hashAlgo`: - For fixed-output derivations, the hashing algorithm (e.g. `sha256`), optionally prefixed by `r:` if `hash` denotes a NAR hash rather than a flat file hash. + For an output which will be [content addresed], the name of the hash algorithm used. + Valid algorithm strings are: + + - `md5` + - `sha1` + - `sha256` + - `sha512` * `hash`: For fixed-output derivations, the expected content hash in base-16. @@ -32,7 +52,8 @@ is a JSON object with the following fields: > "outputs": { > "out": { > "path": "/nix/store/2543j7c6jn75blc3drf4g5vhb1rhdq29-source", - > "hashAlgo": "r:sha256", + > "method": "nar", + > "hashAlgo": "sha256", > "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62" > } > } diff --git a/doc/manual/src/protocols/json/store-object-info.md b/doc/manual/src/protocols/json/store-object-info.md index ba4ab098f..6b4f48437 100644 --- a/doc/manual/src/protocols/json/store-object-info.md +++ b/doc/manual/src/protocols/json/store-object-info.md @@ -3,7 +3,7 @@ > **Warning** > > This JSON format is currently -> [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) +> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) > and subject to change. Info about a [store object]. @@ -24,41 +24,45 @@ Info about a [store object]. An array of [store paths][store path], possibly including this one. -* `ca` (optional): +* `ca`: - Content address of this store object's file system object, used to compute its store path. + If the store object is [content-addressed], + this is the content address of this store object's file system object, used to compute its store path. + Otherwise (i.e. if it is [input-addressed]), this is `null`. -[store path]: @docroot@/glossary.md#gloss-store-path +[store path]: @docroot@/store/store-path.md [file system object]: @docroot@/store/file-system-object.md -[Nix Archive]: @docroot@/glossary.md#gloss-nar +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive ## Impure fields These are not intrinsic properties of the store object. In other words, the same store object residing in different store could have different values for these properties. -* `deriver` (optional): +* `deriver`: - The path to the [derivation] from which this store object is produced. + If known, the path to the [derivation] from which this store object was produced. + Otherwise `null`. [derivation]: @docroot@/glossary.md#gloss-store-derivation * `registrationTime` (optional): - When this derivation was added to the store. + If known, when this derivation was added to the store. + Otherwise `null`. -* `ultimate` (optional): +* `ultimate`: Whether this store object is trusted because we built it ourselves, rather than substituted a build product from elsewhere. -* `signatures` (optional): +* `signatures`: Signatures claiming that this store object is what it claims to be. Not relevant for [content-addressed] store objects, but useful for [input-addressed] store objects. - [content-addressed]: @docroot@/glossary.md#gloss-content-addressed-store-object - [input-addressed]: @docroot@/glossary.md#gloss-input-addressed-store-object +[content-addressed]: @docroot@/store/store-object/content-address.md +[input-addressed]: @docroot@/glossary.md#gloss-input-addressed-store-object ### `.narinfo` extra fields @@ -83,7 +87,7 @@ This information is not intrinsic to the store object, but about how it is store ## Computed closure fields -These fields are not stored at all, but computed by traverising the other other fields across all the store objects in a [closure]. +These fields are not stored at all, but computed by traversing the other fields across all the store objects in a [closure]. * `closureSize`: diff --git a/doc/manual/src/protocols/nix-archive.md b/doc/manual/src/protocols/nix-archive.md new file mode 100644 index 000000000..640b527f1 --- /dev/null +++ b/doc/manual/src/protocols/nix-archive.md @@ -0,0 +1,43 @@ +# Nix Archive (NAR) format + +This is the complete specification of the [Nix Archive] format. +The Nix Archive format closely follows the abstract specification of a [file system object] tree, +because it is designed to serialize exactly that data structure. + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#nix-archive +[file system object]: @docroot@/store/file-system-object.md + +The format of this specification is close to [Extended Backus–Naur form](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form), with the exception of the `str(..)` function / parameterized rule, which length-prefixes and pads strings. +This makes the resulting binary format easier to parse. + +Regular users do *not* need to know this information. +But for those interested in exactly how Nix works, e.g. if they are reimplementing it, this information can be useful. + +```ebnf +nar = str("nix-archive-1"), nar-obj; + +nar-obj = str("("), nar-obj-inner, str(")"); + +nar-obj-inner + = str("type"), str("regular") regular + | str("type"), str("symlink") symlink + | str("type"), str("directory") directory + ; + +regular = [ str("executable"), str("") ], str("contents"), str(contents); + +symlink = str("target"), str(target); + +(* side condition: directory entries must be ordered by their names *) +directory = { directory-entry }; + +directory-entry = str("entry"), str("("), str("name"), str(name), str("node"), nar-obj, str(")"); +``` + +The `str` function / parameterized rule is defined as follows: + +- `str(s)` = `int(|s|), pad(s);` + +- `int(n)` = the 64-bit little endian representation of the number `n` + +- `pad(s)` = the byte sequence `s`, padded with 0s to a multiple of 8 byte diff --git a/doc/manual/src/protocols/store-path.md b/doc/manual/src/protocols/store-path.md index 565c4fa75..52352d358 100644 --- a/doc/manual/src/protocols/store-path.md +++ b/doc/manual/src/protocols/store-path.md @@ -1,12 +1,14 @@ # Complete Store Path Calculation -This is the complete specification for how store paths are calculated. +This is the complete specification for how [store path]s are calculated. The format of this specification is close to [Extended Backus–Naur form](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form), but must deviate for a few things such as hash functions which we treat as bidirectional for specification purposes. Regular users do *not* need to know this information --- store paths can be treated as black boxes computed from the properties of the store objects they refer to. But for those interested in exactly how Nix works, e.g. if they are reimplementing it, this information can be useful. +[store path](@docroot@/store/store-path.md) + ## Store path proper ```ebnf @@ -34,18 +36,23 @@ where - `type` = one of: - ```ebnf - | "text" ( ":" store-path )* + | "text" { ":" store-path } ``` - for encoded derivations written to the store. + This is for the + ["Text"](@docroot@/store/store-object/content-address.md#method-text) + method of content addressing store objects. The optional trailing store paths are the references of the store object. - ```ebnf - | "source" ( ":" store-path )* + | "source" { ":" store-path } [ ":self" ] ``` - For paths copied to the store and hashed via a [Nix Archive (NAR)] and [SHA-256][sha-256]. - Just like in the text case, we can have the store objects referenced by their paths. + This is for the + ["Nix Archive"](@docroot@/store/store-object/content-address.md#method-nix-archive) + method of content addressing store objects, + if the hash algorithm is [SHA-256]. + Just like in the "Text" case, we can have the store objects referenced by their paths. Additionally, we can have an optional `:self` label to denote self reference. - ```ebnf @@ -53,8 +60,12 @@ where ``` For either the outputs built from derivations, - paths copied to the store hashed that area single file hashed directly, or the via a hash algorithm other than [SHA-256][sha-256]. - (in that case "source" is used; this is only necessary for compatibility). + or content-addressed store objects that are not using one of the two above cases. + To be explicit about the latter, that is currently these methods: + + - ["Flat"](@docroot@/store/store-object/content-address.md#method-flat) + - ["Git"](@docroot@/store/store-object/content-address.md#method-git) + - ["Nix Archive"](@docroot@/store/store-object/content-address.md#method-nix-archive) if the hash algorithm is not [SHA-256]. `id` is the name of the output (usually, "out"). For content-addressed store objects, `id`, is always "out". @@ -113,8 +124,8 @@ where Note that `id` = `"out"`, regardless of the name part of the store path. Also note that NAR + SHA-256 must not use this case, and instead must use the `type` = `"source:" ...` case. -[Nix Archive (NAR)]: @docroot@/glossary.md#gloss-NAR -[sha-256]: https://en.m.wikipedia.org/wiki/SHA-256 +[Nix Archive (NAR)]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive +[SHA-256]: https://en.m.wikipedia.org/wiki/SHA-256 ### Historical Note diff --git a/doc/manual/src/protocols/tarball-fetcher.md b/doc/manual/src/protocols/tarball-fetcher.md index 274fa6d63..5cff05d66 100644 --- a/doc/manual/src/protocols/tarball-fetcher.md +++ b/doc/manual/src/protocols/tarball-fetcher.md @@ -22,7 +22,7 @@ Link: ; rel="immutable" *flakeref* must be a tarball flakeref. It can contain the tarball flake attributes `narHash`, `rev`, `revCount` and `lastModified`. If `narHash` is included, its -value must be the NAR hash of the unpacked tarball (as computed via +value must be the [NAR hash][Nix Archive] of the unpacked tarball (as computed via `nix hash path`). Nix checks the contents of the returned tarball against the `narHash` attribute. The `rev` and `revCount` attributes are useful when the tarball flake is a mirror of a fetcher type that @@ -40,3 +40,31 @@ Link: ///archive/.tar.gz +``` + +> **Example** +> +> +> ```nix +> # flake.nix +> { +> inputs = { +> foo.url = "https://gitea.example.org/some-person/some-flake/archive/main.tar.gz"; +> bar.url = "https://gitea.example.org/some-other-person/other-flake/archive/442793d9ec0584f6a6e82fa253850c8085bb150a.tar.gz"; +> qux = { +> url = "https://forgejo.example.org/another-person/some-non-flake-repo/archive/development.tar.gz"; +> flake = false; +> }; +> }; +> outputs = { foo, bar, qux }: { /* ... */ }; +> } +``` + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive diff --git a/doc/manual/src/quick-start.md b/doc/manual/src/quick-start.md index 75853ced7..9eb7a3265 100644 --- a/doc/manual/src/quick-start.md +++ b/doc/manual/src/quick-start.md @@ -34,7 +34,7 @@ For more in-depth information you are kindly referred to subsequent chapters. lolcat: command not found ``` -1. Search for more packages on to try them out. +1. Search for more packages on [search.nixos.org](https://search.nixos.org/) to try them out. 1. Free up storage space: diff --git a/doc/manual/src/release-notes/index.md b/doc/manual/src/release-notes/index.md index cc805e631..d4e6292a6 100644 --- a/doc/manual/src/release-notes/index.md +++ b/doc/manual/src/release-notes/index.md @@ -1,12 +1,13 @@ # Nix Release Notes +The Nix release cycle is calendar-based as follows: + Nix has a release cycle of roughly 6 weeks. Notable changes and additions are announced in the release notes for each version. -Bugfixes can be backported on request to previous Nix releases. -We typically backport only as far back as the Nix version used in the latest NixOS release, which is announced in the [NixOS release notes](https://nixos.org/manual/nixos/stable/release-notes.html#ch-release-notes). - -Backports never skip releases. -If a feature is backported to version `x.y`, it must also be available in version `x.(y+1)`. -This ensures that upgrading from an older version with backports is still safe and no backported functionality will go missing. +The supported Nix versions are: +- The latest release +- The version used in the stable NixOS release, which is announced in the [NixOS release notes](https://nixos.org/manual/nixos/stable/release-notes.html#ch-release-notes). +Bugfixes and security issues are backported to every supported version. +Patch releases are published as needed. diff --git a/doc/manual/src/release-notes/rl-2.15.md b/doc/manual/src/release-notes/rl-2.15.md index 133121999..e7e52631b 100644 --- a/doc/manual/src/release-notes/rl-2.15.md +++ b/doc/manual/src/release-notes/rl-2.15.md @@ -11,7 +11,7 @@ As the choice of hash formats is no longer binary, the `--base16` flag is also added to explicitly specify the Base16 format, which is still the default. -* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](../glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents. +* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](@docroot@/glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents. The new `^` syntax for store paths introduced in Nix 2.13 allows explicitly referencing output paths of a derivation. Using this is better and more clear than relying on the now-removed `.drv` special handling. diff --git a/doc/manual/src/release-notes/rl-2.18.md b/doc/manual/src/release-notes/rl-2.18.md index 4bbc52b50..eb26fc9e7 100644 --- a/doc/manual/src/release-notes/rl-2.18.md +++ b/doc/manual/src/release-notes/rl-2.18.md @@ -13,7 +13,7 @@ - The `discard-references` feature has been stabilized. This means that the - [unsafeDiscardReferences](@docroot@/contributing/experimental-features.md#xp-feature-discard-references) + [unsafeDiscardReferences](@docroot@/development/experimental-features.md#xp-feature-discard-references) attribute is no longer guarded by an experimental flag and can be used freely. @@ -21,7 +21,7 @@ This only affects `nix-build --json` when "building" non-derivation things like fetched sources, which is a no-op. - A new builtin [`outputOf`](@docroot@/language/builtins.md#builtins-outputOf) has been added. - It is part of the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. + It is part of the [`dynamic-derivations`](@docroot@/development/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. - Flake follow paths at depths greater than 2 are now handled correctly, preventing "follows a non-existent input" errors. diff --git a/doc/manual/src/release-notes/rl-2.19.md b/doc/manual/src/release-notes/rl-2.19.md index ba6eb9c64..e2e2f85cc 100644 --- a/doc/manual/src/release-notes/rl-2.19.md +++ b/doc/manual/src/release-notes/rl-2.19.md @@ -17,8 +17,8 @@ - `nix-shell` shebang lines now support single-quoted arguments. -- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). - This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). +- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/development/experimental-features.md#xp-fetch-tree). + This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/development/experimental-features.md#xp-fetch-tree). - The interface for creating and updating lock files has been overhauled: @@ -33,7 +33,7 @@ - The flake-specific flags `--recreate-lock-file` and `--update-input` have been removed from all commands operating on installables. They are superceded by `nix flake update`. -- Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). +- Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches). - [`nix path-info --json`](@docroot@/command-ref/new-cli/nix3-path-info.md) (experimental) now returns a JSON map rather than JSON list. diff --git a/doc/manual/src/release-notes/rl-2.20.md b/doc/manual/src/release-notes/rl-2.20.md index 8ede168a4..eb724f600 100644 --- a/doc/manual/src/release-notes/rl-2.20.md +++ b/doc/manual/src/release-notes/rl-2.20.md @@ -200,3 +200,9 @@ while performing various operations (including `nix develop`, `nix flake update`, and so on). With several fixes to Nix's signal handlers, Nix commands will now exit quickly after Ctrl-C is pressed. + +- `nix copy` to a `ssh-ng` store now needs `--substitute-on-destination` (a.k.a. `-s`) + in order to substitute paths on the remote store instead of copying them. + The behavior is consistent with `nix copy` to a different kind of remote store. + Previously this behavior was controlled by the + `builders-use-substitutes` setting and `--substitute-on-destination` was ignored. diff --git a/doc/manual/src/release-notes/rl-2.21.md b/doc/manual/src/release-notes/rl-2.21.md new file mode 100644 index 000000000..75114f117 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.21.md @@ -0,0 +1,302 @@ +# Release 2.21.0 (2024-03-11) + +- Fix a fixed-output derivation sandbox escape (CVE-2024-27297) + + Cooperating Nix derivations could send file descriptors to files in the Nix + store to each other via Unix domain sockets in the abstract namespace. This + allowed one derivation to modify the output of the other derivation, after Nix + has registered the path as "valid" and immutable in the Nix database. + In particular, this allowed the output of fixed-output derivations to be + modified from their expected content. + + This isn't the case any more. + +- CLI options `--arg-from-file` and `--arg-from-stdin` [#10122](https://github.com/NixOS/nix/pull/10122) + + The new CLI option `--arg-from-file` *name* *path* passes the contents + of file *path* as a string value via the function argument *name* to a + Nix expression. Similarly, the new option `--arg-from-stdin` *name* + reads the contents of the string from standard input. + +- Concise error printing in `nix repl` [#9928](https://github.com/NixOS/nix/pull/9928) + + Previously, if an element of a list or attribute set threw an error while + evaluating, `nix repl` would print the entire error (including source location + information) inline. This output was clumsy and difficult to parse: + + ``` + nix-repl> { err = builtins.throw "uh oh!"; } + { err = «error: + … while calling the 'throw' builtin + at «string»:1:9: + 1| { err = builtins.throw "uh oh!"; } + | ^ + + error: uh oh!»; } + ``` + + Now, only the error message is displayed, making the output much more readable. + ``` + nix-repl> { err = builtins.throw "uh oh!"; } + { err = «error: uh oh!»; } + ``` + + However, if the whole expression being evaluated throws an error, source + locations and (if applicable) a stack trace are printed, just like you'd expect: + + ``` + nix-repl> builtins.throw "uh oh!" + error: + … while calling the 'throw' builtin + at «string»:1:1: + 1| builtins.throw "uh oh!" + | ^ + + error: uh oh! + ``` + +- `--debugger` can now access bindings from `let` expressions [#8827](https://github.com/NixOS/nix/issues/8827) [#9918](https://github.com/NixOS/nix/pull/9918) + + Breakpoints and errors in the bindings of a `let` expression can now access + those bindings in the debugger. Previously, only the body of `let` expressions + could access those bindings. + +- Enter the `--debugger` when `builtins.trace` is called if `debugger-on-trace` is set [#9914](https://github.com/NixOS/nix/pull/9914) + + If the `debugger-on-trace` option is set and `--debugger` is given, + `builtins.trace` calls will behave similarly to `builtins.break` and will enter + the debug REPL. This is useful for determining where warnings are being emitted + from. + +- Debugger prints source position information [#9913](https://github.com/NixOS/nix/pull/9913) + + The `--debugger` now prints source location information, instead of the + pointers of source location information. Before: + + ``` + nix-repl> :bt + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + 0x600001522598 + ``` + + After: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + /nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27 + + 131| + 132| bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs; + | ^ + 133| in + ``` + +- The `--debugger` will start more reliably in `let` expressions and function calls [#6649](https://github.com/NixOS/nix/issues/6649) [#9917](https://github.com/NixOS/nix/pull/9917) + + Previously, if you attempted to evaluate this file with the debugger: + + ```nix + let + a = builtins.trace "before inner break" ( + builtins.break "hello" + ); + b = builtins.trace "before outer break" ( + builtins.break a + ); + in + b + ``` + + Nix would correctly enter the debugger at `builtins.break a`, but if you asked + it to `:continue`, it would skip over the `builtins.break "hello"` expression + entirely. + + Now, Nix will correctly enter the debugger at both breakpoints. + +- Nested debuggers are no longer supported [#9920](https://github.com/NixOS/nix/pull/9920) + + Previously, evaluating an expression that throws an error in the debugger would + enter a second, nested debugger: + + ``` + nix-repl> builtins.throw "what" + error: what + + + Starting REPL to allow you to inspect the current state of the evaluator. + + Welcome to Nix 2.18.1. Type :? for help. + + nix-repl> + ``` + + Now, it just prints the error message like `nix repl`: + + ``` + nix-repl> builtins.throw "what" + error: + … while calling the 'throw' builtin + at «string»:1:1: + 1| builtins.throw "what" + | ^ + + error: what + ``` + +- Consistent order of function arguments in printed expressions [#9874](https://github.com/NixOS/nix/pull/9874) + + Function arguments are now printed in lexicographic order rather than the internal, creation-time based symbol order. + +- Fix duplicate attribute error positions for `inherit` [#9874](https://github.com/NixOS/nix/pull/9874) + + When an `inherit` caused a duplicate attribute error the position of the error was not reported correctly, placing the error with the inherit itself or at the start of the bindings block instead of the offending attribute name. + +- `inherit (x) ...` evaluates `x` only once [#9847](https://github.com/NixOS/nix/pull/9847) + + `inherit (x) a b ...` now evaluates the expression `x` only once for all inherited attributes rather than once for each inherited attribute. + This does not usually have a measurable impact, but side-effects (such as `builtins.trace`) would be duplicated and expensive expressions (such as derivations) could cause a measurable slowdown. + +- Store paths are allowed to start with `.` [#912](https://github.com/NixOS/nix/issues/912) [#9091](https://github.com/NixOS/nix/pull/9091) [#9095](https://github.com/NixOS/nix/pull/9095) [#9120](https://github.com/NixOS/nix/pull/9120) [#9121](https://github.com/NixOS/nix/pull/9121) [#9122](https://github.com/NixOS/nix/pull/9122) [#9130](https://github.com/NixOS/nix/pull/9130) [#9219](https://github.com/NixOS/nix/pull/9219) [#9224](https://github.com/NixOS/nix/pull/9224) [#9867](https://github.com/NixOS/nix/pull/9867) + + Leading periods were allowed by accident in Nix 2.4. The Nix team has considered this to be a bug, but this behavior has since been relied on by users, leading to unnecessary difficulties. + From now on, leading periods are supported. The names `.` and `..` are disallowed, as well as those starting with `.-` or `..-`. + + Nix versions that denied leading periods are documented [in the issue](https://github.com/NixOS/nix/issues/912#issuecomment-1919583286). + +- `nix repl` pretty-prints values [#9931](https://github.com/NixOS/nix/pull/9931) + + `nix repl` will now pretty-print values: + + ``` + { + attrs = { + a = { + b = { + c = { }; + }; + }; + }; + list = [ 1 ]; + list' = [ + 1 + 2 + 3 + ]; + } + ``` + +- Introduction of `--regex` and `--all` in `nix profile remove` and `nix profile upgrade` [#10166](https://github.com/NixOS/nix/pull/10166) + + Previously the command-line arguments for `nix profile remove` and `nix profile upgrade` matched the package entries using regular expression. + For instance: + + ``` + nix profile remove '.*vim.*' + ``` + + This would remove all packages that contain `vim` in their name. + + In most cases, only singular package names were used to remove and upgrade packages. Mixing this with regular expressions sometimes lead to unintended behavior. For instance, `python3.1` could match `python311`. + + To avoid unintended behavior, the arguments are now only matching exact names. + + Matching using regular expressions is still possible by using the new `--regex` flag: + + ``` + nix profile remove --regex '.*vim.*' + ``` + + One of the most useful cases for using regular expressions was to upgrade all packages. This was previously accomplished by: + + ``` + nix profile upgrade '.*' + ``` + + With the introduction of the `--all` flag, this now becomes more straightforward: + + ``` + nix profile upgrade --all + ``` + +- Visual clutter in `--debugger` is reduced [#9919](https://github.com/NixOS/nix/pull/9919) + + Before: + ``` + info: breakpoint reached + + + Starting REPL to allow you to inspect the current state of the evaluator. + + Welcome to Nix 2.20.0pre20231222_dirty. Type :? for help. + + nix-repl> :continue + error: uh oh + + + Starting REPL to allow you to inspect the current state of the evaluator. + + Welcome to Nix 2.20.0pre20231222_dirty. Type :? for help. + + nix-repl> + ``` + + After: + + ``` + info: breakpoint reached + + Nix 2.20.0pre20231222_dirty debugger + Type :? for help. + nix-repl> :continue + error: uh oh + + nix-repl> + ``` + +- Cycle detection in `nix repl` is simpler and more reliable [#8672](https://github.com/NixOS/nix/issues/8672) [#9926](https://github.com/NixOS/nix/pull/9926) + + The cycle detection in `nix repl`, `nix eval`, `builtins.trace`, and everywhere + else values are printed is now simpler and matches the cycle detection in + `nix-instantiate --eval` output. + + Before: + + ``` + nix eval --expr 'let self = { inherit self; }; in self' + { self = { self = «repeated»; }; } + ``` + + After: + + ``` + { self = «repeated»; } + ``` + +- In the debugger, `while evaluating the attribute` errors now include position information [#9915](https://github.com/NixOS/nix/pull/9915) + + Before: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + 0x600001522598 + ``` + + After: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + /nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27 + + 131| + 132| bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs; + | ^ + 133| in + ``` + +- Stack size is increased on macOS [#9860](https://github.com/NixOS/nix/pull/9860) + + Previously, Nix would set the stack size to 64MiB on Linux, but would leave the + stack size set to the default (approximately 8KiB) on macOS. Now, the stack + size is correctly set to 64MiB on macOS as well, which should reduce stack + overflow segfaults in deeply-recursive Nix expressions. + diff --git a/doc/manual/src/release-notes/rl-2.22.md b/doc/manual/src/release-notes/rl-2.22.md new file mode 100644 index 000000000..c78d3d692 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.22.md @@ -0,0 +1,21 @@ +# Release 2.22.0 (2024-04-23) + +### Significant changes + +- Remove experimental repl-flake [#10103](https://github.com/NixOS/nix/issues/10103) [#10299](https://github.com/NixOS/nix/pull/10299) + + The `repl-flake` experimental feature has been removed. The `nix repl` command now works like the rest of the new CLI in that `nix repl {path}` now tries to load a flake at `{path}` (or fails if the `flakes` experimental feature isn't enabled). + +### Other changes + +- `nix eval` prints derivations as `.drv` paths [#10200](https://github.com/NixOS/nix/pull/10200) + + `nix eval` will now print derivations as their `.drv` paths, rather than as + attribute sets. This makes commands like `nix eval nixpkgs#bash` terminate + instead of infinitely looping into recursive self-referential attributes: + + ```ShellSession + $ nix eval nixpkgs#bash + «derivation /nix/store/m32cbgbd598f4w299g0hwyv7gbw6rqcg-bash-5.2p26.drv» + ``` + diff --git a/doc/manual/src/release-notes/rl-2.23.md b/doc/manual/src/release-notes/rl-2.23.md new file mode 100644 index 000000000..ac842fdc0 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.23.md @@ -0,0 +1,102 @@ +# Release 2.23.0 (2024-06-03) + +- New builtin: `builtins.warn` [#306026](https://github.com/NixOS/nix/issues/306026) [#10592](https://github.com/NixOS/nix/pull/10592) + + `builtins.warn` behaves like `builtins.trace "warning: ${msg}"`, has an accurate log level, and is controlled by the options + [`debugger-on-trace`](@docroot@/command-ref/conf-file.md#conf-debugger-on-trace), + [`debugger-on-warn`](@docroot@/command-ref/conf-file.md#conf-debugger-on-warn) and + [`abort-on-warn`](@docroot@/command-ref/conf-file.md#conf-abort-on-warn). + +- Make `nix build --keep-going` consistent with `nix-build --keep-going` + + This means that if e.g. multiple fixed-output derivations fail to + build, all hash mismatches are displayed. + +- Modify `nix derivation {add,show}` JSON format [#9866](https://github.com/NixOS/nix/issues/9866) [#10722](https://github.com/NixOS/nix/pull/10722) + + The JSON format for derivations has been slightly revised to better conform to our [JSON guidelines](@docroot@/development/cli-guideline.md#returning-future-proof-json). + In particular, the hash algorithm and content addressing method of content-addresed derivation outputs are now separated into two fields `hashAlgo` and `method`, + rather than one field with an arcane `:`-separated format. + + This JSON format is only used by the experimental `nix derivation` family of commands, at this time. + Future revisions are expected as the JSON format is still not entirely in compliance even after these changes. + +- Warn on unknown settings anywhere in the command line [#10701](https://github.com/NixOS/nix/pull/10701) + + All `nix` commands will now properly warn when an unknown option is specified anywhere in the command line. + + Before: + + ```console + $ nix-instantiate --option foobar baz --expr '{}' + warning: unknown setting 'foobar' + $ nix-instantiate '{}' --option foobar baz --expr + $ nix eval --expr '{}' --option foobar baz + { } + ``` + + After: + + ```console + $ nix-instantiate --option foobar baz --expr '{}' + warning: unknown setting 'foobar' + $ nix-instantiate '{}' --option foobar baz --expr + warning: unknown setting 'foobar' + $ nix eval --expr '{}' --option foobar baz + warning: unknown setting 'foobar' + { } + ``` + +- `nix env shell` is the new `nix shell`, and `nix shell` remains an accepted alias [#10504](https://github.com/NixOS/nix/issues/10504) [#10807](https://github.com/NixOS/nix/pull/10807) + + This is part of an effort to bring more structure to the CLI subcommands. + + `nix env` will be about the process environment. + Future commands may include `nix env run` and `nix env print-env`. + + It is also somewhat analogous to the [planned](https://github.com/NixOS/nix/issues/10504) `nix dev shell` (currently `nix develop`), which is less about environment variables, and more about running a development shell, which is a more powerful command, but also requires more setup. + +- Flake operations that expect derivations now print the failing value and its type [#10778](https://github.com/NixOS/nix/pull/10778) + + In errors like `flake output attribute 'nixosConfigurations.yuki.config' is not a derivation or path`, the message now includes the failing value and type. + + Before: + + ``` + error: flake output attribute 'nixosConfigurations.yuki.config' is not a derivation or path + ```` + + After: + + ``` + error: expected flake output attribute 'nixosConfigurations.yuki.config' to be a derivation or path but found a set: { appstream = «thunk»; assertions = «thunk»; boot = { bcache = «thunk»; binfmt = «thunk»; binfmtMiscRegistrations = «thunk»; blacklistedKernelModules = «thunk»; bootMount = «thunk»; bootspec = «thunk»; cleanTmpDir = «thunk»; consoleLogLevel = «thunk»; «43 attributes elided» }; «48 attributes elided» } + ``` + +- `fetchTree` now fetches Git repositories shallowly by default [#10028](https://github.com/NixOS/nix/pull/10028) + + `builtins.fetchTree` now clones Git repositories shallowly by default, which reduces network traffic and disk usage significantly in many cases. + + Previously, the default behavior was to clone the full history of a specific tag or branch (e.g. `ref`) and only afterwards extract the files of one specific revision. + + From now on, the `ref` and `allRefs` arguments will be ignored, except if shallow cloning is disabled by setting `shallow = false`. + + The defaults for `builtins.fetchGit` remain unchanged. Here, shallow cloning has to be enabled manually by passing `shallow = true`. + +- Store object info JSON format now uses `null` rather than omitting fields [#9995](https://github.com/NixOS/nix/pull/9995) + + The [store object info JSON format](@docroot@/protocols/json/store-object-info.md), used for e.g. `nix path-info`, no longer omits fields to indicate absent information, but instead includes the fields with a `null` value. + For example, `"ca": null` is used to to indicate a store object that isn't content-addressed rather than omitting the `ca` field entirely. + This makes records of this sort more self-describing, and easier to consume programmatically. + + We will follow this design principle going forward; + the [JSON guidelines](@docroot@/development/json-guideline.md) in the contributing section have been updated accordingly. + +- Large path warnings [#10661](https://github.com/NixOS/nix/pull/10661) + + Nix can now warn when evaluation of a Nix expression causes a large + path to be copied to the Nix store. The threshold for this warning can + be configured using [the `warn-large-path-threshold` + setting](@docroot@/command-ref/conf-file.md#warn-large-path-threshold), + e.g. `--warn-large-path-threshold 100M` will warn about paths larger + than 100 MiB. + diff --git a/doc/manual/src/release-notes/rl-2.24.md b/doc/manual/src/release-notes/rl-2.24.md new file mode 100644 index 000000000..5bcc1d79c --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.24.md @@ -0,0 +1,324 @@ +# Release 2.24.0 (2024-07-31) + +### Significant changes + +- Harden user sandboxing + + The build directory has been hardened against interference with the outside world by nesting it inside another directory owned by (and only readable by) the daemon user. + + This is a low severity security fix, [CVE-2024-38531](https://www.cve.org/CVERecord?id=CVE-2024-38531). + + Credit: [**@alois31**](https://github.com/alois31), [**Linus Heckemann (@lheckemann)**](https://github.com/lheckemann) + Co-authors: [**@edolstra**](https://github.com/edolstra) + +- `nix-shell ` looks for `shell.nix` [#496](https://github.com/NixOS/nix/issues/496) [#2279](https://github.com/NixOS/nix/issues/2279) [#4529](https://github.com/NixOS/nix/issues/4529) [#5431](https://github.com/NixOS/nix/issues/5431) [#11053](https://github.com/NixOS/nix/issues/11053) [#11057](https://github.com/NixOS/nix/pull/11057) + + `nix-shell $x` now looks for `$x/shell.nix` when `$x` resolves to a directory. + + Although this might be seen as a breaking change, its primarily interactive usage makes it a minor issue. + This adjustment addresses a commonly reported problem. + + This also applies to `nix-shell` shebang scripts. Consider the following example: + + ```shell + #!/usr/bin/env nix-shell + #!nix-shell -i bash + ``` + + This will now load `shell.nix` from the script's directory, if it exists; `default.nix` otherwise. + + The old behavior can be opted into by setting the option [`nix-shell-always-looks-for-shell-nix`](@docroot@/command-ref/conf-file.md#conf-nix-shell-always-looks-for-shell-nix) to `false`. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + +- `nix-repl`'s `:doc` shows documentation comments [#3904](https://github.com/NixOS/nix/issues/3904) [#10771](https://github.com/NixOS/nix/issues/10771) [#1652](https://github.com/NixOS/nix/pull/1652) [#9054](https://github.com/NixOS/nix/pull/9054) [#11072](https://github.com/NixOS/nix/pull/11072) + + `nix repl` has a `:doc` command that previously only rendered documentation for internally defined functions. + This feature has been extended to also render function documentation comments, in accordance with [RFC 145]. + + Example: + + ``` + nix-repl> :doc lib.toFunction + Function toFunction + … defined at /home/user/h/nixpkgs/lib/trivial.nix:1072:5 + + Turns any non-callable values into constant functions. Returns + callable values as is. + + Inputs + + v + + : Any value + + Examples + + :::{.example} + + ## lib.trivial.toFunction usage example + + | nix-repl> lib.toFunction 1 2 + | 1 + | + | nix-repl> lib.toFunction (x: x + 1) 2 + | 3 + + ::: + ``` + + Known limitations: + - It does not render documentation for "formals", such as `{ /** the value to return */ x, ... }: x`. + - Some extensions to markdown are not yet supported, as you can see in the example above. + + We'd like to acknowledge [Yingchi Long (@inclyc)](https://github.com/inclyc) for proposing a proof of concept for this functionality in [#9054](https://github.com/NixOS/nix/pull/9054), as well as [@sternenseemann](https://github.com/sternenseemann) and [Johannes Kirschbauer (@hsjobeki)](https://github.com/hsjobeki) for their contributions, proposals, and their work on [RFC 145]. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + + [RFC 145]: https://github.com/NixOS/rfcs/pull/145 + +### Other changes + +- Solve `cached failure of attribute X` [#9165](https://github.com/NixOS/nix/issues/9165) [#10513](https://github.com/NixOS/nix/issues/10513) [#10564](https://github.com/NixOS/nix/pull/10564) + + This eliminates all "cached failure of attribute X" messages by forcing evaluation of the original value when needed to show the exception to the user. This enhancement improves error reporting by providing the underlying message and stack trace. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Run the flake regressions test suite [#10603](https://github.com/NixOS/nix/pull/10603) + + This update introduces a GitHub action to run a subset of the [flake regressions test suite](https://github.com/NixOS/flake-regressions), which includes 259 flakes with their expected evaluation results. Currently, the action runs the first 25 flakes due to the full test suite's extensive runtime. A manually triggered action may be implemented later to run the entire test suite. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Support unit prefixes in configuration settings [#10668](https://github.com/NixOS/nix/pull/10668) + + Configuration settings in Nix now support unit prefixes, allowing for more intuitive and readable configurations. For example, you can now specify [`--min-free 1G`](@docroot@/command-ref/opt-common.md#opt-min-free) to set the minimum free space to 1 gigabyte. + + This enhancement was extracted from [#7851](https://github.com/NixOS/nix/pull/7851) and is also useful for PR [#10661](https://github.com/NixOS/nix/pull/10661). + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- `nix build`: show all FOD errors with `--keep-going` [#10734](https://github.com/NixOS/nix/pull/10734) + + The [`nix build`](@docroot@/command-ref/new-cli/nix3-build.md) command has been updated to improve the behavior of the [`--keep-going`] flag. Now, when `--keep-going` is used, all hash-mismatch errors of failing fixed-output derivations (FODs) are displayed, similar to the behavior for other build failures. This enhancement ensures that all relevant build errors are shown, making it easier for users to update multiple derivations at once or to diagnose and fix issues. + + Author: [**Jörg Thalheim (@Mic92)**](https://github.com/Mic92), [**Maximilian Bosch (@Ma27)**](https://github.com/Ma27) + + [`--keep-going`](@docroot@/command-ref/opt-common.md#opt-keep-going) + +- Build with Meson [#2503](https://github.com/NixOS/nix/issues/2503) [#10378](https://github.com/NixOS/nix/pull/10378) [#10855](https://github.com/NixOS/nix/pull/10855) [#10904](https://github.com/NixOS/nix/pull/10904) [#10908](https://github.com/NixOS/nix/pull/10908) [#10914](https://github.com/NixOS/nix/pull/10914) [#10933](https://github.com/NixOS/nix/pull/10933) [#10936](https://github.com/NixOS/nix/pull/10936) [#10954](https://github.com/NixOS/nix/pull/10954) [#10955](https://github.com/NixOS/nix/pull/10955) [#10963](https://github.com/NixOS/nix/pull/10963) [#10967](https://github.com/NixOS/nix/pull/10967) [#10973](https://github.com/NixOS/nix/pull/10973) [#11034](https://github.com/NixOS/nix/pull/11034) [#11054](https://github.com/NixOS/nix/pull/11054) [#11055](https://github.com/NixOS/nix/pull/11055) [#11060](https://github.com/NixOS/nix/pull/11060) [#11064](https://github.com/NixOS/nix/pull/11064) [#11155](https://github.com/NixOS/nix/pull/11155) + + These changes aim to replace the use of autotools and `make` with Meson for building various components of Nix. Additionally, each library is built in its own derivation, leveraging Meson's "subprojects" feature to allow a single development shell for building all libraries while also supporting separate builds. This approach aims to improve productivity and build modularity, compared to both make and a monolithic Meson-based derivation. + + Special thanks to everyone who has contributed to the Meson port, particularly [**@p01arst0rm**](https://github.com/p01arst0rm) and [**@Qyriad**](https://github.com/Qyriad). + + Authors: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314), [**Tom Bereknyei**](https://github.com/tomberek), [**Théophane Hufschmitt (@thufschmitt)**](https://github.com/thufschmitt), [**Valentin Gagarin (@fricklerhandwerk)**](https://github.com/fricklerhandwerk), [**Robert Hensing (@roberth)**](https://github.com/roberth) + Co-authors: [**@p01arst0rm**](https://github.com/p01arst0rm), [**@Qyriad**](https://github.com/Qyriad) + +- Evaluation cache: fix cache regressions [#10570](https://github.com/NixOS/nix/issues/10570) [#11086](https://github.com/NixOS/nix/pull/11086) + + This update addresses two bugs in the evaluation cache system: + + 1. Regression in #10570: The evaluation cache was not being persisted in `nix develop`. + 2. Nix could sometimes try to commit the evaluation cache SQLite transaction without there being an active transaction, resulting in non-error errors being printed. + + Author: [**Lexi Mattick (@kognise)**](https://github.com/kognise) + +- Introduce `libnixflake` [#9063](https://github.com/NixOS/nix/pull/9063) + + A new library, `libnixflake`, has been introduced to better separate the Flakes layer within Nix. This change refactors the codebase to encapsulate Flakes-specific functionality within its own library. + + See the commits in the pull request for detailed changes, with the only significant code modifications happening in the initial commit. + + This change was alluded to in [RFC 134](https://github.com/nixos/rfcs/blob/master/rfcs/0134-nix-store-layer.md) and is a step towards a more modular and maintainable codebase. + + Author: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314) + +- CLI options `--arg-from-file` and `--arg-from-stdin` [#9913](https://github.com/NixOS/nix/pull/9913) + +- The `--debugger` now prints source location information, instead of the + pointers of source location information. Before: + + ``` + nix-repl> :bt + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + 0x600001522598 + ``` + + After: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + /nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27 + + 131| + 132| bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs; + | ^ + 133| in + ``` + +- Stop vendoring `toml11` + + We don't apply any patches to it, and vendoring it locks users into + bugs (it hasn't been updated since its introduction in late 2021). + + Author: [**Winter (@winterqt)**](https://github.com/winterqt) + +- Rename hash format `base32` to `nix32` [#8678](https://github.com/NixOS/nix/pull/8678) + + Hash format `base32` was renamed to `nix32` since it used a special nix-specific character set for + [Base32](https://en.wikipedia.org/wiki/Base32). + + **Deprecation**: Use `nix32` instead of `base32` as `toHashFormat` + + For the builtin `convertHash`, the `toHashFormat` parameter now accepts the same hash formats as the `--to`/`--from` + parameters of the `nix hash conert` command: `"base16"`, `"nix32"`, `"base64"`, and `"sri"`. The former `"base32"` value + remains as a deprecated alias for `"nix32"`. Please convert your code from: + + ```nix + builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base32";} + ``` + + to + + ```nix + builtins.convertHash { inherit hash hashAlgo; toHashFormat = "nix32";} + ``` + +- Add `pipe-operators` experimental feature [#11131](https://github.com/NixOS/nix/pull/11131) + + This is a draft implementation of [RFC 0148](https://github.com/NixOS/rfcs/pull/148). + + The `pipe-operators` experimental feature adds [`<|` and `|>` operators][pipe operators] to the Nix language. + *a* `|>` *b* is equivalent to the function application *b* *a*, and + *a* `<|` *b* is equivalent to the function application *a* *b*. + + For example: + + ``` + nix-repl> 1 |> builtins.add 2 |> builtins.mul 3 + 9 + + nix-repl> builtins.add 1 <| builtins.mul 2 <| 3 + 7 + ``` + + `<|` and `|>` are right and left associative, respectively, and have lower precedence than any other operator. + These properties may change in future releases. + + See [the RFC](https://github.com/NixOS/rfcs/pull/148) for more examples and rationale. + + [pipe operators]: @docroot@/language/operators.md#pipe-operators + +- `nix-shell` shebang uses relative path [#4232](https://github.com/NixOS/nix/issues/4232) [#5088](https://github.com/NixOS/nix/pull/5088) [#11058](https://github.com/NixOS/nix/pull/11058) + + + Relative [path](@docroot@/language/types.md#type-path) literals in `nix-shell` shebang scripts' options are now resolved relative to the [script's location](@docroot@/glossary.md?highlight=base%20directory#gloss-base-directory). + Previously they were resolved relative to the current working directory. + + For example, consider the following script in `~/myproject/say-hi`: + + ```shell + #!/usr/bin/env nix-shell + #!nix-shell --expr 'import ./shell.nix' + #!nix-shell --arg toolset './greeting-tools.nix' + #!nix-shell -i bash + hello + ``` + + Older versions of `nix-shell` would resolve `shell.nix` relative to the current working directory, such as the user's home directory in this example: + + ```console + [hostname:~]$ ./myproject/say-hi + error: + … while calling the 'import' builtin + at «string»:1:2: + 1| (import ./shell.nix) + | ^ + + error: path '/home/user/shell.nix' does not exist + ``` + + Since this release, `nix-shell` resolves `shell.nix` relative to the script's location, and `~/myproject/shell.nix` is used. + + ```console + $ ./myproject/say-hi + Hello, world! + ``` + + **Opt-out** + + This is technically a breaking change, so we have added an option so you can adapt independently of your Nix update. + The old behavior can be opted into by setting the option [`nix-shell-shebang-arguments-relative-to-script`](@docroot@/command-ref/conf-file.md#conf-nix-shell-shebang-arguments-relative-to-script) to `false`. + This option will be removed in a future release. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + +- Improve handling of tarballs that don't consist of a single top-level directory [#11195](https://github.com/NixOS/nix/pull/11195) + + In previous Nix releases, the tarball fetcher (used by `builtins.fetchTarball`) erroneously merged top-level directories into a single directory, and silently discarded top-level files that are not directories. This is no longer the case. The new behaviour is that *only* if the tarball consists of a single directory, the top-level path component of the files in the tarball is removed (similar to `tar`'s `--strip-components=1`). + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Improve handling of tarballs that don't consist of a single top-level directory [#11195](https://github.com/NixOS/nix/pull/11195) + + In previous Nix releases, the tarball fetcher (used by `builtins.fetchTarball`) erroneously merged top-level directories into a single directory, and silently discarded top-level files that are not directories. This is no longer the case. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Setting to warn about large paths [#10778](https://github.com/NixOS/nix/pull/10778) + + Nix can now warn when evaluation of a Nix expression causes a large + path to be copied to the Nix store. The threshold for this warning can + be configured using the `warn-large-path-threshold` setting, + e.g. `--warn-large-path-threshold 100M`. + + +# Contributors + +This release was made possible by the following 43 contributors: + +- Andreas Rammhold [**(@andir)**](https://github.com/andir) +- Andrew Marshall [**(@amarshall)**](https://github.com/amarshall) +- Brian McKenna [**(@puffnfresh)**](https://github.com/puffnfresh) +- Cameron [**(@SkamDart)**](https://github.com/SkamDart) +- Cole Helbling [**(@cole-h)**](https://github.com/cole-h) +- Corbin Simpson [**(@MostAwesomeDude)**](https://github.com/MostAwesomeDude) +- Eelco Dolstra [**(@edolstra)**](https://github.com/edolstra) +- Emily [**(@emilazy)**](https://github.com/emilazy) +- Enno Richter [**(@elohmeier)**](https://github.com/elohmeier) +- Farid Zakaria [**(@fzakaria)**](https://github.com/fzakaria) +- HaeNoe [**(@haenoe)**](https://github.com/haenoe) +- Hamir Mahal [**(@hamirmahal)**](https://github.com/hamirmahal) +- Harmen [**(@alicebob)**](https://github.com/alicebob) +- Ivan Trubach [**(@tie)**](https://github.com/tie) +- Jared Baur [**(@jmbaur)**](https://github.com/jmbaur) +- John Ericson [**(@Ericson2314)**](https://github.com/Ericson2314) +- Jonathan De Troye [**(@detroyejr)**](https://github.com/detroyejr) +- Jörg Thalheim [**(@Mic92)**](https://github.com/Mic92) +- Klemens Nanni [**(@klemensn)**](https://github.com/klemensn) +- Las Safin [**(@L-as)**](https://github.com/L-as) +- Lexi Mattick [**(@kognise)**](https://github.com/kognise) +- Matthew Bauer [**(@matthewbauer)**](https://github.com/matthewbauer) +- Max “Goldstein†Siling [**(@GoldsteinE)**](https://github.com/GoldsteinE) +- Mingye Wang [**(@Artoria2e5)**](https://github.com/Artoria2e5) +- Philip Taron [**(@philiptaron)**](https://github.com/philiptaron) +- Pierre Bourdon [**(@delroth)**](https://github.com/delroth) +- Pino Toscano [**(@pinotree)**](https://github.com/pinotree) +- RTUnreal [**(@RTUnreal)**](https://github.com/RTUnreal) +- Robert Hensing [**(@roberth)**](https://github.com/roberth) +- Romain Neil [**(@romain-neil)**](https://github.com/romain-neil) +- Ryan Hendrickson [**(@rhendric)**](https://github.com/rhendric) +- Sergei Trofimovich [**(@trofi)**](https://github.com/trofi) +- Shogo Takata [**(@pineapplehunter)**](https://github.com/pineapplehunter) +- Siddhant Kumar [**(@siddhantk232)**](https://github.com/siddhantk232) +- Silvan Mosberger [**(@infinisil)**](https://github.com/infinisil) +- Théophane Hufschmitt [**(@thufschmitt)**](https://github.com/thufschmitt) +- Valentin Gagarin [**(@fricklerhandwerk)**](https://github.com/fricklerhandwerk) +- Winter [**(@winterqt)**](https://github.com/winterqt) +- jade [**(@lf-)**](https://github.com/lf-) +- kirillrdy [**(@kirillrdy)**](https://github.com/kirillrdy) +- pennae [**(@pennae)**](https://github.com/pennae) +- poweredbypie [**(@poweredbypie)**](https://github.com/poweredbypie) +- tomberek [**(@tomberek)**](https://github.com/tomberek) diff --git a/doc/manual/src/release-notes/rl-2.4.md b/doc/manual/src/release-notes/rl-2.4.md index 8b566fc7b..1201e53b6 100644 --- a/doc/manual/src/release-notes/rl-2.4.md +++ b/doc/manual/src/release-notes/rl-2.4.md @@ -23,7 +23,7 @@ more than 2800 commits from 195 contributors since release 2.3. * The **`nix` command** has seen a lot of work and is now almost at feature parity with the old command-line interface (the `nix-*` commands). It aims to be [more modern, consistent and pleasant to - use](../contributing/cli-guideline.md) than the old CLI. It is still + use](../development/cli-guideline.md) than the old CLI. It is still marked as experimental but its interface should not change much anymore in future releases. diff --git a/doc/manual/src/store/file-system-object/content-address.md b/doc/manual/src/store/file-system-object/content-address.md new file mode 100644 index 000000000..410d7fb7c --- /dev/null +++ b/doc/manual/src/store/file-system-object/content-address.md @@ -0,0 +1,85 @@ +# Content-Addressing File System Objects + +For many operations, Nix needs to calculate [a content addresses](@docroot@/glossary.md#gloss-content-address) of [a file system object][file system object]. +Usually this is needed as part of +[content addressing store objects](../store-object/content-address.md), +since store objects always have a root file system object. +But some command-line utilities also just work on "raw" file system objects, not part of any store object. + +Every content addressing scheme Nix uses ultimately involves feeding data into a [hash function](https://en.wikipedia.org/wiki/Hash_function), and getting back an opaque fixed-size digest which is deemed a content address. +The various *methods* of content addressing thus differ in how abstract data (in this case, a file system object and its descendents) are fed into the hash function. + +## Serialising File System Objects { #serial } + +The simplest method is to serialise the entire file system object tree into a single binary string, and then hash that binary string, yielding the content address. +In this section we describe the currently-supported methods of serialising file system objects. + +### Flat { #serial-flat } + +A single file object can just be hashed by its contents. +This is not enough information to encode the fact that the file system object is a file, +but if we *already* know that the FSO is a single non-executable file by other means, it is sufficient. + +Because the hashed data is just the raw file, as is, this choice is good for compatibility with other systems. +For example, Unix commands like `sha256sum` or `sha1sum` will produce hashes for single files that match this. + +### Nix Archive (NAR) { #serial-nix-archive } + +For the other cases of [file system objects][file system object], especially directories with arbitrary descendents, we need a more complex serialisation format. +Examples of such serialisations are the ZIP and TAR file formats. +However, for our purposes these formats have two problems: + +- They do not have a canonical serialisation, meaning that given an FSO, there can +be many different serialisations. + For instance, TAR files can have variable amounts of padding between archive members; + and some archive formats leave the order of directory entries undefined. + This would be bad because we use serialisation to compute cryptographic hashes over file system objects, and for those hashes to be useful as a content address or for integrity checking, uniqueness is crucial. + Otherwise, correct hashes would report false mismatches, and the store would fail to find the content. + +- They store more information than we have in our notion of FSOs, such as time stamps. + This can cause FSOs that Nix should consider equal to hash to different values on different machines, just because the dates differ. + +- As a practical consideration, the TAR format is the only truly universal format in the Unix environment. + It has many problems, such as an inability to deal with long file names and files larger than 2^33 bytes. + Current implementations such as GNU Tar work around these limitations in various ways. + +For these reasons, Nix has its very own archive format—the Nix Archive (NAR) format, +which is carefully designed to avoid the problems described above. + +The exact specification of the Nix Archive format is in `protocols/nix-archive.md` + +## Content addressing File System Objects beyond a single serialisation pass + +Serialising the entire tree and then hashing that binary string is not the only option for content addressing, however. +Another technique is that of a [Merkle graph](https://en.wikipedia.org/wiki/Merkle_tree), where previously computed hashes are included in subsequent byte strings to be hashed. + +In particular, the Merkle graphs can match the original graph structure of file system objects: +we can first hash (serialised) child file system objects, and then hash parent objects using the hashes of their children in the serialisation (to be hashed) of the parent file system objects. + +Currently, there is one such Merkle DAG content addressing method supported. + +### Git ([experimental][xp-feature-git-hashing]) { #git } + +> **Warning** +> +> This method is part of the [`git-hashing`][xp-feature-git-hashing] experimental feature. + +Git's file system model is very close to Nix's, and so Git's content addressing method is a pretty good fit. +Just as with regular Git, files and symlinks are hashed as git "blobs", and directories are hashed as git "trees". + +However, one difference between Nix's and Git's file system model needs special treatment. +Plain files, executable files, and symlinks are not differentiated as distinctly addressable objects, but by their context: by the directory entry that refers to them. +That means so long as the root object is a directory, there is no problem: +every non-directory object is owned by a parent directory, and the entry that refers to it provides the missing information. +However, if the root object is not a directory, then we have no way of knowing which one of an executable file, non-executable file, or symlink it is supposed to be. + +In response to this, we have decided to treat a bare file as non-executable file. +This is similar to do what we do with [flat serialisation](#serial-flat), which also lacks this information. +To avoid an address collision, attempts to hash a bare executable file or symlink will result in an error (just as would happen for flat serialisation also). +Thus, Git can encode some, but not all of Nix's "File System Objects", and this sort of content-addressing is likewise partial. + +In the future, we may support a Git-like hash for such file system objects, or we may adopt another Merkle DAG format which is capable of representing all Nix file system objects. + +[file system object]: ../file-system-object.md +[store object]: ../store-object.md +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/doc/manual/src/store/store-object/content-address.md b/doc/manual/src/store/store-object/content-address.md new file mode 100644 index 000000000..02dce2836 --- /dev/null +++ b/doc/manual/src/store/store-object/content-address.md @@ -0,0 +1,95 @@ +# Content-Addressing Store Objects + +Just [like][fso-ca] [File System Objects][File System Object], +[Store Objects][Store Object] can also be [content-addressed](@docroot@/glossary.md#gloss-content-addressed), +unless they are [input-addressed](@docroot@/glossary.md#gloss-input-addressed-store-object). + +For store objects, the content address we produce will take the form of a [Store Path] rather than regular hash. +In particular, the content-addressing scheme will ensure that the digest of the store path is solely computed from the + +- file system object graph (the root one and its children, if it has any) +- references +- [store directory](../store-path.md#store-directory) +- name + +of the store object, and not any other information, which would not be an intrinsic property of that store object. + +For the full specification of the algorithms involved, see the [specification of store path digests][sp-spec]. + +[File System Object]: ../file-system-object.md +[Store Object]: ../store-object.md +[Store Path]: ../store-path.md + +## Content addressing each part of a store object + +### File System Objects + +With all currently supported store object content addressing methods, the file system object is always [content-addressed][fso-ca] first, and then that hash is incorporated into content address computation for the store object. + +### References + +With all currently supported store object content addressing methods, +other objects are referred to by their regular (string-encoded-) [store paths][Store Path]. + +Self-references however cannot be referred to by their path, because we are in the midst of describing how to compute that path! + +> The alternative would require finding as hash function fixed point, i.e. the solution to an equation in the form +> ``` +> digest = hash(..... || digest || ....) +> ``` +> which is computationally infeasible. +> As far as we know, this is equivalent to finding a hash collision. + +Instead we just have a "has self reference" boolean, which will end up affecting the digest. + +### Name and Store Directory + +These two items affect the digest in a way that is standard for store path digest computations and not specific to content-addressing. +Consult the [specification of store path digests][sp-spec] for further details. + +## Content addressing Methods + +For historical reasons, we don't support all features in all combinations. +Each currently supported method of content addressing chooses a single method of file system object hashing, and may offer some restrictions on references. +The names and store directories are unrestricted however. + +### Flat { #method-flat } + +This uses the corresponding [Flat](../file-system-object/content-address.md#serial-flat) method of file system object content addressing. + +References are not supported: store objects with flat hashing *and* references can not be created. + +### Text { #method-text } + +This also uses the corresponding [Flat](../file-system-object/content-address.md#serial-flat) method of file system object content addressing. + +References to other store objects are supported, but self references are not. + +This is the only store-object content-addressing method that is not named identically with a corresponding file system object method. +It is somewhat obscure, mainly used for "drv files" +(derivations serialized as store objects in their ["ATerm" file format](@docroot@/protocols/derivation-aterm.md)). +Prefer another method if possible. + +### Nix Archive { #method-nix-archive } + +This uses the corresponding [Nix Archive](../file-system-object/content-address.md#serial-nix-archive) method of file system object content addressing. + +References (to other store objects and self references alike) are supported so long as the hash algorithm is SHA-256, but not (neither kind) otherwise. + +### Git { #method-git } + +> **Warning** +> +> This method is part of the [`git-hashing`][xp-feature-git-hashing] experimental feature. + +This uses the corresponding [Git](../file-system-object/content-address.md#serial-git) method of file system object content addressing. + +References are not supported. + +Only SHA-1 is supported at this time. +If [SHA-256-based Git](https://git-scm.com/docs/hash-function-transition) +becomes more widespread, this restriction will be revisited. + +[fso-ca]: ../file-system-object/content-address.md +[sp-spec]: @docroot@/protocols/store-path.md +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/doc/manual/src/store/store-path.md b/doc/manual/src/store/store-path.md index b5ad0c654..beec2389b 100644 --- a/doc/manual/src/store/store-path.md +++ b/doc/manual/src/store/store-path.md @@ -1,5 +1,11 @@ # Store Path +> **Example** +> +> `/nix/store/a040m110amc4h71lds2jmr8qrkj2jhxd-git-2.38.1` +> +> A rendered store path + Nix implements references to [store objects](./index.md#store-object) as *store paths*. Think of a store path as an [opaque], [unique identifier]: @@ -37,6 +43,10 @@ A store path is rendered to a file system path as the concatenation of > store directory digest name > ``` +Exactly how the digest is calculated depends on the type of store path. +Store path digests are *supposed* to be opaque, and so for most operations, it is not necessary to know the details. +That said, the manual has a full [specification of store path digests](@docroot@/protocols/store-path.md). + ## Store Directory Every [Nix store](./index.md) has a store directory. @@ -46,7 +56,7 @@ But if the store has a file system representation, the store directory contains [file system objects]: ./file-system-object.md -This means a store path is not just derived from the referenced store object itself, but depends on the store the store object is in. +This means a store path is not just derived from the referenced store object itself, but depends on the store that the store object is in. > **Note** > diff --git a/flake.lock b/flake.lock index bb2e400c0..2ac413a69 100644 --- a/flake.lock +++ b/flake.lock @@ -3,11 +3,11 @@ "flake-compat": { "flake": false, "locked": { - "lastModified": 1673956053, - "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", "owner": "edolstra", "repo": "flake-compat", - "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", "type": "github" }, "original": { @@ -16,38 +16,100 @@ "type": "github" } }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1719994518, + "narHash": "sha256-pQMhCCHyQGRzdfAkdJ4cIWiw+JNuWsTX7f0ZYSyz0VY=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "9227223f6d922fee3c7b190b2cc238a99527bbb7", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "git-hooks-nix": { + "inputs": { + "flake-compat": [], + "gitignore": [], + "nixpkgs": [ + "nixpkgs" + ], + "nixpkgs-stable": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1721042469, + "narHash": "sha256-6FPUl7HVtvRHCCBQne7Ylp4p+dpP3P/OYuzjztZ4s70=", + "owner": "cachix", + "repo": "git-hooks.nix", + "rev": "f451c19376071a90d8c58ab1a953c6e9840527fd", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "git-hooks.nix", + "type": "github" + } + }, "libgit2": { "flake": false, "locked": { - "lastModified": 1697646580, - "narHash": "sha256-oX4Z3S9WtJlwvj0uH9HlYcWv+x1hqp8mhXl7HsLu2f0=", + "lastModified": 1715853528, + "narHash": "sha256-J2rCxTecyLbbDdsyBWn9w7r3pbKRMkI9E7RvRgAqBdY=", "owner": "libgit2", "repo": "libgit2", - "rev": "45fd9ed7ae1a9b74b957ef4f337bc3c8b3df01b5", + "rev": "36f7e21ad757a3dacc58cf7944329da6bc1d6e96", "type": "github" }, "original": { "owner": "libgit2", + "ref": "v1.8.1", "repo": "libgit2", "type": "github" } }, "nixpkgs": { "locked": { - "lastModified": 1709083642, - "narHash": "sha256-7kkJQd4rZ+vFrzWu8sTRtta5D1kBG0LSRYAfhtmMlSo=", + "lastModified": 1721548954, + "narHash": "sha256-7cCC8+Tdq1+3OPyc3+gVo9dzUNkNIQfwSDJ2HSi2u3o=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "b550fe4b4776908ac2a861124307045f8e717c8e", + "rev": "63d37ccd2d178d54e7fb691d7ec76000740ea24a", "type": "github" }, "original": { "owner": "NixOS", - "ref": "release-23.11", + "ref": "nixos-24.05", "repo": "nixpkgs", "type": "github" } }, + "nixpkgs-23-11": { + "locked": { + "lastModified": 1717159533, + "narHash": "sha256-oamiKNfr2MS6yH64rUn99mIZjc45nGJlj9eGth/3Xuw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446", + "type": "github" + } + }, "nixpkgs-regression": { "locked": { "lastModified": 1643052045, @@ -67,8 +129,11 @@ "root": { "inputs": { "flake-compat": "flake-compat", + "flake-parts": "flake-parts", + "git-hooks-nix": "git-hooks-nix", "libgit2": "libgit2", "nixpkgs": "nixpkgs", + "nixpkgs-23-11": "nixpkgs-23-11", "nixpkgs-regression": "nixpkgs-regression" } } diff --git a/flake.nix b/flake.nix index 42aaace67..9e8592e3a 100644 --- a/flake.nix +++ b/flake.nix @@ -1,18 +1,28 @@ { description = "The purely functional package manager"; - # TODO switch to nixos-23.11-small - # https://nixpk.gs/pr-tracker.html?pr=291954 - inputs.nixpkgs.url = "github:NixOS/nixpkgs/release-23.11"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; + inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446"; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; - inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; }; + inputs.libgit2 = { url = "github:libgit2/libgit2/v1.8.1"; flake = false; }; + + # dev tooling + inputs.flake-parts.url = "github:hercules-ci/flake-parts"; + inputs.git-hooks-nix.url = "github:cachix/git-hooks.nix"; + # work around https://github.com/NixOS/nix/issues/7730 + inputs.flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; + inputs.git-hooks-nix.inputs.nixpkgs.follows = "nixpkgs"; + inputs.git-hooks-nix.inputs.nixpkgs-stable.follows = "nixpkgs"; + # work around 7730 and https://github.com/NixOS/nix/issues/7807 + inputs.git-hooks-nix.inputs.flake-compat.follows = ""; + inputs.git-hooks-nix.inputs.gitignore.follows = ""; + + outputs = inputs@{ self, nixpkgs, nixpkgs-regression, libgit2, ... }: - outputs = { self, nixpkgs, nixpkgs-regression, libgit2, ... }: let inherit (nixpkgs) lib; - inherit (lib) fileset; officialRelease = false; @@ -31,14 +41,9 @@ crossSystems = [ "armv6l-unknown-linux-gnueabihf" "armv7l-unknown-linux-gnueabihf" - "x86_64-unknown-freebsd13" + "riscv64-unknown-linux-gnu" "x86_64-unknown-netbsd" - ]; - - # Nix doesn't yet build on this platform, so we put it in a - # separate list. We just use this for `devShells` and - # `nixpkgsFor`, which this depends on. - shellCrossSystems = crossSystems ++ [ + "x86_64-unknown-freebsd" "x86_64-w64-mingw32" ]; @@ -50,6 +55,18 @@ "stdenv" ]; + /** + `flatMapAttrs attrs f` applies `f` to each attribute in `attrs` and + merges the results into a single attribute set. + + This can be nested to form a build matrix where all the attributes + generated by the innermost `f` are returned as is. + (Provided that the names are unique.) + + See https://nixos.org/manual/nixpkgs/stable/index.html#function-library-lib.attrsets.concatMapAttrs + */ + flatMapAttrs = attrs: f: lib.concatMapAttrs f attrs; + forAllSystems = lib.genAttrs systems; forAllCrossSystems = lib.genAttrs crossSystems; @@ -63,6 +80,17 @@ }) stdenvs); + + # We don't apply flake-parts to the whole flake so that non-development attributes + # load without fetching any development inputs. + devFlake = inputs.flake-parts.lib.mkFlake { inherit inputs; } { + imports = [ ./maintainers/flake-module.nix ]; + systems = lib.subtractLists crossSystems systems; + perSystem = { system, ... }: { + _module.args.pkgs = nixpkgsFor.${system}.native; + }; + }; + # Memoize nixpkgs for different platforms for efficiency. nixpkgsFor = forAllSystems (system: let @@ -84,31 +112,9 @@ in { inherit stdenvs native; static = native.pkgsStatic; - cross = lib.genAttrs shellCrossSystems (crossSystem: make-pkgs crossSystem "stdenv"); + cross = forAllCrossSystems (crossSystem: make-pkgs crossSystem "stdenv"); }); - installScriptFor = tarballs: - nixpkgsFor.x86_64-linux.native.callPackage ./scripts/installer.nix { - inherit tarballs; - }; - - testNixVersions = pkgs: client: daemon: - pkgs.callPackage ./package.nix { - pname = - "nix-tests" - + lib.optionalString - (lib.versionAtLeast daemon.version "2.4pre20211005" && - lib.versionAtLeast client.version "2.4pre20211005") - "-${client.version}-against-${daemon.version}"; - - inherit fileset; - - test-client = client; - test-daemon = daemon; - - doBuild = false; - }; - binaryTarball = nix: pkgs: pkgs.callPackage ./scripts/binary-tarball.nix { inherit nix; }; @@ -120,244 +126,130 @@ { nixStable = prev.nix; - default-busybox-sandbox-shell = final.busybox.override { - useMusl = true; - enableStatic = true; - enableMinimal = true; - extraConfig = '' - CONFIG_FEATURE_FANCY_ECHO y - CONFIG_FEATURE_SH_MATH y - CONFIG_FEATURE_SH_MATH_64 y + # A new scope, so that we can use `callPackage` to inject our own interdependencies + # without "polluting" the top level "`pkgs`" attrset. + # This also has the benefit of providing us with a distinct set of packages + # we can iterate over. + nixComponents = lib.makeScope final.nixDependencies.newScope (import ./packaging/components.nix); - CONFIG_ASH y - CONFIG_ASH_OPTIMIZE_FOR_SIZE y - - CONFIG_ASH_ALIAS y - CONFIG_ASH_BASH_COMPAT y - CONFIG_ASH_CMDCMD y - CONFIG_ASH_ECHO y - CONFIG_ASH_GETOPTS y - CONFIG_ASH_INTERNAL_GLOB y - CONFIG_ASH_JOB_CONTROL y - CONFIG_ASH_PRINTF y - CONFIG_ASH_TEST y - ''; - }; - - libgit2-nix = final.libgit2.overrideAttrs (attrs: { - src = libgit2; - version = libgit2.lastModifiedDate; - cmakeFlags = attrs.cmakeFlags or [] - ++ [ "-DUSE_SSH=exec" ]; + # The dependencies are in their own scope, so that they don't have to be + # in Nixpkgs top level `pkgs` or `nixComponents`. + nixDependencies = lib.makeScope final.newScope (import ./packaging/dependencies.nix { + inherit inputs stdenv versionSuffix; + pkgs = final; }); - boehmgc-nix = (final.boehmgc.override { - enableLargeConfig = true; - }).overrideAttrs(o: { - patches = (o.patches or []) ++ [ - ./dep-patches/boehmgc-coroutine-sp-fallback.diff + nix = final.nixComponents.nix; - # https://github.com/ivmai/bdwgc/pull/586 - ./dep-patches/boehmgc-traceable_allocator-public.diff - ]; - }); - - changelog-d-nix = final.buildPackages.callPackage ./misc/changelog-d.nix { }; - - nix = - let - officialRelease = false; - versionSuffix = - if officialRelease - then "" - else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}"; - - in final.callPackage ./package.nix { - inherit - fileset - stdenv - versionSuffix - ; - officialRelease = false; - boehmgc = final.boehmgc-nix; - libgit2 = final.libgit2-nix; - busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell; - } // { - # this is a proper separate downstream package, but put - # here also for back compat reasons. - perl-bindings = final.nix-perl-bindings; - }; - - nix-perl-bindings = final.callPackage ./perl { - inherit fileset stdenv; + nix_noTests = final.nix.override { + doInstallCheck = false; + doCheck = false; }; + # See https://github.com/NixOS/nixpkgs/pull/214409 + # Remove when fixed in this flake's nixpkgs + pre-commit = + if prev.stdenv.hostPlatform.system == "i686-linux" + then (prev.pre-commit.override (o: { dotnet-sdk = ""; })).overridePythonAttrs (o: { doCheck = false; }) + else prev.pre-commit; + }; in { # A Nixpkgs overlay that overrides the 'nix' and - # 'nix.perl-bindings' packages. + # 'nix-perl-bindings' packages. overlays.default = overlayFor (p: p.stdenv); - hydraJobs = { - - # Binary package for various platforms. - build = forAllSystems (system: self.packages.${system}.nix); - - shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation); - - buildStatic = lib.genAttrs linux64BitSystems (system: self.packages.${system}.nix-static); - - buildCross = forAllCrossSystems (crossSystem: - lib.genAttrs ["x86_64-linux"] (system: self.packages.${system}."nix-${crossSystem}")); - - buildNoGc = forAllSystems (system: - self.packages.${system}.nix.override { enableGC = false; } - ); - - buildNoTests = forAllSystems (system: - self.packages.${system}.nix.override { - doCheck = false; - doInstallCheck = false; - installUnitTests = false; - } - ); - - # Toggles some settings for better coverage. Windows needs these - # library combinations, and Debian build Nix with GNU readline too. - buildReadlineNoMarkdown = forAllSystems (system: - self.packages.${system}.nix.override { - enableMarkdown = false; - readlineFlavor = "readline"; - } - ); - - # Perl bindings for various platforms. - perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nix.perl-bindings); - - # Binary tarball for various platforms, containing a Nix store - # with the closure of 'nix' package, and the second half of - # the installation script. - binaryTarball = forAllSystems (system: binaryTarball nixpkgsFor.${system}.native.nix nixpkgsFor.${system}.native); - - binaryTarballCross = lib.genAttrs ["x86_64-linux"] (system: - forAllCrossSystems (crossSystem: - binaryTarball - self.packages.${system}."nix-${crossSystem}" - nixpkgsFor.${system}.cross.${crossSystem})); - - # The first half of the installation script. This is uploaded - # to https://nixos.org/nix/install. It downloads the binary - # tarball for the user's system and calls the second half of the - # installation script. - installerScript = installScriptFor [ - # Native - self.hydraJobs.binaryTarball."x86_64-linux" - self.hydraJobs.binaryTarball."i686-linux" - self.hydraJobs.binaryTarball."aarch64-linux" - self.hydraJobs.binaryTarball."x86_64-darwin" - self.hydraJobs.binaryTarball."aarch64-darwin" - # Cross - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" - ]; - installerScriptForGHA = installScriptFor [ - # Native - self.hydraJobs.binaryTarball."x86_64-linux" - self.hydraJobs.binaryTarball."x86_64-darwin" - # Cross - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" - ]; - - # docker image with Nix inside - dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); - - # Line coverage analysis. - coverage = nixpkgsFor.x86_64-linux.native.nix.override { - pname = "nix-coverage"; - withCoverageChecks = true; - }; - - # API docs for Nix's unstable internal C++ interfaces. - internal-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ./package.nix { - inherit fileset; - doBuild = false; - enableInternalAPIDocs = true; - }; - - # System tests. - tests = import ./tests/nixos { inherit lib nixpkgs nixpkgsFor; } // { - - # Make sure that nix-env still produces the exact same result - # on a particular version of Nixpkgs. - evalNixpkgs = - let - inherit (nixpkgsFor.x86_64-linux.native) runCommand nix; - in - runCommand "eval-nixos" { buildInputs = [ nix ]; } - '' - type -p nix-env - # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593. - time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages - [[ $(sha1sum < packages | cut -c1-40) = ff451c521e61e4fe72bdbe2d0ca5d1809affa733 ]] - mkdir $out - ''; - - nixpkgsLibTests = - forAllSystems (system: - import (nixpkgs + "/lib/tests/release.nix") - { pkgs = nixpkgsFor.${system}.native; - nixVersions = [ self.packages.${system}.nix ]; - } - ); - }; - - metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" { - pkgs = nixpkgsFor.x86_64-linux.native; - nixpkgs = nixpkgs-regression; - }; - - installTests = forAllSystems (system: - let pkgs = nixpkgsFor.${system}.native; in - pkgs.runCommand "install-tests" { - againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix; - againstCurrentUnstable = - # FIXME: temporarily disable this on macOS because of #3605. - if system == "x86_64-linux" - then testNixVersions pkgs pkgs.nix pkgs.nixUnstable - else null; - # Disabled because the latest stable version doesn't handle - # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work - # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable; - } "touch $out"); - - installerTests = import ./tests/installer { - binaryTarballs = self.hydraJobs.binaryTarball; - inherit nixpkgsFor; - }; - + hydraJobs = import ./packaging/hydra.nix { + inherit + inputs + binaryTarball + forAllCrossSystems + forAllSystems + lib + linux64BitSystems + nixpkgsFor + self + ; }; checks = forAllSystems (system: { binaryTarball = self.hydraJobs.binaryTarball.${system}; - perlBindings = self.hydraJobs.perlBindings.${system}; installTests = self.hydraJobs.installTests.${system}; nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system}; rl-next = let pkgs = nixpkgsFor.${system}.native; in pkgs.buildPackages.runCommand "test-rl-next-release-notes" { } '' - LANG=C.UTF-8 ${pkgs.changelog-d-nix}/bin/changelog-d ${./doc/manual/rl-next} >$out + LANG=C.UTF-8 ${pkgs.changelog-d}/bin/changelog-d ${./doc/manual/rl-next} >$out ''; + repl-completion = nixpkgsFor.${system}.native.callPackage ./tests/repl-completion.nix { }; } // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) { dockerImage = self.hydraJobs.dockerImage.${system}; - }); + } // (lib.optionalAttrs (!(builtins.elem system linux32BitSystems))) { + # Some perl dependencies are broken on i686-linux. + # Since the support is only best-effort there, disable the perl + # bindings - packages = forAllSystems (system: rec { - inherit (nixpkgsFor.${system}.native) nix changelog-d-nix; - default = nix; - } // (lib.optionalAttrs (builtins.elem system linux64BitSystems) { - nix-static = nixpkgsFor.${system}.static.nix; + # Temporarily disabled because GitHub Actions OOM issues. Once + # the old build system is gone and we are back to one build + # system, we should reenable this. + #perlBindings = self.hydraJobs.perlBindings.${system}; + } + # Add "passthru" tests + // flatMapAttrs ({ + "" = nixpkgsFor.${system}.native; + } // lib.optionalAttrs (! nixpkgsFor.${system}.native.stdenv.hostPlatform.isDarwin) { + # TODO: enable static builds for darwin, blocked on: + # https://github.com/NixOS/nixpkgs/issues/320448 + "static-" = nixpkgsFor.${system}.static; + }) + (nixpkgsPrefix: nixpkgs: + flatMapAttrs nixpkgs.nixComponents + (pkgName: pkg: + flatMapAttrs pkg.tests or {} + (testName: test: { + "${nixpkgsPrefix}${pkgName}-${testName}" = test; + }) + ) + ) + // devFlake.checks.${system} or {} + ); + + packages = forAllSystems (system: + { # Here we put attributes that map 1:1 into packages., ie + # for which we don't apply the full build matrix such as cross or static. + inherit (nixpkgsFor.${system}.native) + changelog-d; + default = self.packages.${system}.nix; + nix-internal-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-internal-api-docs; + nix-external-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-external-api-docs; + } + # We need to flatten recursive attribute sets of derivations to pass `flake check`. + // flatMapAttrs + { # Components we'll iterate over in the upcoming lambda + "nix" = { }; + # Temporarily disabled because GitHub Actions OOM issues. Once + # the old build system is gone and we are back to one build + # system, we should reenable these. + #"nix-util" = { }; + #"nix-store" = { }; + #"nix-fetchers" = { }; + } + (pkgName: {}: { + # These attributes go right into `packages.`. + "${pkgName}" = nixpkgsFor.${system}.native.nixComponents.${pkgName}; + "${pkgName}-static" = nixpkgsFor.${system}.static.nixComponents.${pkgName}; + } + // flatMapAttrs (lib.genAttrs crossSystems (_: { })) (crossSystem: {}: { + # These attributes go right into `packages.`. + "${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName}; + }) + // flatMapAttrs (lib.genAttrs stdenvs (_: { })) (stdenvName: {}: { + # These attributes go right into `packages.`. + "${pkgName}-${stdenvName}" = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nixComponents.${pkgName}; + }) + ) + // lib.optionalAttrs (builtins.elem system linux64BitSystems) { dockerImage = let pkgs = nixpkgsFor.${system}.native; @@ -372,21 +264,27 @@ ln -s ${image} $image echo "file binary-dist $image" >> $out/nix-support/hydra-build-products ''; - } // builtins.listToAttrs (map - (crossSystem: { - name = "nix-${crossSystem}"; - value = nixpkgsFor.${system}.cross.${crossSystem}.nix; - }) - crossSystems) - // builtins.listToAttrs (map - (stdenvName: { - name = "nix-${stdenvName}"; - value = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nix; - }) - stdenvs))); + }); devShells = let - makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; forDevShell = true; }).overrideAttrs (attrs: { + makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; forDevShell = true; }).overrideAttrs (attrs: + let + modular = devFlake.getSystem stdenv.buildPlatform.system; + transformFlag = prefix: flag: + assert builtins.isString flag; + let + rest = builtins.substring 2 (builtins.stringLength flag) flag; + in + "-D${prefix}:${rest}"; + havePerl = stdenv.buildPlatform == stdenv.hostPlatform && stdenv.hostPlatform.isUnix; + ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags; + in { + pname = "shell-for-" + attrs.pname; + + # Remove the version suffix to avoid unnecessary attempts to substitute in nix develop + version = lib.fileContents ./.version; + name = attrs.pname; + installFlags = "sysconfdir=$(out)/etc"; shellHook = '' PATH=$prefix/bin:$PATH @@ -397,11 +295,62 @@ XDG_DATA_DIRS+=:$out/share ''; + # We use this shell with the local checkout, not unpackPhase. + src = null; + + env = { + # Needed for Meson to find Boost. + # https://github.com/NixOS/nixpkgs/issues/86131. + BOOST_INCLUDEDIR = "${lib.getDev pkgs.nixDependencies.boost}/include"; + BOOST_LIBRARYDIR = "${lib.getLib pkgs.nixDependencies.boost}/lib"; + # For `make format`, to work without installing pre-commit + _NIX_PRE_COMMIT_HOOKS_CONFIG = + "${(pkgs.formats.yaml { }).generate "pre-commit-config.yaml" modular.pre-commit.settings.rawConfig}"; + }; + + mesonFlags = + map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents.nix-util.mesonFlags) + ++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents.nix-store.mesonFlags) + ++ map (transformFlag "libfetchers") (ignoreCrossFile pkgs.nixComponents.nix-fetchers.mesonFlags) + ++ lib.optionals havePerl (map (transformFlag "perl") (ignoreCrossFile pkgs.nixComponents.nix-perl-bindings.mesonFlags)) + ++ map (transformFlag "libexpr") (ignoreCrossFile pkgs.nixComponents.nix-expr.mesonFlags) + ++ map (transformFlag "libcmd") (ignoreCrossFile pkgs.nixComponents.nix-cmd.mesonFlags) + ; + nativeBuildInputs = attrs.nativeBuildInputs or [] + ++ pkgs.nixComponents.nix-util.nativeBuildInputs + ++ pkgs.nixComponents.nix-store.nativeBuildInputs + ++ pkgs.nixComponents.nix-fetchers.nativeBuildInputs + ++ lib.optionals havePerl pkgs.nixComponents.nix-perl-bindings.nativeBuildInputs + ++ pkgs.nixComponents.nix-internal-api-docs.nativeBuildInputs + ++ pkgs.nixComponents.nix-external-api-docs.nativeBuildInputs + ++ lib.optional + (!stdenv.buildPlatform.canExecute stdenv.hostPlatform + # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479 + && !(stdenv.hostPlatform.isWindows && stdenv.buildPlatform.isDarwin) + && stdenv.hostPlatform.emulatorAvailable pkgs.buildPackages + && lib.meta.availableOn stdenv.buildPlatform (stdenv.hostPlatform.emulator pkgs.buildPackages)) + pkgs.buildPackages.mesonEmulatorHook + ++ [ + pkgs.buildPackages.cmake + pkgs.buildPackages.shellcheck + pkgs.buildPackages.changelog-d + modular.pre-commit.settings.package + (pkgs.writeScriptBin "pre-commit-hooks-install" + modular.pre-commit.settings.installationScript) + ] # TODO: Remove the darwin check once # https://github.com/NixOS/nixpkgs/pull/291814 is available ++ lib.optional (stdenv.cc.isClang && !stdenv.buildPlatform.isDarwin) pkgs.buildPackages.bear ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) pkgs.buildPackages.clang-tools; + + buildInputs = attrs.buildInputs or [] + ++ [ + pkgs.gtest + pkgs.rapidcheck + ] + ++ lib.optional havePerl pkgs.perl + ; }); in forAllSystems (system: @@ -413,8 +362,8 @@ in (makeShells "native" nixpkgsFor.${system}.native) // (lib.optionalAttrs (!nixpkgsFor.${system}.native.stdenv.isDarwin) - (makeShells "static" nixpkgsFor.${system}.static)) // - (lib.genAttrs shellCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv)) // + (makeShells "static" nixpkgsFor.${system}.static) // + (forAllCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv))) // { default = self.devShells.${system}.native-stdenvPackages; } diff --git a/local.mk b/local.mk index 3f3abb9f0..b27c7031e 100644 --- a/local.mk +++ b/local.mk @@ -2,9 +2,14 @@ GLOBAL_CXXFLAGS += -Wno-deprecated-declarations -Werror=switch # Allow switch-enum to be overridden for files that do not support it, usually because of dependency headers. ERROR_SWITCH_ENUM = -Werror=switch-enum -$(foreach i, config.h $(wildcard src/lib*/*.hh), \ +$(foreach i, config.h $(wildcard src/lib*/*.hh) $(filter-out %_internal.h, $(wildcard src/lib*c/*.h)), \ $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) +ifdef HOST_UNIX + $(foreach i, $(wildcard src/lib*/unix/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) +endif + $(GCH): src/libutil/util.hh config.h -GCH_CXXFLAGS = -I src/libutil +GCH_CXXFLAGS = $(INCLUDE_libutil) diff --git a/m4/gcc_bug_80431.m4 b/m4/gcc_bug_80431.m4 index e42f01956..cdc4ddb40 100644 --- a/m4/gcc_bug_80431.m4 +++ b/m4/gcc_bug_80431.m4 @@ -46,11 +46,13 @@ AC_DEFUN([ENSURE_NO_GCC_BUG_80431], ]])], [status_80431=0], [status_80431=$?], - [ - # Assume we're bug-free when cross-compiling - ]) + [status_80431='']) AC_LANG_POP(C++) AS_CASE([$status_80431], + [''],[ + AC_MSG_RESULT(cannot check because cross compiling) + AC_MSG_NOTICE(assume we are bug free) + ], [0],[ AC_MSG_RESULT(yes) ], diff --git a/maintainers/README.md b/maintainers/README.md index fa321c7c0..b92833497 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -30,17 +30,18 @@ We aim to achieve this by improving the contributor experience and attracting mo ## Members - Eelco Dolstra (@edolstra) – Team lead -- Théophane Hufschmitt (@thufschmitt) - Valentin Gagarin (@fricklerhandwerk) - Thomas Bereknyei (@tomberek) - Robert Hensing (@roberth) - John Ericson (@Ericson2314) +The team is on Github as [@NixOS/nix-team](https://github.com/orgs/NixOS/teams/nix-team). + ## Meeting protocol -The team meets twice a week: +The team meets twice a week (times are denoted in the [Europe/Amsterdam](https://en.m.wikipedia.org/wiki/Time_in_the_Netherlands) time zone): -- Discussion meeting: [Fridays 13:00-14:00 CET](https://calendar.google.com/calendar/event?eid=MHNtOGVuNWtrZXNpZHR2bW1sM3QyN2ZjaGNfMjAyMjExMjVUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) +- Discussion meeting: [Wednesday 21:00-22:00 Europe/Amsterdam](https://www.google.com/calendar/event?eid=ZG5rZzNyajRjajducGV2NGY5aGkzYWIwdnJfMjAyNDA1MDhUMTkwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) 1. Triage issues and pull requests from the [No Status](#no-status) column (30 min) 2. Discuss issues and pull requests from the [To discuss](#to-discuss) column (30 min). @@ -49,15 +50,19 @@ The team meets twice a week: - mark it as draft if it is blocked on the contributor - escalate it back to the team by moving it to To discuss, and leaving a comment as to why the issue needs to be discussed again. -- Work meeting: [Mondays 13:00-15:00 CET](https://calendar.google.com/calendar/event?eid=NTM1MG1wNGJnOGpmOTZhYms3bTB1bnY5cWxfMjAyMjExMjFUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) +- Work meeting: [Mondays 14:00-16:00 Europe/Amsterdam](https://www.google.com/calendar/event?eid=Ym52NDdzYnRic2NzcDcybjZiNDhpNzhpa3NfMjAyNDA1MTNUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) 1. Code review on pull requests from [In review](#in-review). 2. Other chores and tasks. Meeting notes are collected on a [collaborative scratchpad](https://pad.lassul.us/Cv7FpYx-Ri-4VjUykQOLAw). -Notes on issues and pull requests are posted as comments and linked from the meeting notes, so they are easy to find from both places. +Notes on issues and pull requests are posted as comments and linked from the meeting notes, so they are can be found from both places. [All meeting notes](https://discourse.nixos.org/search?expanded=true&q=Nix%20team%20meeting%20minutes%20%23%20%23dev%3Anix%20in%3Atitle%20order%3Alatest_topic) are published on Discourse under the [Nix category](https://discourse.nixos.org/c/dev/nix/50). +Team meetings are generally open to anyone interested. +We can make exceptions to discuss sensitive issues, such as security incidents or people matters. +Contact any team member to get a calendar invite for reminders and updates. + ## Project board protocol The team uses a [GitHub project board](https://github.com/orgs/NixOS/projects/19/views/1) for tracking its work. diff --git a/maintainers/data/release-credits-email-to-handle.json b/maintainers/data/release-credits-email-to-handle.json new file mode 100644 index 000000000..cddc1a6e7 --- /dev/null +++ b/maintainers/data/release-credits-email-to-handle.json @@ -0,0 +1,52 @@ +{ + "bogus": "bogus", + "edolstra@gmail.com": "edolstra", + "roberth@users.noreply.github.com": "roberth", + "toscano.pino@tiscali.it": "pinotree", + "valentin@gagarin.work": "fricklerhandwerk", + "mr.trubach@icloud.com": "tie", + "robert@roberthensing.nl": "roberth", + "lix@jade.fyi": "lf-", + "cole.e.helbling@outlook.com": "cole-h", + "joerg@thalheim.io": "Mic92", + "John.Ericson@Obsidian.Systems": "Ericson2314", + "ryan.hendrickson@alum.mit.edu": "rhendric", + "67135060+poweredbypie@users.noreply.github.com": "poweredbypie", + "detroyejr@outlook.com": "detroyejr", + "silvan.mosberger@tweag.io": "infinisil", + "vcs@emily.moe": "emilazy", + "farid.m.zakaria@gmail.com": "fzakaria", + "22859658+RTUnreal@users.noreply.github.com": "RTUnreal", + "me@las.rs": "L-as", + "philip.taron@gmail.com": "philiptaron", + "root@goldstein.rs": "GoldsteinE", + "tomberek@users.noreply.github.com": "tomberek", + "lexi.mattick@neuralink.com": "kognise", + "andrew@johnandrewmarshall.com": "amarshall", + "contact@romain-neil.fr": "romain-neil", + "Mic92@users.noreply.github.com": "Mic92", + "valentin.gagarin@tweag.io": "fricklerhandwerk", + "siddhantk232@gmail.com": "siddhantk232", + "kn@openbsd.org": "klemensn", + "slyich@gmail.com": "trofi", + "theophane.hufschmitt@tweag.io": "thufschmitt", + "alicebob@lijzij.de": "alicebob", + "winter@winter.cafe": "winterqt", + "brian@brianmckenna.org": "puffnfresh", + "git@haenoe.party": "haenoe", + "peshogo@gmail.com": "pineapplehunter", + "poweredbypie@users.noreply.github.com": "poweredbypie", + "arthur200126@gmail.com": "Artoria2e5", + "tomberek@gmail.com": "tomberek", + "jaredbaur@fastmail.com": "jmbaur", + "andreas@rammhold.de": "andir", + "hamirmahal@gmail.com": "hamirmahal", + "git@JohnEricson.me": "Ericson2314", + "8763518+SkamDart@users.noreply.github.com": "SkamDart", + "kirillrdy@gmail.com": "kirillrdy", + "pennae@lix.systems": "pennae", + "delroth@gmail.com": "delroth", + "enno@nerdworks.de": "elohmeier", + "mjbauer95@gmail.com": "matthewbauer", + "MostAwesomeDude@gmail.com": "MostAwesomeDude" +} \ No newline at end of file diff --git a/maintainers/data/release-credits-handle-to-name.json b/maintainers/data/release-credits-handle-to-name.json new file mode 100644 index 000000000..abf9ed05b --- /dev/null +++ b/maintainers/data/release-credits-handle-to-name.json @@ -0,0 +1,45 @@ +{ + "fzakaria": "Farid Zakaria", + "kognise": "Lexi Mattick", + "L-as": "Las Safin", + "haenoe": "HaeNoe", + "andir": "Andreas Rammhold", + "matthewbauer": "Matthew Bauer", + "emilazy": "Emily", + "pineapplehunter": "Shogo Takata", + "RTUnreal": null, + "jmbaur": "Jared Baur", + "Ericson2314": "John Ericson", + "pinotree": "Pino Toscano", + "tie": "Ivan Trubach", + "poweredbypie": null, + "fricklerhandwerk": "Valentin Gagarin", + "Mic92": "J\u00f6rg Thalheim", + "alicebob": "Harmen", + "elohmeier": "Enno Richter", + "delroth": "Pierre Bourdon", + "kirillrdy": null, + "thufschmitt": "Th\u00e9ophane Hufschmitt", + "detroyejr": "Jonathan De Troye", + "klemensn": "Klemens Nanni", + "tomberek": null, + "rhendric": "Ryan Hendrickson", + "philiptaron": "Philip Taron", + "puffnfresh": "Brian McKenna", + "lf-": "jade", + "romain-neil": "Romain Neil", + "hamirmahal": "Hamir Mahal", + "edolstra": "Eelco Dolstra", + "Artoria2e5": "Mingye Wang", + "SkamDart": "Cameron", + "roberth": "Robert Hensing", + "amarshall": "Andrew Marshall", + "trofi": "Sergei Trofimovich", + "cole-h": "Cole Helbling", + "infinisil": "Silvan Mosberger", + "siddhantk232": "Siddhant Kumar", + "winterqt": "Winter", + "GoldsteinE": "Max \u201cGoldstein\u201d Siling", + "pennae": null, + "MostAwesomeDude": "Corbin Simpson" +} \ No newline at end of file diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix new file mode 100644 index 000000000..be91df536 --- /dev/null +++ b/maintainers/flake-module.nix @@ -0,0 +1,677 @@ +{ lib, getSystem, inputs, ... }: + +{ + imports = [ + inputs.git-hooks-nix.flakeModule + ]; + + perSystem = { config, pkgs, ... }: { + + # https://flake.parts/options/pre-commit-hooks-nix.html#options + pre-commit.settings = { + hooks = { + clang-format = { + enable = true; + excludes = [ + # We don't want to format test data + # ''tests/(?!nixos/).*\.nix'' + ''^tests/unit/[^/]*/data/.*$'' + + # Don't format vendored code + ''^doc/manual/redirects\.js$'' + ''^doc/manual/theme/highlight\.js$'' + + # We haven't applied formatting to these files yet + ''^doc/manual/redirects\.js$'' + ''^doc/manual/theme/highlight\.js$'' + ''^precompiled-headers\.h$'' + ''^src/build-remote/build-remote\.cc$'' + ''^src/libcmd/built-path\.cc$'' + ''^src/libcmd/built-path\.hh$'' + ''^src/libcmd/command\.cc$'' + ''^src/libcmd/command\.hh$'' + ''^src/libcmd/common-eval-args\.cc$'' + ''^src/libcmd/common-eval-args\.hh$'' + ''^src/libcmd/editor-for\.cc$'' + ''^src/libcmd/installable-attr-path\.cc$'' + ''^src/libcmd/installable-attr-path\.hh$'' + ''^src/libcmd/installable-derived-path\.cc$'' + ''^src/libcmd/installable-derived-path\.hh$'' + ''^src/libcmd/installable-flake\.cc$'' + ''^src/libcmd/installable-flake\.hh$'' + ''^src/libcmd/installable-value\.cc$'' + ''^src/libcmd/installable-value\.hh$'' + ''^src/libcmd/installables\.cc$'' + ''^src/libcmd/installables\.hh$'' + ''^src/libcmd/legacy\.hh$'' + ''^src/libcmd/markdown\.cc$'' + ''^src/libcmd/misc-store-flags\.cc$'' + ''^src/libcmd/repl-interacter\.cc$'' + ''^src/libcmd/repl-interacter\.hh$'' + ''^src/libcmd/repl\.cc$'' + ''^src/libcmd/repl\.hh$'' + ''^src/libexpr-c/nix_api_expr\.cc$'' + ''^src/libexpr-c/nix_api_external\.cc$'' + ''^src/libexpr/attr-path\.cc$'' + ''^src/libexpr/attr-path\.hh$'' + ''^src/libexpr/attr-set\.cc$'' + ''^src/libexpr/attr-set\.hh$'' + ''^src/libexpr/eval-cache\.cc$'' + ''^src/libexpr/eval-cache\.hh$'' + ''^src/libexpr/eval-error\.cc$'' + ''^src/libexpr/eval-inline\.hh$'' + ''^src/libexpr/eval-settings\.cc$'' + ''^src/libexpr/eval-settings\.hh$'' + ''^src/libexpr/eval\.cc$'' + ''^src/libexpr/eval\.hh$'' + ''^src/libexpr/function-trace\.cc$'' + ''^src/libexpr/gc-small-vector\.hh$'' + ''^src/libexpr/get-drvs\.cc$'' + ''^src/libexpr/get-drvs\.hh$'' + ''^src/libexpr/json-to-value\.cc$'' + ''^src/libexpr/nixexpr\.cc$'' + ''^src/libexpr/nixexpr\.hh$'' + ''^src/libexpr/parser-state\.hh$'' + ''^src/libexpr/pos-table\.hh$'' + ''^src/libexpr/primops\.cc$'' + ''^src/libexpr/primops\.hh$'' + ''^src/libexpr/primops/context\.cc$'' + ''^src/libexpr/primops/fetchClosure\.cc$'' + ''^src/libexpr/primops/fetchMercurial\.cc$'' + ''^src/libexpr/primops/fetchTree\.cc$'' + ''^src/libexpr/primops/fromTOML\.cc$'' + ''^src/libexpr/print-ambiguous\.cc$'' + ''^src/libexpr/print-ambiguous\.hh$'' + ''^src/libexpr/print-options\.hh$'' + ''^src/libexpr/print\.cc$'' + ''^src/libexpr/print\.hh$'' + ''^src/libexpr/search-path\.cc$'' + ''^src/libexpr/symbol-table\.hh$'' + ''^src/libexpr/value-to-json\.cc$'' + ''^src/libexpr/value-to-json\.hh$'' + ''^src/libexpr/value-to-xml\.cc$'' + ''^src/libexpr/value-to-xml\.hh$'' + ''^src/libexpr/value\.hh$'' + ''^src/libexpr/value/context\.cc$'' + ''^src/libexpr/value/context\.hh$'' + ''^src/libfetchers/attrs\.cc$'' + ''^src/libfetchers/cache\.cc$'' + ''^src/libfetchers/cache\.hh$'' + ''^src/libfetchers/fetch-settings\.cc$'' + ''^src/libfetchers/fetch-settings\.hh$'' + ''^src/libfetchers/fetch-to-store\.cc$'' + ''^src/libfetchers/fetchers\.cc$'' + ''^src/libfetchers/fetchers\.hh$'' + ''^src/libfetchers/filtering-source-accessor\.cc$'' + ''^src/libfetchers/filtering-source-accessor\.hh$'' + ''^src/libfetchers/fs-source-accessor\.cc$'' + ''^src/libfetchers/fs-source-accessor\.hh$'' + ''^src/libfetchers/git-utils\.cc$'' + ''^src/libfetchers/git-utils\.hh$'' + ''^src/libfetchers/github\.cc$'' + ''^src/libfetchers/indirect\.cc$'' + ''^src/libfetchers/memory-source-accessor\.cc$'' + ''^src/libfetchers/path\.cc$'' + ''^src/libfetchers/registry\.cc$'' + ''^src/libfetchers/registry\.hh$'' + ''^src/libfetchers/tarball\.cc$'' + ''^src/libfetchers/tarball\.hh$'' + ''^src/libfetchers/git\.cc$'' + ''^src/libfetchers/mercurial\.cc$'' + ''^src/libflake/flake/config\.cc$'' + ''^src/libflake/flake/flake\.cc$'' + ''^src/libflake/flake/flake\.hh$'' + ''^src/libflake/flake/flakeref\.cc$'' + ''^src/libflake/flake/flakeref\.hh$'' + ''^src/libflake/flake/lockfile\.cc$'' + ''^src/libflake/flake/lockfile\.hh$'' + ''^src/libflake/flake/url-name\.cc$'' + ''^src/libmain/common-args\.cc$'' + ''^src/libmain/common-args\.hh$'' + ''^src/libmain/loggers\.cc$'' + ''^src/libmain/loggers\.hh$'' + ''^src/libmain/progress-bar\.cc$'' + ''^src/libmain/shared\.cc$'' + ''^src/libmain/shared\.hh$'' + ''^src/libmain/unix/stack\.cc$'' + ''^src/libstore/binary-cache-store\.cc$'' + ''^src/libstore/binary-cache-store\.hh$'' + ''^src/libstore/build-result\.hh$'' + ''^src/libstore/builtins\.hh$'' + ''^src/libstore/builtins/buildenv\.cc$'' + ''^src/libstore/builtins/buildenv\.hh$'' + ''^src/libstore/common-protocol-impl\.hh$'' + ''^src/libstore/common-protocol\.cc$'' + ''^src/libstore/common-protocol\.hh$'' + ''^src/libstore/common-ssh-store-config\.hh$'' + ''^src/libstore/content-address\.cc$'' + ''^src/libstore/content-address\.hh$'' + ''^src/libstore/daemon\.cc$'' + ''^src/libstore/daemon\.hh$'' + ''^src/libstore/derivations\.cc$'' + ''^src/libstore/derivations\.hh$'' + ''^src/libstore/derived-path-map\.cc$'' + ''^src/libstore/derived-path-map\.hh$'' + ''^src/libstore/derived-path\.cc$'' + ''^src/libstore/derived-path\.hh$'' + ''^src/libstore/downstream-placeholder\.cc$'' + ''^src/libstore/downstream-placeholder\.hh$'' + ''^src/libstore/dummy-store\.cc$'' + ''^src/libstore/export-import\.cc$'' + ''^src/libstore/filetransfer\.cc$'' + ''^src/libstore/filetransfer\.hh$'' + ''^src/libstore/gc-store\.hh$'' + ''^src/libstore/globals\.cc$'' + ''^src/libstore/globals\.hh$'' + ''^src/libstore/http-binary-cache-store\.cc$'' + ''^src/libstore/legacy-ssh-store\.cc$'' + ''^src/libstore/legacy-ssh-store\.hh$'' + ''^src/libstore/length-prefixed-protocol-helper\.hh$'' + ''^src/libstore/linux/personality\.cc$'' + ''^src/libstore/linux/personality\.hh$'' + ''^src/libstore/local-binary-cache-store\.cc$'' + ''^src/libstore/local-fs-store\.cc$'' + ''^src/libstore/local-fs-store\.hh$'' + ''^src/libstore/log-store\.cc$'' + ''^src/libstore/log-store\.hh$'' + ''^src/libstore/machines\.cc$'' + ''^src/libstore/machines\.hh$'' + ''^src/libstore/make-content-addressed\.cc$'' + ''^src/libstore/make-content-addressed\.hh$'' + ''^src/libstore/misc\.cc$'' + ''^src/libstore/names\.cc$'' + ''^src/libstore/names\.hh$'' + ''^src/libstore/nar-accessor\.cc$'' + ''^src/libstore/nar-accessor\.hh$'' + ''^src/libstore/nar-info-disk-cache\.cc$'' + ''^src/libstore/nar-info-disk-cache\.hh$'' + ''^src/libstore/nar-info\.cc$'' + ''^src/libstore/nar-info\.hh$'' + ''^src/libstore/outputs-spec\.cc$'' + ''^src/libstore/outputs-spec\.hh$'' + ''^src/libstore/parsed-derivations\.cc$'' + ''^src/libstore/path-info\.cc$'' + ''^src/libstore/path-info\.hh$'' + ''^src/libstore/path-references\.cc$'' + ''^src/libstore/path-regex\.hh$'' + ''^src/libstore/path-with-outputs\.cc$'' + ''^src/libstore/path\.cc$'' + ''^src/libstore/path\.hh$'' + ''^src/libstore/pathlocks\.cc$'' + ''^src/libstore/pathlocks\.hh$'' + ''^src/libstore/profiles\.cc$'' + ''^src/libstore/profiles\.hh$'' + ''^src/libstore/realisation\.cc$'' + ''^src/libstore/realisation\.hh$'' + ''^src/libstore/remote-fs-accessor\.cc$'' + ''^src/libstore/remote-fs-accessor\.hh$'' + ''^src/libstore/remote-store-connection\.hh$'' + ''^src/libstore/remote-store\.cc$'' + ''^src/libstore/remote-store\.hh$'' + ''^src/libstore/s3-binary-cache-store\.cc$'' + ''^src/libstore/s3\.hh$'' + ''^src/libstore/serve-protocol-impl\.cc$'' + ''^src/libstore/serve-protocol-impl\.hh$'' + ''^src/libstore/serve-protocol\.cc$'' + ''^src/libstore/serve-protocol\.hh$'' + ''^src/libstore/sqlite\.cc$'' + ''^src/libstore/sqlite\.hh$'' + ''^src/libstore/ssh-store\.cc$'' + ''^src/libstore/ssh\.cc$'' + ''^src/libstore/ssh\.hh$'' + ''^src/libstore/store-api\.cc$'' + ''^src/libstore/store-api\.hh$'' + ''^src/libstore/store-dir-config\.hh$'' + ''^src/libstore/build/derivation-goal\.cc$'' + ''^src/libstore/build/derivation-goal\.hh$'' + ''^src/libstore/build/drv-output-substitution-goal\.cc$'' + ''^src/libstore/build/drv-output-substitution-goal\.hh$'' + ''^src/libstore/build/entry-points\.cc$'' + ''^src/libstore/build/goal\.cc$'' + ''^src/libstore/build/goal\.hh$'' + ''^src/libstore/unix/build/hook-instance\.cc$'' + ''^src/libstore/unix/build/local-derivation-goal\.cc$'' + ''^src/libstore/unix/build/local-derivation-goal\.hh$'' + ''^src/libstore/build/substitution-goal\.cc$'' + ''^src/libstore/build/substitution-goal\.hh$'' + ''^src/libstore/build/worker\.cc$'' + ''^src/libstore/build/worker\.hh$'' + ''^src/libstore/builtins/fetchurl\.cc$'' + ''^src/libstore/builtins/unpack-channel\.cc$'' + ''^src/libstore/gc\.cc$'' + ''^src/libstore/local-overlay-store\.cc$'' + ''^src/libstore/local-overlay-store\.hh$'' + ''^src/libstore/local-store\.cc$'' + ''^src/libstore/local-store\.hh$'' + ''^src/libstore/unix/user-lock\.cc$'' + ''^src/libstore/unix/user-lock\.hh$'' + ''^src/libstore/optimise-store\.cc$'' + ''^src/libstore/unix/pathlocks\.cc$'' + ''^src/libstore/posix-fs-canonicalise\.cc$'' + ''^src/libstore/posix-fs-canonicalise\.hh$'' + ''^src/libstore/uds-remote-store\.cc$'' + ''^src/libstore/uds-remote-store\.hh$'' + ''^src/libstore/windows/build\.cc$'' + ''^src/libstore/worker-protocol-impl\.hh$'' + ''^src/libstore/worker-protocol\.cc$'' + ''^src/libstore/worker-protocol\.hh$'' + ''^src/libutil-c/nix_api_util_internal\.h$'' + ''^src/libutil/archive\.cc$'' + ''^src/libutil/archive\.hh$'' + ''^src/libutil/args\.cc$'' + ''^src/libutil/args\.hh$'' + ''^src/libutil/args/root\.hh$'' + ''^src/libutil/callback\.hh$'' + ''^src/libutil/canon-path\.cc$'' + ''^src/libutil/canon-path\.hh$'' + ''^src/libutil/chunked-vector\.hh$'' + ''^src/libutil/closure\.hh$'' + ''^src/libutil/comparator\.hh$'' + ''^src/libutil/compute-levels\.cc$'' + ''^src/libutil/config-impl\.hh$'' + ''^src/libutil/config\.cc$'' + ''^src/libutil/config\.hh$'' + ''^src/libutil/current-process\.cc$'' + ''^src/libutil/current-process\.hh$'' + ''^src/libutil/english\.cc$'' + ''^src/libutil/english\.hh$'' + ''^src/libutil/environment-variables\.cc$'' + ''^src/libutil/error\.cc$'' + ''^src/libutil/error\.hh$'' + ''^src/libutil/exit\.hh$'' + ''^src/libutil/experimental-features\.cc$'' + ''^src/libutil/experimental-features\.hh$'' + ''^src/libutil/file-content-address\.cc$'' + ''^src/libutil/file-content-address\.hh$'' + ''^src/libutil/file-descriptor\.cc$'' + ''^src/libutil/file-descriptor\.hh$'' + ''^src/libutil/file-path-impl\.hh$'' + ''^src/libutil/file-path\.hh$'' + ''^src/libutil/file-system\.cc$'' + ''^src/libutil/file-system\.hh$'' + ''^src/libutil/finally\.hh$'' + ''^src/libutil/fmt\.hh$'' + ''^src/libutil/fs-sink\.cc$'' + ''^src/libutil/fs-sink\.hh$'' + ''^src/libutil/git\.cc$'' + ''^src/libutil/git\.hh$'' + ''^src/libutil/hash\.cc$'' + ''^src/libutil/hash\.hh$'' + ''^src/libutil/hilite\.cc$'' + ''^src/libutil/hilite\.hh$'' + ''^src/libutil/source-accessor\.hh$'' + ''^src/libutil/json-impls\.hh$'' + ''^src/libutil/json-utils\.cc$'' + ''^src/libutil/json-utils\.hh$'' + ''^src/libutil/linux/cgroup\.cc$'' + ''^src/libutil/linux/namespaces\.cc$'' + ''^src/libutil/logging\.cc$'' + ''^src/libutil/logging\.hh$'' + ''^src/libutil/lru-cache\.hh$'' + ''^src/libutil/memory-source-accessor\.cc$'' + ''^src/libutil/memory-source-accessor\.hh$'' + ''^src/libutil/pool\.hh$'' + ''^src/libutil/position\.cc$'' + ''^src/libutil/position\.hh$'' + ''^src/libutil/posix-source-accessor\.cc$'' + ''^src/libutil/posix-source-accessor\.hh$'' + ''^src/libutil/processes\.hh$'' + ''^src/libutil/ref\.hh$'' + ''^src/libutil/references\.cc$'' + ''^src/libutil/references\.hh$'' + ''^src/libutil/regex-combinators\.hh$'' + ''^src/libutil/serialise\.cc$'' + ''^src/libutil/serialise\.hh$'' + ''^src/libutil/signals\.hh$'' + ''^src/libutil/signature/local-keys\.cc$'' + ''^src/libutil/signature/local-keys\.hh$'' + ''^src/libutil/signature/signer\.cc$'' + ''^src/libutil/signature/signer\.hh$'' + ''^src/libutil/source-accessor\.cc$'' + ''^src/libutil/source-accessor\.hh$'' + ''^src/libutil/source-path\.cc$'' + ''^src/libutil/source-path\.hh$'' + ''^src/libutil/split\.hh$'' + ''^src/libutil/suggestions\.cc$'' + ''^src/libutil/suggestions\.hh$'' + ''^src/libutil/sync\.hh$'' + ''^src/libutil/terminal\.cc$'' + ''^src/libutil/terminal\.hh$'' + ''^src/libutil/thread-pool\.cc$'' + ''^src/libutil/thread-pool\.hh$'' + ''^src/libutil/topo-sort\.hh$'' + ''^src/libutil/types\.hh$'' + ''^src/libutil/unix/file-descriptor\.cc$'' + ''^src/libutil/unix/file-path\.cc$'' + ''^src/libutil/unix/monitor-fd\.hh$'' + ''^src/libutil/unix/processes\.cc$'' + ''^src/libutil/unix/signals-impl\.hh$'' + ''^src/libutil/unix/signals\.cc$'' + ''^src/libutil/unix-domain-socket\.cc$'' + ''^src/libutil/unix/users\.cc$'' + ''^src/libutil/url-parts\.hh$'' + ''^src/libutil/url\.cc$'' + ''^src/libutil/url\.hh$'' + ''^src/libutil/users\.cc$'' + ''^src/libutil/users\.hh$'' + ''^src/libutil/util\.cc$'' + ''^src/libutil/util\.hh$'' + ''^src/libutil/variant-wrapper\.hh$'' + ''^src/libutil/windows/environment-variables\.cc$'' + ''^src/libutil/windows/file-descriptor\.cc$'' + ''^src/libutil/windows/file-path\.cc$'' + ''^src/libutil/windows/processes\.cc$'' + ''^src/libutil/windows/users\.cc$'' + ''^src/libutil/windows/windows-error\.cc$'' + ''^src/libutil/windows/windows-error\.hh$'' + ''^src/libutil/xml-writer\.cc$'' + ''^src/libutil/xml-writer\.hh$'' + ''^src/nix-build/nix-build\.cc$'' + ''^src/nix-channel/nix-channel\.cc$'' + ''^src/nix-collect-garbage/nix-collect-garbage\.cc$'' + ''^src/nix-env/buildenv.nix$'' + ''^src/nix-env/nix-env\.cc$'' + ''^src/nix-env/user-env\.cc$'' + ''^src/nix-env/user-env\.hh$'' + ''^src/nix-instantiate/nix-instantiate\.cc$'' + ''^src/nix-store/dotgraph\.cc$'' + ''^src/nix-store/graphml\.cc$'' + ''^src/nix-store/nix-store\.cc$'' + ''^src/nix/add-to-store\.cc$'' + ''^src/nix/app\.cc$'' + ''^src/nix/build\.cc$'' + ''^src/nix/bundle\.cc$'' + ''^src/nix/cat\.cc$'' + ''^src/nix/config-check\.cc$'' + ''^src/nix/config\.cc$'' + ''^src/nix/copy\.cc$'' + ''^src/nix/derivation-add\.cc$'' + ''^src/nix/derivation-show\.cc$'' + ''^src/nix/derivation\.cc$'' + ''^src/nix/develop\.cc$'' + ''^src/nix/diff-closures\.cc$'' + ''^src/nix/dump-path\.cc$'' + ''^src/nix/edit\.cc$'' + ''^src/nix/eval\.cc$'' + ''^src/nix/flake\.cc$'' + ''^src/nix/fmt\.cc$'' + ''^src/nix/hash\.cc$'' + ''^src/nix/log\.cc$'' + ''^src/nix/ls\.cc$'' + ''^src/nix/main\.cc$'' + ''^src/nix/make-content-addressed\.cc$'' + ''^src/nix/nar\.cc$'' + ''^src/nix/optimise-store\.cc$'' + ''^src/nix/path-from-hash-part\.cc$'' + ''^src/nix/path-info\.cc$'' + ''^src/nix/prefetch\.cc$'' + ''^src/nix/profile\.cc$'' + ''^src/nix/realisation\.cc$'' + ''^src/nix/registry\.cc$'' + ''^src/nix/repl\.cc$'' + ''^src/nix/run\.cc$'' + ''^src/nix/run\.hh$'' + ''^src/nix/search\.cc$'' + ''^src/nix/sigs\.cc$'' + ''^src/nix/store-copy-log\.cc$'' + ''^src/nix/store-delete\.cc$'' + ''^src/nix/store-gc\.cc$'' + ''^src/nix/store-info\.cc$'' + ''^src/nix/store-repair\.cc$'' + ''^src/nix/store\.cc$'' + ''^src/nix/unix/daemon\.cc$'' + ''^src/nix/upgrade-nix\.cc$'' + ''^src/nix/verify\.cc$'' + ''^src/nix/why-depends\.cc$'' + + ''^tests/functional/plugins/plugintest\.cc'' + ''^tests/functional/test-libstoreconsumer/main\.cc'' + ''^tests/nixos/ca-fd-leak/sender\.c'' + ''^tests/nixos/ca-fd-leak/smuggler\.c'' + ''^tests/nixos/user-sandboxing/attacker\.c'' + ''^tests/unit/libexpr-support/tests/libexpr\.hh'' + ''^tests/unit/libexpr-support/tests/value/context\.cc'' + ''^tests/unit/libexpr-support/tests/value/context\.hh'' + ''^tests/unit/libexpr/derived-path\.cc'' + ''^tests/unit/libexpr/error_traces\.cc'' + ''^tests/unit/libexpr/eval\.cc'' + ''^tests/unit/libexpr/json\.cc'' + ''^tests/unit/libexpr/main\.cc'' + ''^tests/unit/libexpr/primops\.cc'' + ''^tests/unit/libexpr/search-path\.cc'' + ''^tests/unit/libexpr/trivial\.cc'' + ''^tests/unit/libexpr/value/context\.cc'' + ''^tests/unit/libexpr/value/print\.cc'' + ''^tests/unit/libfetchers/public-key\.cc'' + ''^tests/unit/libflake/flakeref\.cc'' + ''^tests/unit/libflake/url-name\.cc'' + ''^tests/unit/libstore-support/tests/derived-path\.cc'' + ''^tests/unit/libstore-support/tests/derived-path\.hh'' + ''^tests/unit/libstore-support/tests/nix_api_store\.hh'' + ''^tests/unit/libstore-support/tests/outputs-spec\.cc'' + ''^tests/unit/libstore-support/tests/outputs-spec\.hh'' + ''^tests/unit/libstore-support/tests/path\.cc'' + ''^tests/unit/libstore-support/tests/path\.hh'' + ''^tests/unit/libstore-support/tests/protocol\.hh'' + ''^tests/unit/libstore/common-protocol\.cc'' + ''^tests/unit/libstore/content-address\.cc'' + ''^tests/unit/libstore/derivation\.cc'' + ''^tests/unit/libstore/derived-path\.cc'' + ''^tests/unit/libstore/downstream-placeholder\.cc'' + ''^tests/unit/libstore/machines\.cc'' + ''^tests/unit/libstore/nar-info-disk-cache\.cc'' + ''^tests/unit/libstore/nar-info\.cc'' + ''^tests/unit/libstore/outputs-spec\.cc'' + ''^tests/unit/libstore/path-info\.cc'' + ''^tests/unit/libstore/path\.cc'' + ''^tests/unit/libstore/serve-protocol\.cc'' + ''^tests/unit/libstore/worker-protocol\.cc'' + ''^tests/unit/libutil-support/tests/characterization\.hh'' + ''^tests/unit/libutil-support/tests/hash\.cc'' + ''^tests/unit/libutil-support/tests/hash\.hh'' + ''^tests/unit/libutil/args\.cc'' + ''^tests/unit/libutil/canon-path\.cc'' + ''^tests/unit/libutil/chunked-vector\.cc'' + ''^tests/unit/libutil/closure\.cc'' + ''^tests/unit/libutil/compression\.cc'' + ''^tests/unit/libutil/config\.cc'' + ''^tests/unit/libutil/file-content-address\.cc'' + ''^tests/unit/libutil/git\.cc'' + ''^tests/unit/libutil/hash\.cc'' + ''^tests/unit/libutil/hilite\.cc'' + ''^tests/unit/libutil/json-utils\.cc'' + ''^tests/unit/libutil/logging\.cc'' + ''^tests/unit/libutil/lru-cache\.cc'' + ''^tests/unit/libutil/pool\.cc'' + ''^tests/unit/libutil/references\.cc'' + ''^tests/unit/libutil/suggestions\.cc'' + ''^tests/unit/libutil/tests\.cc'' + ''^tests/unit/libutil/url\.cc'' + ''^tests/unit/libutil/xml-writer\.cc'' + ]; + }; + shellcheck = { + enable = true; + excludes = [ + # We haven't linted these files yet + ''^config/install-sh$'' + ''^misc/bash/completion\.sh$'' + ''^misc/fish/completion\.fish$'' + ''^misc/zsh/completion\.zsh$'' + ''^scripts/create-darwin-volume\.sh$'' + ''^scripts/install-darwin-multi-user\.sh$'' + ''^scripts/install-multi-user\.sh$'' + ''^scripts/install-nix-from-closure\.sh$'' + ''^scripts/install-systemd-multi-user\.sh$'' + ''^src/nix/get-env\.sh$'' + ''^tests/functional/build\.sh$'' + ''^tests/functional/ca/build-dry\.sh$'' + ''^tests/functional/ca/build-with-garbage-path\.sh$'' + ''^tests/functional/ca/common\.sh$'' + ''^tests/functional/ca/concurrent-builds\.sh$'' + ''^tests/functional/ca/eval-store\.sh$'' + ''^tests/functional/ca/gc\.sh$'' + ''^tests/functional/ca/import-derivation\.sh$'' + ''^tests/functional/ca/new-build-cmd\.sh$'' + ''^tests/functional/ca/nix-shell\.sh$'' + ''^tests/functional/ca/post-hook\.sh$'' + ''^tests/functional/ca/recursive\.sh$'' + ''^tests/functional/ca/repl\.sh$'' + ''^tests/functional/ca/selfref-gc\.sh$'' + ''^tests/functional/ca/why-depends\.sh$'' + ''^tests/functional/characterisation-test-infra\.sh$'' + ''^tests/functional/check\.sh$'' + ''^tests/functional/common/vars-and-functions\.sh$'' + ''^tests/functional/completions\.sh$'' + ''^tests/functional/compute-levels\.sh$'' + ''^tests/functional/config\.sh$'' + ''^tests/functional/db-migration\.sh$'' + ''^tests/functional/debugger\.sh$'' + ''^tests/functional/dependencies\.builder0\.sh$'' + ''^tests/functional/dependencies\.sh$'' + ''^tests/functional/dump-db\.sh$'' + ''^tests/functional/dyn-drv/build-built-drv\.sh$'' + ''^tests/functional/dyn-drv/common\.sh$'' + ''^tests/functional/dyn-drv/dep-built-drv\.sh$'' + ''^tests/functional/dyn-drv/eval-outputOf\.sh$'' + ''^tests/functional/dyn-drv/old-daemon-error-hack\.sh$'' + ''^tests/functional/dyn-drv/recursive-mod-json\.sh$'' + ''^tests/functional/eval-store\.sh$'' + ''^tests/functional/eval\.sh$'' + ''^tests/functional/export-graph\.sh$'' + ''^tests/functional/export\.sh$'' + ''^tests/functional/extra-sandbox-profile\.sh$'' + ''^tests/functional/fetchClosure\.sh$'' + ''^tests/functional/fetchGit\.sh$'' + ''^tests/functional/fetchGitRefs\.sh$'' + ''^tests/functional/fetchGitSubmodules\.sh$'' + ''^tests/functional/fetchGitVerification\.sh$'' + ''^tests/functional/fetchMercurial\.sh$'' + ''^tests/functional/fetchurl\.sh$'' + ''^tests/functional/fixed\.builder1\.sh$'' + ''^tests/functional/fixed\.builder2\.sh$'' + ''^tests/functional/fixed\.sh$'' + ''^tests/functional/flakes/absolute-paths\.sh$'' + ''^tests/functional/flakes/check\.sh$'' + ''^tests/functional/flakes/common\.sh$'' + ''^tests/functional/flakes/config\.sh$'' + ''^tests/functional/flakes/develop\.sh$'' + ''^tests/functional/flakes/flakes\.sh$'' + ''^tests/functional/flakes/follow-paths\.sh$'' + ''^tests/functional/flakes/prefetch\.sh$'' + ''^tests/functional/flakes/run\.sh$'' + ''^tests/functional/flakes/show\.sh$'' + ''^tests/functional/fmt\.sh$'' + ''^tests/functional/fmt\.simple\.sh$'' + ''^tests/functional/gc-auto\.sh$'' + ''^tests/functional/gc-concurrent\.builder\.sh$'' + ''^tests/functional/gc-concurrent\.sh$'' + ''^tests/functional/gc-concurrent2\.builder\.sh$'' + ''^tests/functional/gc-non-blocking\.sh$'' + ''^tests/functional/gc\.sh$'' + ''^tests/functional/git-hashing/common\.sh$'' + ''^tests/functional/git-hashing/simple\.sh$'' + ''^tests/functional/hash-convert\.sh$'' + ''^tests/functional/help\.sh$'' + ''^tests/functional/impure-derivations\.sh$'' + ''^tests/functional/impure-env\.sh$'' + ''^tests/functional/impure-eval\.sh$'' + ''^tests/functional/install-darwin\.sh$'' + ''^tests/functional/lang\.sh$'' + ''^tests/functional/legacy-ssh-store\.sh$'' + ''^tests/functional/linux-sandbox\.sh$'' + ''^tests/functional/local-overlay-store/add-lower-inner\.sh$'' + ''^tests/functional/local-overlay-store/add-lower\.sh$'' + ''^tests/functional/local-overlay-store/bad-uris\.sh$'' + ''^tests/functional/local-overlay-store/build-inner\.sh$'' + ''^tests/functional/local-overlay-store/build\.sh$'' + ''^tests/functional/local-overlay-store/check-post-init-inner\.sh$'' + ''^tests/functional/local-overlay-store/check-post-init\.sh$'' + ''^tests/functional/local-overlay-store/common\.sh$'' + ''^tests/functional/local-overlay-store/delete-duplicate-inner\.sh$'' + ''^tests/functional/local-overlay-store/delete-duplicate\.sh$'' + ''^tests/functional/local-overlay-store/delete-refs-inner\.sh$'' + ''^tests/functional/local-overlay-store/delete-refs\.sh$'' + ''^tests/functional/local-overlay-store/gc-inner\.sh$'' + ''^tests/functional/local-overlay-store/gc\.sh$'' + ''^tests/functional/local-overlay-store/optimise-inner\.sh$'' + ''^tests/functional/local-overlay-store/optimise\.sh$'' + ''^tests/functional/local-overlay-store/redundant-add-inner\.sh$'' + ''^tests/functional/local-overlay-store/redundant-add\.sh$'' + ''^tests/functional/local-overlay-store/remount\.sh$'' + ''^tests/functional/local-overlay-store/stale-file-handle-inner\.sh$'' + ''^tests/functional/local-overlay-store/stale-file-handle\.sh$'' + ''^tests/functional/local-overlay-store/verify-inner\.sh$'' + ''^tests/functional/local-overlay-store/verify\.sh$'' + ''^tests/functional/logging\.sh$'' + ''^tests/functional/misc\.sh$'' + ''^tests/functional/multiple-outputs\.sh$'' + ''^tests/functional/nar-access\.sh$'' + ''^tests/functional/nested-sandboxing\.sh$'' + ''^tests/functional/nested-sandboxing/command\.sh$'' + ''^tests/functional/nix-build\.sh$'' + ''^tests/functional/nix-channel\.sh$'' + ''^tests/functional/nix-collect-garbage-d\.sh$'' + ''^tests/functional/nix-copy-ssh-common\.sh$'' + ''^tests/functional/nix-copy-ssh-ng\.sh$'' + ''^tests/functional/nix-copy-ssh\.sh$'' + ''^tests/functional/nix-daemon-untrusting\.sh$'' + ''^tests/functional/nix-profile\.sh$'' + ''^tests/functional/nix-shell\.sh$'' + ''^tests/functional/nix_path\.sh$'' + ''^tests/functional/optimise-store\.sh$'' + ''^tests/functional/output-normalization\.sh$'' + ''^tests/functional/parallel\.builder\.sh$'' + ''^tests/functional/parallel\.sh$'' + ''^tests/functional/pass-as-file\.sh$'' + ''^tests/functional/path-from-hash-part\.sh$'' + ''^tests/functional/path-info\.sh$'' + ''^tests/functional/placeholders\.sh$'' + ''^tests/functional/plugins\.sh$'' + ''^tests/functional/post-hook\.sh$'' + ''^tests/functional/pure-eval\.sh$'' + ''^tests/functional/push-to-store-old\.sh$'' + ''^tests/functional/push-to-store\.sh$'' + ''^tests/functional/read-only-store\.sh$'' + ''^tests/functional/readfile-context\.sh$'' + ''^tests/functional/recursive\.sh$'' + ''^tests/functional/referrers\.sh$'' + ''^tests/functional/remote-store\.sh$'' + ''^tests/functional/repair\.sh$'' + ''^tests/functional/restricted\.sh$'' + ''^tests/functional/search\.sh$'' + ''^tests/functional/secure-drv-outputs\.sh$'' + ''^tests/functional/selfref-gc\.sh$'' + ''^tests/functional/shell\.sh$'' + ''^tests/functional/shell\.shebang\.sh$'' + ''^tests/functional/signing\.sh$'' + ''^tests/functional/simple\.builder\.sh$'' + ''^tests/functional/simple\.sh$'' + ''^tests/functional/ssh-relay\.sh$'' + ''^tests/functional/store-info\.sh$'' + ''^tests/functional/structured-attrs\.sh$'' + ''^tests/functional/substitute-with-invalid-ca\.sh$'' + ''^tests/functional/suggestions\.sh$'' + ''^tests/functional/supplementary-groups\.sh$'' + ''^tests/functional/tarball\.sh$'' + ''^tests/functional/test-infra\.sh$'' + ''^tests/functional/test-libstoreconsumer\.sh$'' + ''^tests/functional/timeout\.sh$'' + ''^tests/functional/toString-path\.sh$'' + ''^tests/functional/user-envs-migration\.sh$'' + ''^tests/functional/user-envs-test-case\.sh$'' + ''^tests/functional/user-envs\.builder\.sh$'' + ''^tests/functional/user-envs\.sh$'' + ''^tests/functional/why-depends\.sh$'' + ''^tests/functional/zstd\.sh$'' + ''^tests/unit/libutil/data/git/check-data\.sh$'' + ]; + }; + # TODO: nixfmt, https://github.com/NixOS/nixfmt/issues/153 + }; + }; + }; + + # We'll be pulling from this in the main flake + flake.getSystem = getSystem; +} diff --git a/maintainers/local.mk b/maintainers/local.mk new file mode 100644 index 000000000..88d594d67 --- /dev/null +++ b/maintainers/local.mk @@ -0,0 +1,15 @@ + +.PHONY: format +print-top-help += echo ' format: Format source code' + +# This uses the cached .pre-commit-hooks.yaml file +format: + @if ! type -p pre-commit &>/dev/null; then \ + echo "make format: pre-commit not found. Please use \`nix develop\`."; \ + exit 1; \ + fi; \ + if test -z "$$_NIX_PRE_COMMIT_HOOKS_CONFIG"; then \ + echo "make format: _NIX_PRE_COMMIT_HOOKS_CONFIG not set. Please use \`nix develop\`."; \ + exit 1; \ + fi; \ + pre-commit run --config $$_NIX_PRE_COMMIT_HOOKS_CONFIG --all-files diff --git a/maintainers/release-credits b/maintainers/release-credits new file mode 100755 index 000000000..7a5c87d7d --- /dev/null +++ b/maintainers/release-credits @@ -0,0 +1,184 @@ +#!/usr/bin/env nix +# vim: set filetype=python: +#!nix develop --impure --expr +#!nix `` +#!nix let flake = builtins.getFlake ("git+file://" + toString ../.); +#!nix pkgs = flake.inputs.nixpkgs.legacyPackages.${builtins.currentSystem}; +#!nix in pkgs.mkShell { nativeBuildInputs = [ +#!nix (pkgs.python3.withPackages (ps: with ps; [ requests ])) +#!nix ]; } +#!nix `` --command python3 + +# This script lists out the contributors for a given release. +# It must be run from the root of the Nix repository. + +import os +import sys +import json +import requests + +github_token = os.environ.get("GITHUB_TOKEN") +if not github_token: + print("GITHUB_TOKEN is not set. If you hit the rate limit, set it", file=sys.stderr) + # Might be ok, as we have a cache. + # raise ValueError("GITHUB_TOKEN must be set") + +# 1. Read the current version in .version +version = os.environ.get("VERSION") +if not version: + version = open(".version").read().strip() + +print(f"Generating release credits for Nix {version}", file=sys.stderr) + +# 2. Compute previous version +vcomponents = version.split(".") +if len(vcomponents) >= 2: + prev_version = f"{vcomponents[0]}.{int(vcomponents[1])-1}.0" +else: + raise ValueError(".version must have at least two components") + +# For unreleased versions +endref = "HEAD" +# For older releases +# endref = version + +# 2. Find the merge base between the current version and the previous version +mergeBase = os.popen(f"git merge-base {prev_version} {endref}").read().strip() +print(f"Merge base between {prev_version} and {endref} is {mergeBase}", file=sys.stderr) + +# 3. Find the date of the merge base +mergeBaseDate = os.popen(f"git show -s --format=%ci {mergeBase}").read().strip()[0:10] +print(f"Merge base date is {mergeBaseDate}", file=sys.stderr) + +# 4. Get the commits between the merge base and the current version + +def get_commits(): + raw = os.popen(f"git log --pretty=format:'%H\t%an\t%ae' {mergeBase}..{endref}").read().strip() + lines = raw.split("\n") + return [ { "hash": items[0], "author": items[1], "email": items[2] } + for line in lines + for items in (line.split("\t"),) + ] + +def commits_to_first_commit_by_email(commits): + by_email = dict() + for commit in commits: + email = commit["email"] + if email not in by_email: + by_email[email] = commit + return by_email + + +samples = commits_to_first_commit_by_email(get_commits()) + +# For quick testing, only pick two samples from the dict +# samples = dict(list(samples.items())[:2]) + +# Query the GitHub API to get handle +def get_github_commit(commit): + url = f"https://api.github.com/repos/NixOS/nix/commits/{commit['hash']}" + headers = {'Authorization': f'token {github_token}'} + response = requests.get(url, headers=headers) + response.raise_for_status() + return response.json() + +class Cache: + def __init__(self, filename, require = True): + self.filename = filename + try: + with open(filename, "r") as f: + self.values = json.load(f) + except FileNotFoundError: + if require: + raise + self.values = dict() + def save(self): + with open(self.filename, "w") as f: + json.dump(self.values, f, indent=4) + print(f"Saved cache to {self.filename}", file=sys.stderr) + +# The email to handle cache maps email addresses to either +# - a handle (string) +# - None (if no handle was found) +email_to_handle_cache = Cache("maintainers/data/release-credits-email-to-handle.json") + +handles = set() +emails = dict() + +for sample in samples: + s = samples[sample] + email = s["email"] + if not email in email_to_handle_cache.values: + print(f"Querying GitHub API for {s['hash']}, to get handle for {s['email']}") + ghc = get_github_commit(samples[sample]) + gha = ghc["author"] + if gha and gha["login"]: + handle = gha["login"] + print(f"Handle: {handle}") + email_to_handle_cache.values[email] = handle + else: + print(f"Found no handle for {s['email']}") + email_to_handle_cache.values[email] = None + handle = email_to_handle_cache.values[email] + if handle is not None: + handles.add(handle) + else: + emails[email] = s["author"] + +# print(email_to_handle_cache.values) + +email_to_handle_cache.save() + +handle_to_name_cache = Cache("maintainers/data/release-credits-handle-to-name.json") + +print(f"Found {len(handles)} handles", file=sys.stderr) + +for handle in handles: + if not handle in handle_to_name_cache.values: + print(f"Querying GitHub API for {handle}, to get name", file=sys.stderr) + url = f"https://api.github.com/users/{handle}" + headers = {'Authorization': f'token {github_token}'} + response = requests.get(url, headers=headers) + response.raise_for_status() + user = response.json() + name = user["name"] + print(f"Name: {name}", file=sys.stderr) + handle_to_name_cache.values[handle] = name + +handle_to_name_cache.save() + +entries = list() + +def is_bot(handle): + return handle.endswith("[bot]") + +for handle in handles: + name = handle_to_name_cache.values[handle] + if name is None: + # This way it looks more regular + name = handle + + # Note to bots: I'm only doing this because it would distract humans, and + # therefore delay your creation. I'm sorry; it's for your benefit. + if not is_bot(handle): + entries += [ f"- {name} [**(@{handle})**](https://github.com/{handle})" ] + +def shuffle(entries): + salt = os.urandom(16) + return sorted(entries, key=lambda x: hash((x, salt))) + +# Fair ordering is undecidable +entries = shuffle(entries) + +# For a sanity check, we could sort the entries by handle instead. +# entries = sorted(entries) + +print("") +print(f"This release was made possible by the following {len(entries)} contributors:") +print("") + +for entry in entries: + print(entry) + +for email in emails: + print(f"- {emails[email]}") diff --git a/maintainers/release-notes b/maintainers/release-notes index 2d84485c1..c0c4ee734 100755 --- a/maintainers/release-notes +++ b/maintainers/release-notes @@ -1,5 +1,6 @@ #!/usr/bin/env nix -#!nix shell .#changelog-d-nix --command bash +# vim: set filetype=bash: +#!nix shell .#changelog-d --command bash # --- CONFIGURATION --- @@ -151,6 +152,13 @@ section_title="Release $version_full ($DATE)" echo "# $section_title" echo changelog-d doc/manual/rl-next | sed -e 's/ *$//' + + if ! $IS_PATCH; then + echo + echo "# Contributors" + echo + VERSION=$version_full ./maintainers/release-credits + fi ) | tee -a $file log "Wrote $file" diff --git a/maintainers/release-process.md b/maintainers/release-process.md index da6886ea9..7a2b3c0a7 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -39,6 +39,10 @@ release: * Proof-read / edit / rearrange the release notes if needed. Breaking changes and highlights should go to the top. +* Run `maintainers/release-credits` to make sure the credits script works + and produces a sensible output. Some emails might not automatically map to + a GitHub handle. + * Push. ```console diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 4e2c379f0..731988568 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -11,6 +11,8 @@ use JSON::PP; use LWP::UserAgent; use Net::Amazon::S3; +delete $ENV{'shell'}; # shut up a LWP::UserAgent.pm warning + my $evalId = $ARGV[0] or die "Usage: $0 EVAL-ID\n"; my $releasesBucketName = "nix-releases"; @@ -36,11 +38,11 @@ sub fetch { my $evalUrl = "https://hydra.nixos.org/eval/$evalId"; my $evalInfo = decode_json(fetch($evalUrl, 'application/json')); #print Dumper($evalInfo); -my $flakeUrl = $evalInfo->{flake} or die; -my $flakeInfo = decode_json(`nix flake metadata --json "$flakeUrl"` or die); -my $nixRev = $flakeInfo->{revision} or die; +my $flakeUrl = $evalInfo->{flake}; +my $flakeInfo = decode_json(`nix flake metadata --json "$flakeUrl"` or die) if $flakeUrl; +my $nixRev = ($flakeInfo ? $flakeInfo->{revision} : $evalInfo->{jobsetevalinputs}->{nix}->{revision}) or die; -my $buildInfo = decode_json(fetch("$evalUrl/job/build.x86_64-linux", 'application/json')); +my $buildInfo = decode_json(fetch("$evalUrl/job/build.nix.x86_64-linux", 'application/json')); #print Dumper($buildInfo); my $releaseName = $buildInfo->{nixname}; @@ -83,12 +85,19 @@ my $channelsBucket = $s3_us->bucket($channelsBucketName) or die; sub getStorePath { my ($jobName, $output) = @_; my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json')); - return $buildInfo->{buildoutputs}->{$output or "out"}->{path} or die "cannot get store path for '$jobName'"; + return $buildInfo->{buildoutputs}->{$output or "out"}->{path} // die "cannot get store path for '$jobName'"; } sub copyManual { - my $manual = getStorePath("build.x86_64-linux", "doc"); - print "$manual\n"; + my $manual; + eval { + $manual = getStorePath("build.nix.x86_64-linux", "doc"); + }; + if ($@) { + warn "$@"; + return; + } + print "Manual: $manual\n"; my $manualNar = "$tmpDir/$releaseName-manual.nar.xz"; print "$manualNar\n"; @@ -154,19 +163,34 @@ downloadFile("binaryTarball.x86_64-linux", "1"); downloadFile("binaryTarball.aarch64-linux", "1"); downloadFile("binaryTarball.x86_64-darwin", "1"); downloadFile("binaryTarball.aarch64-darwin", "1"); -downloadFile("binaryTarballCross.x86_64-linux.armv6l-unknown-linux-gnueabihf", "1"); -downloadFile("binaryTarballCross.x86_64-linux.armv7l-unknown-linux-gnueabihf", "1"); +eval { + downloadFile("binaryTarballCross.x86_64-linux.armv6l-unknown-linux-gnueabihf", "1"); +}; +warn "$@" if $@; +eval { + downloadFile("binaryTarballCross.x86_64-linux.armv7l-unknown-linux-gnueabihf", "1"); +}; +warn "$@" if $@; +eval { + downloadFile("binaryTarballCross.x86_64-linux.riscv64-unknown-linux-gnu", "1"); +}; +warn "$@" if $@; downloadFile("installerScript", "1"); # Upload docker images to dockerhub. my $dockerManifest = ""; my $dockerManifestLatest = ""; +my $haveDocker = 0; for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) { my $system = $platforms->[0]; my $dockerPlatform = $platforms->[1]; my $fn = "nix-$version-docker-image-$dockerPlatform.tar.gz"; - downloadFile("dockerImage.$system", "1", $fn); + eval { + downloadFile("dockerImage.$system", "1", $fn); + }; + die "$@" if $@; + $haveDocker = 1; print STDERR "loading docker image for $dockerPlatform...\n"; system("docker load -i $tmpDir/$fn") == 0 or die; @@ -194,31 +218,34 @@ for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) { $dockerManifestLatest .= " --amend $latestTag" } -print STDERR "creating multi-platform docker manifest...\n"; -system("docker manifest rm nixos/nix:$version"); -system("docker manifest create nixos/nix:$version $dockerManifest") == 0 or die; -if ($isLatest) { - print STDERR "creating latest multi-platform docker manifest...\n"; - system("docker manifest rm nixos/nix:latest"); - system("docker manifest create nixos/nix:latest $dockerManifestLatest") == 0 or die; -} +if ($haveDocker) { + print STDERR "creating multi-platform docker manifest...\n"; + system("docker manifest rm nixos/nix:$version"); + system("docker manifest create nixos/nix:$version $dockerManifest") == 0 or die; + if ($isLatest) { + print STDERR "creating latest multi-platform docker manifest...\n"; + system("docker manifest rm nixos/nix:latest"); + system("docker manifest create nixos/nix:latest $dockerManifestLatest") == 0 or die; + } -print STDERR "pushing multi-platform docker manifest...\n"; -system("docker manifest push nixos/nix:$version") == 0 or die; + print STDERR "pushing multi-platform docker manifest...\n"; + system("docker manifest push nixos/nix:$version") == 0 or die; -if ($isLatest) { - print STDERR "pushing latest multi-platform docker manifest...\n"; - system("docker manifest push nixos/nix:latest") == 0 or die; + if ($isLatest) { + print STDERR "pushing latest multi-platform docker manifest...\n"; + system("docker manifest push nixos/nix:latest") == 0 or die; + } } # Upload nix-fallback-paths.nix. write_file("$tmpDir/fallback-paths.nix", "{\n" . - " x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" . - " i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" . - " aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" . - " x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" . - " aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" . + " x86_64-linux = \"" . getStorePath("build.nix.x86_64-linux") . "\";\n" . + " i686-linux = \"" . getStorePath("build.nix.i686-linux") . "\";\n" . + " aarch64-linux = \"" . getStorePath("build.nix.aarch64-linux") . "\";\n" . + " riscv64-linux = \"" . getStorePath("buildCross.nix.riscv64-unknown-linux-gnu.x86_64-linux") . "\";\n" . + " x86_64-darwin = \"" . getStorePath("build.nix.x86_64-darwin") . "\";\n" . + " aarch64-darwin = \"" . getStorePath("build.nix.aarch64-darwin") . "\";\n" . "}\n"); # Upload release files to S3. diff --git a/meson.build b/meson.build new file mode 100644 index 000000000..1554244ab --- /dev/null +++ b/meson.build @@ -0,0 +1,44 @@ +# This is just a stub project to include all the others as subprojects +# for development shell purposes + +project('nix-dev-shell', 'cpp', + version : files('.version'), + subproject_dir : 'src', +) + +# Internal Libraries +subproject('libutil') +subproject('libstore') +subproject('libfetchers') +subproject('libexpr') +subproject('libflake') +subproject('libmain') +subproject('libcmd') + +# Executables +subproject('nix') + +# Docs +subproject('internal-api-docs') +subproject('external-api-docs') + +# External C wrapper libraries +subproject('libutil-c') +subproject('libstore-c') +subproject('libexpr-c') +subproject('libmain-c') + +# Language Bindings +if not meson.is_cross_build() + subproject('perl') +endif + +# Testing +subproject('nix-util-test-support') +subproject('nix-util-tests') +subproject('nix-store-test-support') +subproject('nix-store-tests') +subproject('nix-fetchers-tests') +subproject('nix-expr-test-support') +subproject('nix-expr-tests') +subproject('nix-flake-tests') diff --git a/misc/changelog-d.cabal.nix b/misc/changelog-d.cabal.nix deleted file mode 100644 index 76f9353cd..000000000 --- a/misc/changelog-d.cabal.nix +++ /dev/null @@ -1,31 +0,0 @@ -{ mkDerivation, aeson, base, bytestring, cabal-install-parsers -, Cabal-syntax, containers, directory, filepath, frontmatter -, generic-lens-lite, lib, mtl, optparse-applicative, parsec, pretty -, regex-applicative, text, pkgs -}: -let rev = "f30f6969e9cd8b56242309639d58acea21c99d06"; -in -mkDerivation { - pname = "changelog-d"; - version = "0.1"; - src = pkgs.fetchurl { - name = "changelog-d-${rev}.tar.gz"; - url = "https://codeberg.org/roberth/changelog-d/archive/${rev}.tar.gz"; - hash = "sha256-8a2+i5u7YoszAgd5OIEW0eYUcP8yfhtoOIhLJkylYJ4="; - } // { inherit rev; }; - isLibrary = false; - isExecutable = true; - libraryHaskellDepends = [ - aeson base bytestring cabal-install-parsers Cabal-syntax containers - directory filepath frontmatter generic-lens-lite mtl parsec pretty - regex-applicative text - ]; - executableHaskellDepends = [ - base bytestring Cabal-syntax directory filepath - optparse-applicative - ]; - doHaddock = false; - description = "Concatenate changelog entries into a single one"; - license = lib.licenses.gpl3Plus; - mainProgram = "changelog-d"; -} diff --git a/misc/changelog-d.nix b/misc/changelog-d.nix deleted file mode 100644 index 1b20f4596..000000000 --- a/misc/changelog-d.nix +++ /dev/null @@ -1,31 +0,0 @@ -# Taken temporarily from -{ - callPackage, - lib, - haskell, - haskellPackages, -}: - -let - hsPkg = haskellPackages.callPackage ./changelog-d.cabal.nix { }; - - addCompletions = haskellPackages.generateOptparseApplicativeCompletions ["changelog-d"]; - - haskellModifications = - lib.flip lib.pipe [ - addCompletions - haskell.lib.justStaticExecutables - ]; - - mkDerivationOverrides = finalAttrs: oldAttrs: { - - version = oldAttrs.version + "-git-${lib.strings.substring 0 7 oldAttrs.src.rev}"; - - meta = oldAttrs.meta // { - homepage = "https://codeberg.org/roberth/changelog-d"; - maintainers = [ lib.maintainers.roberth ]; - }; - - }; -in - (haskellModifications hsPkg).overrideAttrs mkDerivationOverrides diff --git a/misc/systemv/nix-daemon b/misc/systemv/nix-daemon index fea537167..e8326f947 100755 --- a/misc/systemv/nix-daemon +++ b/misc/systemv/nix-daemon @@ -34,6 +34,7 @@ else fi # Source function library. +# shellcheck source=/dev/null . /etc/init.d/functions LOCKFILE=/var/lock/subsys/nix-daemon @@ -41,14 +42,20 @@ RUNDIR=/var/run/nix PIDFILE=${RUNDIR}/nix-daemon.pid RETVAL=0 -base=${0##*/} +# https://www.shellcheck.net/wiki/SC3004 +# Check if gettext exists +if ! type gettext > /dev/null 2>&1 +then + # If not, create a dummy function that returns the input verbatim + gettext() { printf '%s' "$1"; } +fi start() { mkdir -p ${RUNDIR} chown ${NIX_DAEMON_USER}:${NIX_DAEMON_USER} ${RUNDIR} - echo -n $"Starting nix daemon... " + printf '%s' "$(gettext 'Starting nix daemon... ')" daemonize -u $NIX_DAEMON_USER -p ${PIDFILE} $NIX_DAEMON_BIN $NIX_DAEMON_OPTS RETVAL=$? @@ -58,7 +65,7 @@ start() { } stop() { - echo -n $"Shutting down nix daemon: " + printf '%s' "$(gettext 'Shutting down nix daemon: ')" killproc -p ${PIDFILE} $NIX_DAEMON_BIN RETVAL=$? [ $RETVAL -eq 0 ] && rm -f ${LOCKFILE} ${PIDFILE} @@ -67,7 +74,7 @@ stop() { } reload() { - echo -n $"Reloading nix daemon... " + printf '%s' "$(gettext 'Reloading nix daemon... ')" killproc -p ${PIDFILE} $NIX_DAEMON_BIN -HUP RETVAL=$? echo @@ -105,7 +112,7 @@ case "$1" in fi ;; *) - echo $"Usage: $0 {start|stop|status|restart|condrestart}" + printf '%s' "$(gettext "Usage: $0 {start|stop|status|restart|condrestart}")" exit 2 ;; esac diff --git a/mk/common-test.sh b/mk/common-test.sh index 2783d293b..817422c40 100644 --- a/mk/common-test.sh +++ b/mk/common-test.sh @@ -1,27 +1,23 @@ +# shellcheck shell=bash + # Remove overall test dir (at most one of the two should match) and # remove file extension. -test_name=$(echo -n "$test" | sed \ - -e "s|^tests/unit/[^/]*/data/||" \ + +test_name=$(echo -n "${test?must be defined by caller (test runner)}" | sed \ + -e "s|^src/[^/]*-test/data/||" \ -e "s|^tests/functional/||" \ -e "s|\.sh$||" \ ) +# shellcheck disable=SC2016 TESTS_ENVIRONMENT=( "TEST_NAME=$test_name" 'NIX_REMOTE=' 'PS4=+(${BASH_SOURCE[0]-$0}:$LINENO) ' ) -: ${BASH:=/usr/bin/env bash} +read -r -a bash <<< "${BASH:-/usr/bin/env bash}" run () { - cd "$(dirname $1)" && env "${TESTS_ENVIRONMENT[@]}" $BASH -x -e -u -o pipefail $(basename $1) -} - -init_test () { - run "$init" 2>/dev/null > /dev/null -} - -run_test_proper () { - run "$test" + cd "$(dirname "$1")" && env "${TESTS_ENVIRONMENT[@]}" "${bash[@]}" -x -e -u -o pipefail "$(basename "$1")" } diff --git a/mk/compilation-database.mk b/mk/compilation-database.mk new file mode 100644 index 000000000..f69dc0de0 --- /dev/null +++ b/mk/compilation-database.mk @@ -0,0 +1,11 @@ +compile-commands-json-files := + +define write-compile-commands + _srcs := $$(sort $$(foreach src, $$($(1)_SOURCES), $$(src))) + + $(1)_COMPILE_COMMANDS_JSON := $$(addprefix $(buildprefix), $$(addsuffix .compile_commands.json, $$(basename $$(_srcs)))) + + compile-commands-json-files += $$($(1)_COMPILE_COMMANDS_JSON) + + clean-files += $$($(1)_COMPILE_COMMANDS_JSON) +endef diff --git a/mk/cxx-big-literal.mk b/mk/cxx-big-literal.mk index 85365df8e..d64a171c8 100644 --- a/mk/cxx-big-literal.mk +++ b/mk/cxx-big-literal.mk @@ -1,5 +1,5 @@ %.gen.hh: % - @echo 'R"foo(' >> $@.tmp + @echo 'R"__NIX_STR(' >> $@.tmp $(trace-gen) cat $< >> $@.tmp - @echo ')foo"' >> $@.tmp + @echo ')__NIX_STR"' >> $@.tmp @mv $@.tmp $@ diff --git a/mk/debug-test.sh b/mk/debug-test.sh index 52482c01e..0dd4406c3 100755 --- a/mk/debug-test.sh +++ b/mk/debug-test.sh @@ -3,12 +3,8 @@ set -eu -o pipefail test=$1 -init=${2-} dir="$(dirname "${BASH_SOURCE[0]}")" source "$dir/common-test.sh" -if [ -n "$init" ]; then - (init_test) -fi -run_test_proper +run "$test" diff --git a/mk/lib.mk b/mk/lib.mk index fe0add1c9..1e7af6ad5 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -68,6 +68,7 @@ include mk/patterns.mk include mk/templates.mk include mk/cxx-big-literal.mk include mk/tests.mk +include mk/compilation-database.mk # Include all sub-Makefiles. @@ -86,17 +87,23 @@ $(foreach script, $(bin-scripts), $(eval $(call install-program-in,$(script),$(b $(foreach script, $(bin-scripts), $(eval programs-list += $(script))) $(foreach script, $(noinst-scripts), $(eval programs-list += $(script))) $(foreach template, $(template-files), $(eval $(call instantiate-template,$(template)))) -install_test_init=tests/functional/init.sh $(foreach test, $(install-tests), \ - $(eval $(call run-test,$(test),$(install_test_init))) \ + $(eval $(call run-test,$(test))) \ $(eval installcheck: $(test).test)) $(foreach test-group, $(install-tests-groups), \ - $(eval $(call run-test-group,$(test-group),$(install_test_init))) \ + $(eval $(call run-test-group,$(test-group))) \ $(eval installcheck: $(test-group).test-group) \ $(foreach test, $($(test-group)-tests), \ - $(eval $(call run-test,$(test),$(install_test_init))) \ + $(eval $(call run-test,$(test))) \ $(eval $(test-group).test-group: $(test).test))) +# Compilation database. +$(foreach lib, $(libraries), $(eval $(call write-compile-commands,$(lib)))) +$(foreach prog, $(programs), $(eval $(call write-compile-commands,$(prog)))) + +compile_commands.json: $(compile-commands-json-files) + @jq --slurp '.' $^ >$@ + # Include makefiles requiring built programs. $(foreach mf, $(makefiles-late), $(eval $(call include-sub-makefile,$(mf)))) diff --git a/mk/patterns.mk b/mk/patterns.mk index c81150260..4caa2039e 100644 --- a/mk/patterns.mk +++ b/mk/patterns.mk @@ -1,11 +1,41 @@ + +# These are the complete command lines we use to compile C and C++ files. +# - $< is the source file. +# - $1 is the object file to create. +CC_CMD=$(CC) -o $1 -c $< $(CPPFLAGS) $(GLOBAL_CFLAGS) $(CFLAGS) $($1_CFLAGS) -MMD -MF $(call filename-to-dep,$1) -MP +CXX_CMD=$(CXX) -o $1 -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($1_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep,$1) -MP + +# We use COMPILE_COMMANDS_JSON_CMD to turn a compilation command (like CC_CMD +# or CXX_CMD above) into a comple_commands.json file. We rely on bash native +# word splitting to define the positional arguments. +# - $< is the source file being compiled. +COMPILE_COMMANDS_JSON_CMD=jq --null-input '{ directory: $$ENV.PWD, file: "$<", arguments: $$ARGS.positional }' --args -- + + $(buildprefix)%.o: %.cc @mkdir -p "$(dir $@)" - $(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cxx) $(call CXX_CMD,$@) $(buildprefix)%.o: %.cpp @mkdir -p "$(dir $@)" - $(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cxx) $(call CXX_CMD,$@) $(buildprefix)%.o: %.c @mkdir -p "$(dir $@)" - $(trace-cc) $(CC) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CFLAGS) $(CFLAGS) $($@_CFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cc) $(call CC_CMD,$@) + +# In the following we need to replace the .compile_commands.json extension in $@ with .o +# to make the object file. This is needed because CC_CMD and CXX_CMD do further expansions +# based on the object file name (i.e. *_CXXFLAGS and filename-to-dep). + +$(buildprefix)%.compile_commands.json: %.cc + @mkdir -p "$(dir $@)" + $(trace-jq) $(COMPILE_COMMANDS_JSON_CMD) $(call CXX_CMD,$(@:.compile_commands.json=.o)) > $@ + +$(buildprefix)%.compile_commands.json: %.cpp + @mkdir -p "$(dir $@)" + $(trace-jq) $(COMPILE_COMMANDS_JSON_CMD) $(call CXX_CMD,$(@:.compile_commands.json=.o)) > $@ + +$(buildprefix)%.compile_commands.json: %.c + @mkdir -p "$(dir $@)" + $(trace-jq) $(COMPILE_COMMANDS_JSON_CMD) $(call CC_CMD,$(@:.compile_commands.json=.o)) > $@ diff --git a/mk/platform.mk b/mk/platform.mk index fe960dedf..22c114a20 100644 --- a/mk/platform.mk +++ b/mk/platform.mk @@ -29,4 +29,8 @@ ifdef HOST_OS HOST_SOLARIS = 1 HOST_UNIX = 1 endif + ifeq ($(HOST_KERNEL), gnu) + HOST_HURD = 1 + HOST_UNIX = 1 + endif endif diff --git a/mk/precompiled-headers.mk b/mk/precompiled-headers.mk index cdd3daecd..f2803eb79 100644 --- a/mk/precompiled-headers.mk +++ b/mk/precompiled-headers.mk @@ -8,7 +8,7 @@ GCH = $(buildprefix)precompiled-headers.h.gch $(GCH): precompiled-headers.h @rm -f $@ @mkdir -p "$(dir $@)" - $(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS) + $(trace-gen) $(CXX) -c -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS) clean-files += $(GCH) diff --git a/mk/run-test.sh b/mk/run-test.sh index da9c5a473..7f9f1d5f8 100755 --- a/mk/run-test.sh +++ b/mk/run-test.sh @@ -8,7 +8,6 @@ yellow="" normal="" test=$1 -init=${2-} dir="$(dirname "${BASH_SOURCE[0]}")" source "$dir/common-test.sh" @@ -22,20 +21,18 @@ if [ -t 1 ]; then fi run_test () { - if [ -n "$init" ]; then - (init_test 2>/dev/null > /dev/null) - fi - log="$(run_test_proper 2>&1)" && status=0 || status=$? + log="$(run "$test" 2>&1)" && status=0 || status=$? } run_test -if [ $status -eq 0 ]; then +if [[ "$status" = 0 ]]; then echo "$post_run_msg [${green}PASS$normal]" -elif [ $status -eq 99 ]; then +elif [[ "$status" = 77 ]]; then echo "$post_run_msg [${yellow}SKIP$normal]" else echo "$post_run_msg [${red}FAIL$normal]" + # shellcheck disable=SC2001 echo "$log" | sed 's/^/ /' exit "$status" fi diff --git a/mk/tests.mk b/mk/tests.mk index bac9b704a..0a10f6d3b 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -12,8 +12,8 @@ endef define run-test - $(eval $(call run-bash,$1.test,$1 $(test-deps),mk/run-test.sh $1 $2)) - $(eval $(call run-bash,$1.test-debug,$1 $(test-deps),mk/debug-test.sh $1 $2)) + $(eval $(call run-bash,$1.test,$1 $(test-deps),mk/run-test.sh $1)) + $(eval $(call run-bash,$1.test-debug,$1 $(test-deps),mk/debug-test.sh $1)) endef diff --git a/mk/tracing.mk b/mk/tracing.mk index 1fc5573d7..09db1e617 100644 --- a/mk/tracing.mk +++ b/mk/tracing.mk @@ -10,6 +10,8 @@ ifeq ($(V), 0) trace-install = @echo " INST " $@; trace-mkdir = @echo " MKDIR " $@; trace-test = @echo " TEST " $@; + trace-sh = @echo " SH " $@; + trace-jq = @echo " JQ " $@; suppress = @ diff --git a/package.nix b/package.nix index 20796a386..a7c8923e8 100644 --- a/package.nix +++ b/package.nix @@ -13,17 +13,16 @@ , curl , editline , readline -, fileset , flex , git , gtest , jq -, doxygen , libarchive , libcpuid , libgit2 , libseccomp , libsodium +, man , lowdown , mdbook , mdbook-linkcheck @@ -33,7 +32,8 @@ , pkg-config , rapidcheck , sqlite -, util-linux +, toml11 +, unixtools , xz , busybox-sandbox-shell ? null @@ -48,10 +48,8 @@ , pname ? "nix" , versionSuffix ? "" -, officialRelease ? false -# Whether to build Nix. Useful to skip for tasks like (a) just -# generating API docs or (b) testing existing pre-built versions of Nix +# Whether to build Nix. Useful to skip for tasks like testing existing pre-built versions of Nix , doBuild ? true # Run the unit tests as part of the build. See `installUnitTests` for an @@ -74,7 +72,10 @@ # sounds so long as evaluation just takes places within short-lived # processes. (When the process exits, the memory is reclaimed; it is # only leaked *within* the process.) -, enableGC ? true +# +# Temporarily disabled on Windows because the `GC_throw_bad_alloc` +# symbol is missing during linking. +, enableGC ? !stdenv.hostPlatform.isWindows # Whether to enable Markdown rendering in the Nix binary. , enableMarkdown ? !stdenv.hostPlatform.isWindows @@ -87,10 +88,6 @@ # - readline , readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline" -# Whether to build the internal API docs, can be done separately from -# everything else. -, enableInternalAPIDocs ? false - # Whether to install unit tests. This is useful when cross compiling # since we cannot run them natively during the build, but can do so # later. @@ -113,6 +110,8 @@ }: let + inherit (lib) fileset; + version = lib.fileContents ./.version + versionSuffix; # selected attributes with defaults, will be used to define some @@ -161,6 +160,8 @@ in { ./m4 # TODO: do we really need README.md? It doesn't seem used in the build. ./README.md + # This could be put behind a conditional + ./maintainers/local.mk # For make, regardless of what we are building ./local.mk ./Makefile @@ -171,17 +172,11 @@ in { ./doc ./misc ./precompiled-headers.h - ./src + (fileset.difference ./src ./src/perl) ./COPYING ./scripts/local.mk - ] ++ lib.optionals buildUnitTests [ + ] ++ lib.optionals enableManual [ ./doc/manual - ] ++ lib.optionals enableInternalAPIDocs [ - ./doc/internal-api - # Source might not be compiled, but still must be available - # for Doxygen to gather comments. - ./src - ./tests/unit ] ++ lib.optionals buildUnitTests [ ./tests/unit ] ++ lib.optionals doInstallCheck [ @@ -195,8 +190,10 @@ in { ++ lib.optional doBuild "dev" # If we are doing just build or just docs, the one thing will use # "out". We only need additional outputs if we are doing both. - ++ lib.optional (doBuild && (enableManual || enableInternalAPIDocs)) "doc" - ++ lib.optional installUnitTests "check"; + ++ lib.optional (doBuild && enableManual) "doc" + ++ lib.optional installUnitTests "check" + ++ lib.optional doCheck "testresults" + ; nativeBuildInputs = [ autoconf-archive @@ -213,14 +210,14 @@ in { git mercurial openssh + man # for testing `nix-* --help` ] ++ lib.optionals (doInstallCheck || enableManual) [ jq # Also for custom mdBook preprocessor. - ] ++ lib.optional stdenv.hostPlatform.isLinux util-linux - ++ lib.optional enableInternalAPIDocs doxygen + ] ++ lib.optional stdenv.hostPlatform.isStatic unixtools.hexdump ; - buildInputs = lib.optionals doBuild [ - boost + buildInputs = lib.optionals doBuild ( + [ brotli bzip2 curl @@ -229,6 +226,7 @@ in { libsodium openssl sqlite + toml11 xz ({ inherit readline editline; }.${readlineFlavor}) ] ++ lib.optionals enableMarkdown [ @@ -240,46 +238,22 @@ in { ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid # There have been issues building these dependencies ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin)) - (aws-sdk-cpp.override { - apis = ["s3" "transfer"]; - customMemoryManagement = false; - }) - ; + aws-sdk-cpp + ); - propagatedBuildInputs = [ + propagatedBuildInputs = lib.optionals doBuild ([ + boost nlohmann_json - ] ++ lib.optional enableGC boehmgc; + ] ++ lib.optional enableGC boehmgc + ); dontBuild = !attrs.doBuild; doCheck = attrs.doCheck; - disallowedReferences = [ boost ]; - - preConfigure = lib.optionalString (doBuild && ! stdenv.hostPlatform.isStatic) ( - '' - # Copy libboost_context so we don't get all of Boost in our closure. - # https://github.com/NixOS/nixpkgs/issues/45462 - mkdir -p $out/lib - cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib - rm -f $out/lib/*.a - '' + lib.optionalString stdenv.hostPlatform.isLinux '' - chmod u+w $out/lib/*.so.* - patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* - '' + lib.optionalString stdenv.hostPlatform.isDarwin '' - for LIB in $out/lib/*.dylib; do - chmod u+w $LIB - install_name_tool -id $LIB $LIB - install_name_tool -delete_rpath ${boost}/lib/ $LIB || true - done - install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib - '' - ); - configureFlags = [ (lib.enableFeature doBuild "build") (lib.enableFeature buildUnitTests "unit-tests") (lib.enableFeature doInstallCheck "functional-tests") - (lib.enableFeature enableInternalAPIDocs "internal-api-docs") (lib.enableFeature enableManual "doc-gen") (lib.enableFeature enableGC "gc") (lib.enableFeature enableMarkdown "markdown") @@ -303,8 +277,11 @@ in { makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1"; - installTargets = lib.optional doBuild "install" - ++ lib.optional enableInternalAPIDocs "internal-api-html"; + preCheck = '' + mkdir $testresults + ''; + + installTargets = lib.optional doBuild "install"; installFlags = "sysconfdir=$(out)/etc"; @@ -319,18 +296,16 @@ in { lib.optionalString stdenv.hostPlatform.isStatic '' mkdir -p $out/nix-support echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products - '' + lib.optionalString stdenv.isDarwin '' - install_name_tool \ - -change ${boost}/lib/libboost_context.dylib \ - $out/lib/libboost_context.dylib \ - $out/lib/libnixutil.dylib '' ) + lib.optionalString enableManual '' mkdir -p ''${!outputDoc}/nix-support echo "doc manual ''${!outputDoc}/share/doc/nix/manual" >> ''${!outputDoc}/nix-support/hydra-build-products - '' + lib.optionalString enableInternalAPIDocs '' - mkdir -p ''${!outputDoc}/nix-support - echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products + ''; + + # So the check output gets links for DLLs in the out output. + preFixup = lib.optionalString (stdenv.hostPlatform.isWindows && builtins.elem "check" finalAttrs.outputs) '' + ln -s "$check/lib/"*.dll "$check/bin" + ln -s "$out/bin/"*.dll "$check/bin" ''; doInstallCheck = attrs.doInstallCheck; @@ -341,20 +316,26 @@ in { # Work around weird bug where it doesn't think there is a Makefile. installCheckPhase = if (!doBuild && doInstallCheck) then '' + runHook preInstallCheck mkdir -p src/nix-channel make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES '' else null; # Needed for tests if we are not doing a build, but testing existing # built Nix. - preInstallCheck = lib.optionalString (! doBuild) '' - mkdir -p src/nix-channel - ''; + preInstallCheck = + lib.optionalString (! doBuild) '' + mkdir -p src/nix-channel + '' + # See https://github.com/NixOS/nix/issues/2523 + # Occurs often in tests since https://github.com/NixOS/nix/pull/9900 + + lib.optionalString stdenv.hostPlatform.isDarwin '' + export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES + ''; separateDebugInfo = !stdenv.hostPlatform.isStatic; - # TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated - # to work with `strictDeps`. + # TODO Always true after https://github.com/NixOS/nixpkgs/issues/318564 strictDeps = !withCoverageChecks; hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; diff --git a/packaging/components.nix b/packaging/components.nix new file mode 100644 index 000000000..870e9ae61 --- /dev/null +++ b/packaging/components.nix @@ -0,0 +1,43 @@ +scope: +let + inherit (scope) callPackage; +in + +# This becomes the pkgs.nixComponents attribute set +{ + nix = callPackage ../package.nix { }; + + nix-util = callPackage ../src/libutil/package.nix { }; + nix-util-c = callPackage ../src/libutil-c/package.nix { }; + nix-util-test-support = callPackage ../tests/unit/libutil-support/package.nix { }; + nix-util-tests = callPackage ../tests/unit/libutil/package.nix { }; + + nix-store = callPackage ../src/libstore/package.nix { }; + nix-store-c = callPackage ../src/libstore-c/package.nix { }; + nix-store-test-support = callPackage ../tests/unit/libstore-support/package.nix { }; + nix-store-tests = callPackage ../tests/unit/libstore/package.nix { }; + + nix-fetchers = callPackage ../src/libfetchers/package.nix { }; + nix-fetchers-tests = callPackage ../tests/unit/libfetchers/package.nix { }; + + nix-expr = callPackage ../src/libexpr/package.nix { }; + nix-expr-c = callPackage ../src/libexpr-c/package.nix { }; + nix-expr-test-support = callPackage ../tests/unit/libexpr-support/package.nix { }; + nix-expr-tests = callPackage ../tests/unit/libexpr/package.nix { }; + + nix-flake = callPackage ../src/libflake/package.nix { }; + nix-flake-tests = callPackage ../tests/unit/libflake/package.nix { }; + + nix-main = callPackage ../src/libmain/package.nix { }; + nix-main-c = callPackage ../src/libmain-c/package.nix { }; + + nix-cmd = callPackage ../src/libcmd/package.nix { }; + + # Will replace `nix` once the old build system is gone. + nix-ng = callPackage ../src/nix/package.nix { }; + + nix-internal-api-docs = callPackage ../src/internal-api-docs/package.nix { }; + nix-external-api-docs = callPackage ../src/external-api-docs/package.nix { }; + + nix-perl-bindings = callPackage ../src/perl/package.nix { }; +} diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix new file mode 100644 index 000000000..e0737593f --- /dev/null +++ b/packaging/dependencies.nix @@ -0,0 +1,165 @@ +# These overrides are applied to the dependencies of the Nix components. + +{ + # Flake inputs; used for sources + inputs, + + # The raw Nixpkgs, not affected by this scope + pkgs, + + stdenv, + versionSuffix, +}: + +let + prevStdenv = stdenv; +in + +let + inherit (pkgs) lib; + + root = ../.; + + stdenv = if prevStdenv.isDarwin && prevStdenv.isx86_64 + then darwinStdenv + else prevStdenv; + + # Fix the following error with the default x86_64-darwin SDK: + # + # error: aligned allocation function of type 'void *(std::size_t, std::align_val_t)' is only available on macOS 10.13 or newer + # + # Despite the use of the 10.13 deployment target here, the aligned + # allocation function Clang uses with this setting actually works + # all the way back to 10.6. + darwinStdenv = pkgs.overrideSDK prevStdenv { darwinMinVersion = "10.13"; }; + + # Nixpkgs implements this by returning a subpath into the fetched Nix sources. + resolvePath = p: p; + + # Indirection for Nixpkgs to override when package.nix files are vendored + filesetToSource = lib.fileset.toSource; + + localSourceLayer = finalAttrs: prevAttrs: + let + workDirPath = + # Ideally we'd pick finalAttrs.workDir, but for now `mkDerivation` has + # the requirement that everything except passthru and meta must be + # serialized by mkDerivation, which doesn't work for this. + prevAttrs.workDir; + + workDirSubpath = lib.path.removePrefix root workDirPath; + sources = assert prevAttrs.fileset._type == "fileset"; prevAttrs.fileset; + src = lib.fileset.toSource { fileset = sources; inherit root; }; + + in + { + sourceRoot = "${src.name}/" + workDirSubpath; + inherit src; + + # Clear what `derivation` can't/shouldn't serialize; see prevAttrs.workDir. + fileset = null; + workDir = null; + }; + + # Work around weird `--as-needed` linker behavior with BSD, see + # https://github.com/mesonbuild/meson/issues/3593 + bsdNoLinkAsNeeded = finalAttrs: prevAttrs: + lib.optionalAttrs stdenv.hostPlatform.isBSD { + mesonFlags = [ (lib.mesonBool "b_asneeded" false) ] ++ prevAttrs.mesonFlags or []; + }; + + miscGoodPractice = finalAttrs: prevAttrs: + { + strictDeps = prevAttrs.strictDeps or true; + enableParallelBuilding = true; + }; + +in +scope: { + inherit stdenv versionSuffix; + version = lib.fileContents ../.version + versionSuffix; + + aws-sdk-cpp = (pkgs.aws-sdk-cpp.override { + apis = [ "s3" "transfer" ]; + customMemoryManagement = false; + }).overrideAttrs { + # only a stripped down version is built, which takes a lot less resources + # to build, so we don't need a "big-parallel" machine. + requiredSystemFeatures = [ ]; + }; + + libseccomp = pkgs.libseccomp.overrideAttrs (_: rec { + version = "2.5.5"; + src = pkgs.fetchurl { + url = "https://github.com/seccomp/libseccomp/releases/download/v${version}/libseccomp-${version}.tar.gz"; + hash = "sha256-JIosik2bmFiqa69ScSw0r+/PnJ6Ut23OAsHJqiX7M3U="; + }; + }); + + boehmgc = pkgs.boehmgc.override { + enableLargeConfig = true; + }; + + # TODO Hack until https://github.com/NixOS/nixpkgs/issues/45462 is fixed. + boost = (pkgs.boost.override { + extraB2Args = [ + "--with-container" + "--with-context" + "--with-coroutine" + ]; + }).overrideAttrs (old: { + # Need to remove `--with-*` to use `--with-libraries=...` + buildPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.buildPhase; + installPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.installPhase; + }); + + libgit2 = pkgs.libgit2.overrideAttrs (attrs: { + src = inputs.libgit2; + version = inputs.libgit2.lastModifiedDate; + cmakeFlags = attrs.cmakeFlags or [] + ++ [ "-DUSE_SSH=exec" ]; + }); + + busybox-sandbox-shell = pkgs.busybox-sandbox-shell or (pkgs.busybox.override { + useMusl = true; + enableStatic = true; + enableMinimal = true; + extraConfig = '' + CONFIG_FEATURE_FANCY_ECHO y + CONFIG_FEATURE_SH_MATH y + CONFIG_FEATURE_SH_MATH_64 y + + CONFIG_ASH y + CONFIG_ASH_OPTIMIZE_FOR_SIZE y + + CONFIG_ASH_ALIAS y + CONFIG_ASH_BASH_COMPAT y + CONFIG_ASH_CMDCMD y + CONFIG_ASH_ECHO y + CONFIG_ASH_GETOPTS y + CONFIG_ASH_INTERNAL_GLOB y + CONFIG_ASH_JOB_CONTROL y + CONFIG_ASH_PRINTF y + CONFIG_ASH_TEST y + ''; + }); + + # TODO change in Nixpkgs, Windows works fine. First commit of + # https://github.com/NixOS/nixpkgs/pull/322977 backported will fix. + toml11 = pkgs.toml11.overrideAttrs (old: { + meta.platforms = lib.platforms.all; + }); + + inherit resolvePath filesetToSource; + + mkMesonDerivation = f: let + exts = [ + miscGoodPractice + bsdNoLinkAsNeeded + localSourceLayer + ]; + in stdenv.mkDerivation + (lib.extends + (lib.foldr lib.composeExtensions (_: _: {}) exts) + f); +} diff --git a/packaging/hydra.nix b/packaging/hydra.nix new file mode 100644 index 000000000..dbe992476 --- /dev/null +++ b/packaging/hydra.nix @@ -0,0 +1,200 @@ +{ inputs +, binaryTarball +, forAllCrossSystems +, forAllSystems +, lib +, linux64BitSystems +, nixpkgsFor +, self +}: +let + inherit (inputs) nixpkgs nixpkgs-regression; + + installScriptFor = tarballs: + nixpkgsFor.x86_64-linux.native.callPackage ../scripts/installer.nix { + inherit tarballs; + }; + + testNixVersions = pkgs: client: daemon: + pkgs.callPackage ../package.nix { + pname = + "nix-tests" + + lib.optionalString + (lib.versionAtLeast daemon.version "2.4pre20211005" && + lib.versionAtLeast client.version "2.4pre20211005") + "-${client.version}-against-${daemon.version}"; + + test-client = client; + test-daemon = daemon; + + doBuild = false; + }; + + # Technically we could just return `pkgs.nixComponents`, but for Hydra it's + # convention to transpose it, and to transpose it efficiently, we need to + # enumerate them manually, so that we don't evaluate unnecessary package sets. + forAllPackages = lib.genAttrs [ + "nix" + "nix-util" + "nix-util-c" + "nix-util-test-support" + "nix-util-tests" + "nix-store" + "nix-store-c" + "nix-store-test-support" + "nix-store-tests" + "nix-fetchers" + "nix-fetchers-tests" + "nix-expr" + "nix-expr-c" + "nix-expr-test-support" + "nix-expr-tests" + "nix-flake" + "nix-flake-tests" + "nix-main" + "nix-main-c" + "nix-cmd" + "nix-ng" + ]; +in +{ + # Binary package for various platforms. + build = forAllPackages (pkgName: + forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.${pkgName})); + + shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation); + + buildStatic = forAllPackages (pkgName: + lib.genAttrs linux64BitSystems (system: nixpkgsFor.${system}.static.nixComponents.${pkgName})); + + buildCross = forAllPackages (pkgName: + forAllCrossSystems (crossSystem: + lib.genAttrs [ "x86_64-linux" ] (system: nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName}))); + + buildNoGc = forAllSystems (system: + self.packages.${system}.nix.override { enableGC = false; } + ); + + buildNoTests = forAllSystems (system: nixpkgsFor.${system}.native.nix_noTests); + + # Toggles some settings for better coverage. Windows needs these + # library combinations, and Debian build Nix with GNU readline too. + buildReadlineNoMarkdown = forAllSystems (system: + self.packages.${system}.nix.override { + enableMarkdown = false; + readlineFlavor = "readline"; + } + ); + + # Perl bindings for various platforms. + perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.nix-perl-bindings); + + # Binary tarball for various platforms, containing a Nix store + # with the closure of 'nix' package, and the second half of + # the installation script. + binaryTarball = forAllSystems (system: binaryTarball nixpkgsFor.${system}.native.nix nixpkgsFor.${system}.native); + + binaryTarballCross = lib.genAttrs [ "x86_64-linux" ] (system: + forAllCrossSystems (crossSystem: + binaryTarball + nixpkgsFor.${system}.cross.${crossSystem}.nix + nixpkgsFor.${system}.cross.${crossSystem})); + + # The first half of the installation script. This is uploaded + # to https://nixos.org/nix/install. It downloads the binary + # tarball for the user's system and calls the second half of the + # installation script. + installerScript = installScriptFor [ + # Native + self.hydraJobs.binaryTarball."x86_64-linux" + self.hydraJobs.binaryTarball."i686-linux" + self.hydraJobs.binaryTarball."aarch64-linux" + self.hydraJobs.binaryTarball."x86_64-darwin" + self.hydraJobs.binaryTarball."aarch64-darwin" + # Cross + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu" + ]; + installerScriptForGHA = installScriptFor [ + # Native + self.hydraJobs.binaryTarball."x86_64-linux" + self.hydraJobs.binaryTarball."aarch64-darwin" + # Cross + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu" + ]; + + # docker image with Nix inside + dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); + + # Line coverage analysis. + coverage = nixpkgsFor.x86_64-linux.native.nix.override { + pname = "nix-coverage"; + withCoverageChecks = true; + }; + + # API docs for Nix's unstable internal C++ interfaces. + internal-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-internal-api-docs; + + # API docs for Nix's C bindings. + external-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-external-api-docs; + + # System tests. + tests = import ../tests/nixos { inherit lib nixpkgs nixpkgsFor self; } // { + + # Make sure that nix-env still produces the exact same result + # on a particular version of Nixpkgs. + evalNixpkgs = + let + inherit (nixpkgsFor.x86_64-linux.native) runCommand nix; + in + runCommand "eval-nixos" { buildInputs = [ nix ]; } + '' + type -p nix-env + # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593. + ( + set -x + time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages + [[ $(sha1sum < packages | cut -c1-40) = e01b031fc9785a572a38be6bc473957e3b6faad7 ]] + ) + mkdir $out + ''; + + nixpkgsLibTests = + forAllSystems (system: + import (nixpkgs + "/lib/tests/test-with-nix.nix") + { + lib = nixpkgsFor.${system}.native.lib; + nix = self.packages.${system}.nix; + pkgs = nixpkgsFor.${system}.native; + } + ); + }; + + metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" { + pkgs = nixpkgsFor.x86_64-linux.native; + nixpkgs = nixpkgs-regression; + }; + + installTests = forAllSystems (system: + let pkgs = nixpkgsFor.${system}.native; in + pkgs.runCommand "install-tests" + { + againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix; + againstCurrentLatest = + # FIXME: temporarily disable this on macOS because of #3605. + if system == "x86_64-linux" + then testNixVersions pkgs pkgs.nix pkgs.nixVersions.latest + else null; + # Disabled because the latest stable version doesn't handle + # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work + # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable; + } "touch $out"); + + installerTests = import ../tests/installer { + binaryTarballs = self.hydraJobs.binaryTarball; + inherit nixpkgsFor; + }; +} diff --git a/perl/.yath.rc b/perl/.yath.rc deleted file mode 100644 index 118bf80c8..000000000 --- a/perl/.yath.rc +++ /dev/null @@ -1,2 +0,0 @@ -[test] --I=rel(lib/Nix) diff --git a/perl/Makefile b/perl/Makefile deleted file mode 100644 index 832668dd1..000000000 --- a/perl/Makefile +++ /dev/null @@ -1,21 +0,0 @@ -makefiles = local.mk - -GLOBAL_CXXFLAGS += -g -Wall -std=c++2a - -# A convenience for concurrent development of Nix and its Perl bindings. -# Not needed in a standalone build of the Perl bindings. -ifneq ("$(wildcard ../src)", "") - GLOBAL_CXXFLAGS += -I ../src -endif - --include Makefile.config - -OPTIMIZE = 1 - -ifeq ($(OPTIMIZE), 1) - GLOBAL_CXXFLAGS += -O3 -else - GLOBAL_CXXFLAGS += -O0 -endif - -include mk/lib.mk diff --git a/perl/Makefile.config.in b/perl/Makefile.config.in deleted file mode 100644 index d856de3ad..000000000 --- a/perl/Makefile.config.in +++ /dev/null @@ -1,18 +0,0 @@ -HOST_OS = @host_os@ -CC = @CC@ -CFLAGS = @CFLAGS@ -CXX = @CXX@ -CXXFLAGS = @CXXFLAGS@ -PACKAGE_NAME = @PACKAGE_NAME@ -PACKAGE_VERSION = @PACKAGE_VERSION@ -SODIUM_LIBS = @SODIUM_LIBS@ -NIX_CFLAGS = @NIX_CFLAGS@ -NIX_LIBS = @NIX_LIBS@ -nixbindir = @nixbindir@ -curl = @curl@ -nixlibexecdir = @nixlibexecdir@ -nixlocalstatedir = @nixlocalstatedir@ -perl = @perl@ -perllibdir = @perllibdir@ -nixstoredir = @nixstoredir@ -nixsysconfdir = @nixsysconfdir@ diff --git a/perl/configure.ac b/perl/configure.ac deleted file mode 100644 index a02cb06c9..000000000 --- a/perl/configure.ac +++ /dev/null @@ -1,84 +0,0 @@ -AC_INIT(nix-perl, m4_esyscmd([bash -c "echo -n $(cat ../.version)$VERSION_SUFFIX"])) -AC_CONFIG_SRCDIR(MANIFEST) -AC_CONFIG_AUX_DIR(../config) - -CFLAGS= -CXXFLAGS= -AC_PROG_CC -AC_PROG_CXX - -AC_CANONICAL_HOST - -# Use 64-bit file system calls so that we can support files > 2 GiB. -AC_SYS_LARGEFILE - -AC_DEFUN([NEED_PROG], -[ -AC_PATH_PROG($1, $2) -if test -z "$$1"; then - AC_MSG_ERROR([$2 is required]) -fi -]) - -NEED_PROG(perl, perl) -NEED_PROG(curl, curl) -NEED_PROG(bzip2, bzip2) -NEED_PROG(xz, xz) - -# Test that Perl has the open/fork feature (Perl 5.8.0 and beyond). -AC_MSG_CHECKING([whether Perl is recent enough]) -if ! $perl -e 'open(FOO, "-|", "true"); while () { print; }; close FOO or die;'; then - AC_MSG_RESULT(no) - AC_MSG_ERROR([Your Perl version is too old. Nix requires Perl 5.8.0 or newer.]) -fi -AC_MSG_RESULT(yes) - - -# Figure out where to install Perl modules. -AC_MSG_CHECKING([for the Perl installation prefix]) -perlversion=$($perl -e 'use Config; print $Config{version};') -perlarchname=$($perl -e 'use Config; print $Config{archname};') -AC_SUBST(perllibdir, [${libdir}/perl5/site_perl/$perlversion/$perlarchname]) -AC_MSG_RESULT($perllibdir) - -# Look for libsodium. -PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"]) - -# Check for the required Perl dependencies (DBI and DBD::SQLite). -perlFlags="-I$perllibdir" - -AC_ARG_WITH(dbi, AC_HELP_STRING([--with-dbi=PATH], - [prefix of the Perl DBI library]), - perlFlags="$perlFlags -I$withval") - -AC_ARG_WITH(dbd-sqlite, AC_HELP_STRING([--with-dbd-sqlite=PATH], - [prefix of the Perl DBD::SQLite library]), - perlFlags="$perlFlags -I$withval") - -AC_MSG_CHECKING([whether DBD::SQLite works]) -if ! $perl $perlFlags -e 'use DBI; use DBD::SQLite;' 2>&5; then - AC_MSG_RESULT(no) - AC_MSG_FAILURE([The Perl modules DBI and/or DBD::SQLite are missing.]) -fi -AC_MSG_RESULT(yes) - -AC_SUBST(perlFlags) - -PKG_CHECK_MODULES([NIX], [nix-store]) - -NEED_PROG([NIX], [nix]) - -# Expand all variables in config.status. -test "$prefix" = NONE && prefix=$ac_default_prefix -test "$exec_prefix" = NONE && exec_prefix='${prefix}' -for name in $ac_subst_vars; do - declare $name="$(eval echo "${!name}")" - declare $name="$(eval echo "${!name}")" - declare $name="$(eval echo "${!name}")" -done - -rm -f Makefile.config -ln -sfn ../mk mk - -AC_CONFIG_FILES([]) -AC_OUTPUT diff --git a/perl/default.nix b/perl/default.nix deleted file mode 100644 index 7103574c9..000000000 --- a/perl/default.nix +++ /dev/null @@ -1,61 +0,0 @@ -{ lib, fileset -, stdenv -, perl, perlPackages -, autoconf-archive, autoreconfHook, pkg-config -, nix, curl, bzip2, xz, boost, libsodium, darwin -}: - -perl.pkgs.toPerlModule (stdenv.mkDerivation (finalAttrs: { - name = "nix-perl-${nix.version}"; - - src = fileset.toSource { - root = ../.; - fileset = fileset.unions ([ - ../.version - ../m4 - ../mk - ./MANIFEST - ./Makefile - ./Makefile.config.in - ./configure.ac - ./lib - ./local.mk - ] ++ lib.optionals finalAttrs.doCheck [ - ./.yath.rc - ./t - ]); - }; - - nativeBuildInputs = - [ autoconf-archive - autoreconfHook - pkg-config - ]; - - buildInputs = - [ nix - curl - bzip2 - xz - perl - boost - ] - ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium - ++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security; - - # `perlPackages.Test2Harness` is marked broken for Darwin - doCheck = !stdenv.isDarwin; - - nativeCheckInputs = [ - perlPackages.Test2Harness - ]; - - configureFlags = [ - "--with-dbi=${perlPackages.DBI}/${perl.libPrefix}" - "--with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix}" - ]; - - enableParallelBuilding = true; - - postUnpack = "sourceRoot=$sourceRoot/perl"; -})) diff --git a/perl/local.mk b/perl/local.mk deleted file mode 100644 index ed4764eb9..000000000 --- a/perl/local.mk +++ /dev/null @@ -1,46 +0,0 @@ -nix_perl_sources := \ - lib/Nix/Store.pm \ - lib/Nix/Manifest.pm \ - lib/Nix/SSH.pm \ - lib/Nix/CopyClosure.pm \ - lib/Nix/Config.pm.in \ - lib/Nix/Utils.pm - -nix_perl_modules := $(nix_perl_sources:.in=) - -$(foreach x, $(nix_perl_modules), $(eval $(call install-data-in, $(x), $(perllibdir)/Nix))) - -lib/Nix/Store.cc: lib/Nix/Store.xs - $(trace-gen) xsubpp $^ -output $@ - -libraries += Store - -Store_DIR := lib/Nix - -Store_SOURCES := $(Store_DIR)/Store.cc - -Store_CXXFLAGS = \ - $(NIX_CFLAGS) \ - -I$(shell perl -e 'use Config; print $$Config{archlibexp};')/CORE \ - -D_FILE_OFFSET_BITS=64 \ - -Wno-unknown-warning-option -Wno-unused-variable -Wno-literal-suffix \ - -Wno-reserved-user-defined-literal -Wno-duplicate-decl-specifier -Wno-pointer-bool-conversion - -Store_LDFLAGS := $(SODIUM_LIBS) $(NIX_LIBS) - -ifdef HOST_CYGWIN - archlib = $(shell perl -E 'use Config; print $$Config{archlib};') - libperl = $(shell perl -E 'use Config; print $$Config{libperl};') - Store_LDFLAGS += $(shell find ${archlib} -name ${libperl}) -endif - -Store_ALLOW_UNDEFINED = 1 - -Store_FORCE_INSTALL = 1 - -Store_INSTALL_DIR = $(perllibdir)/auto/Nix/Store - -clean-files += lib/Nix/Config.pm lib/Nix/Store.cc Makefile.config - -check: all - yath test diff --git a/precompiled-headers.h b/precompiled-headers.h index f52f1cab8..e1a3f8cc0 100644 --- a/precompiled-headers.h +++ b/precompiled-headers.h @@ -42,19 +42,22 @@ #include #include #include -#include -#include -#include #include -#include -#include -#include #include #include #include -#include -#include -#include #include +#ifndef _WIN32 +# include +# include +# include +# include +# include +# include +# include +# include +# include +#endif + #include diff --git a/scripts/bigsur-nixbld-user-migration.sh b/scripts/bigsur-nixbld-user-migration.sh index f1619fd56..0eb312e07 100755 --- a/scripts/bigsur-nixbld-user-migration.sh +++ b/scripts/bigsur-nixbld-user-migration.sh @@ -3,7 +3,7 @@ ((NEW_NIX_FIRST_BUILD_UID=301)) id_available(){ - dscl . list /Users UniqueID | grep -E '\b'$1'\b' >/dev/null + dscl . list /Users UniqueID | grep -E '\b'"$1"'\b' >/dev/null } change_nixbld_names_and_ids(){ @@ -26,18 +26,18 @@ change_nixbld_names_and_ids(){ fi done - if [[ $name == _* ]]; then + if [[ "$name" == _* ]]; then echo " It looks like $name has already been renamed--skipping." else # first 3 are cleanup, it's OK if they aren't here - sudo dscl . delete /Users/$name dsAttrTypeNative:_writers_passwd &>/dev/null || true - sudo dscl . change /Users/$name NFSHomeDirectory "/private/var/empty 1" "/var/empty" &>/dev/null || true + sudo dscl . delete "/Users/$name" dsAttrTypeNative:_writers_passwd &>/dev/null || true + sudo dscl . change "/Users/$name" NFSHomeDirectory "/private/var/empty 1" "/var/empty" &>/dev/null || true # remove existing user from group - sudo dseditgroup -o edit -t user -d $name nixbld || true - sudo dscl . change /Users/$name UniqueID $uid $next_id - sudo dscl . change /Users/$name RecordName $name _$name + sudo dseditgroup -o edit -t user -d "$name" nixbld || true + sudo dscl . change "/Users/$name" UniqueID "$uid" "$next_id" + sudo dscl . change "/Users/$name" RecordName "$name" "_$name" # add renamed user to group - sudo dseditgroup -o edit -t user -a _$name nixbld + sudo dseditgroup -o edit -t user -a "_$name" nixbld echo " $name migrated to _$name (uid: $next_id)" fi done < <(dscl . list /Users UniqueID | grep nixbld | sort -n -k2) diff --git a/scripts/check-hydra-status.sh b/scripts/check-hydra-status.sh deleted file mode 100644 index e62705e94..000000000 --- a/scripts/check-hydra-status.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail -# set -x - - -# mapfile BUILDS_FOR_LATEST_EVAL < <( -# curl -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master/evals | \ -# jq -r '.evals[0].builds[] | @sh') -BUILDS_FOR_LATEST_EVAL=$( -curl -sS -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master/evals | \ - jq -r '.evals[0].builds[]') - -someBuildFailed=0 - -for buildId in $BUILDS_FOR_LATEST_EVAL; do - buildInfo=$(curl --fail -sS -H 'Accept: application/json' "https://hydra.nixos.org/build/$buildId") - - finished=$(echo "$buildInfo" | jq -r '.finished') - - if [[ $finished = 0 ]]; then - continue - fi - - buildStatus=$(echo "$buildInfo" | jq -r '.buildstatus') - - if [[ $buildStatus != 0 ]]; then - someBuildFailed=1 - echo "Job “$(echo "$buildInfo" | jq -r '.job')†failed on hydra: $buildInfo" - fi -done - -exit "$someBuildFailed" diff --git a/scripts/flake-regressions.sh b/scripts/flake-regressions.sh new file mode 100755 index 000000000..d76531134 --- /dev/null +++ b/scripts/flake-regressions.sh @@ -0,0 +1,27 @@ +#! /usr/bin/env bash + +set -e + +echo "Nix version:" +nix --version + +cd flake-regressions + +status=0 + +flakes=$(find tests -mindepth 3 -maxdepth 3 -type d -not -path '*/.*' | sort | head -n25) + +echo "Running flake tests..." + +for flake in $flakes; do + + if ! REGENERATE=0 ./eval-flake.sh "$flake"; then + status=1 + echo "⌠$flake" + else + echo "✅ $flake" + fi + +done + +exit "$status" diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index ad3ee8881..6aee073e3 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -754,7 +754,7 @@ I will: (if it does, I will tell you how to clean them up.) - create local users (see the list above for the users I'll make) - create a local group ($NIX_BUILD_GROUP_NAME) - - install Nix in to $NIX_ROOT + - install Nix in $NIX_ROOT - create a configuration file in /etc/nix - set up the "default profile" by creating some Nix-related files in $ROOT_HOME diff --git a/scripts/install-systemd-multi-user.sh b/scripts/install-systemd-multi-user.sh index 202a9bb54..a62ed7e3a 100755 --- a/scripts/install-systemd-multi-user.sh +++ b/scripts/install-systemd-multi-user.sh @@ -35,7 +35,7 @@ escape_systemd_env() { # Gather all non-empty proxy environment variables into a string create_systemd_proxy_env() { - vars="http_proxy https_proxy ftp_proxy no_proxy HTTP_PROXY HTTPS_PROXY FTP_PROXY NO_PROXY" + vars="http_proxy https_proxy ftp_proxy all_proxy no_proxy HTTP_PROXY HTTPS_PROXY FTP_PROXY ALL_PROXY NO_PROXY" for v in $vars; do if [ "x${!v:-}" != "x" ]; then echo "Environment=${v}=$(escape_systemd_env ${!v})" diff --git a/scripts/install.in b/scripts/install.in index 7d2e52b26..b4e808d8e 100755 --- a/scripts/install.in +++ b/scripts/install.in @@ -50,6 +50,11 @@ case "$(uname -s).$(uname -m)" in path=@tarballPath_armv7l-linux@ system=armv7l-linux ;; + Linux.riscv64) + hash=@tarballHash_riscv64-linux@ + path=@tarballPath_riscv64-linux@ + system=riscv64-linux + ;; Darwin.x86_64) hash=@tarballHash_x86_64-darwin@ path=@tarballPath_x86_64-darwin@ diff --git a/scripts/nix-profile-daemon.fish.in b/scripts/nix-profile-daemon.fish.in index c23aa64f0..346dce5dd 100644 --- a/scripts/nix-profile-daemon.fish.in +++ b/scripts/nix-profile-daemon.fish.in @@ -28,7 +28,7 @@ else end # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. -if test -n "$NIX_SSH_CERT_FILE" +if test -n "$NIX_SSL_CERT_FILE" : # Allow users to override the NIX_SSL_CERT_FILE else if test -e /etc/ssl/certs/ca-certificates.crt # NixOS, Ubuntu, Debian, Gentoo, Arch set --export NIX_SSL_CERT_FILE /etc/ssl/certs/ca-certificates.crt @@ -44,7 +44,7 @@ else if test -e "$NIX_LINK/etc/ca-bundle.crt" # old cacert in Nix profile set --export NIX_SSL_CERT_FILE "$NIX_LINK/etc/ca-bundle.crt" else # Fall back to what is in the nix profiles, favouring whatever is defined last. - for i in $NIX_PROFILES + for i in (string split ' ' $NIX_PROFILES) if test -e "$i/etc/ssl/certs/ca-bundle.crt" set --export NIX_SSL_CERT_FILE "$i/etc/ssl/certs/ca-bundle.crt" end diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index d256b24ed..eb124c0b5 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -1,4 +1,5 @@ # Only execute this file once per shell. +# This file is tested by tests/installer/default.nix. if [ -n "${__ETC_PROFILE_NIX_SOURCED:-}" ]; then return; fi __ETC_PROFILE_NIX_SOURCED=1 @@ -9,11 +10,9 @@ else NIX_LINK_NEW=$HOME/.local/state/nix/profile fi if [ -e "$NIX_LINK_NEW" ]; then - NIX_LINK="$NIX_LINK_NEW" -else - if [ -t 2 ] && [ -e "$NIX_LINK_NEW" ]; then + if [ -t 2 ] && [ -e "$NIX_LINK" ]; then warning="\033[1;35mwarning:\033[0m" - printf "$warning Both %s and legacy %s exist; using the latter.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2 + printf "$warning Both %s and legacy %s exist; using the former.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2 if [ "$(realpath "$NIX_LINK")" = "$(realpath "$NIX_LINK_NEW")" ]; then printf " Since the profiles match, you can safely delete either of them.\n" 1>&2 else @@ -26,6 +25,7 @@ else printf "$warning Profiles do not match. You should manually migrate from %s to %s.\n" "$NIX_LINK" "$NIX_LINK_NEW" 1>&2 fi fi + NIX_LINK="$NIX_LINK_NEW" fi export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK" @@ -69,4 +69,4 @@ else fi export PATH="$NIX_LINK/bin:@localstatedir@/nix/profiles/default/bin:$PATH" -unset NIX_LINK +unset NIX_LINK NIX_LINK_NEW diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index 44bc96e89..e868399b1 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -1,3 +1,4 @@ +# This file is tested by tests/installer/default.nix. if [ -n "$HOME" ] && [ -n "$USER" ]; then # Set up the per-user profile. @@ -9,11 +10,9 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then NIX_LINK_NEW="$HOME/.local/state/nix/profile" fi if [ -e "$NIX_LINK_NEW" ]; then - NIX_LINK="$NIX_LINK_NEW" - else - if [ -t 2 ] && [ -e "$NIX_LINK_NEW" ]; then + if [ -t 2 ] && [ -e "$NIX_LINK" ]; then warning="\033[1;35mwarning:\033[0m" - printf "$warning Both %s and legacy %s exist; using the latter.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2 + printf "$warning Both %s and legacy %s exist; using the former.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2 if [ "$(realpath "$NIX_LINK")" = "$(realpath "$NIX_LINK_NEW")" ]; then printf " Since the profiles match, you can safely delete either of them.\n" 1>&2 else @@ -26,6 +25,7 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then printf "$warning Profiles do not match. You should manually migrate from %s to %s.\n" "$NIX_LINK" "$NIX_LINK_NEW" 1>&2 fi fi + NIX_LINK="$NIX_LINK_NEW" fi # Set up environment. diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 118468477..82ad7d862 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -11,11 +11,13 @@ #include "machines.hh" #include "shared.hh" +#include "plugin.hh" #include "pathlocks.hh" #include "globals.hh" #include "serialise.hh" #include "build-result.hh" #include "store-api.hh" +#include "strings.hh" #include "derivations.hh" #include "local-store.hh" #include "legacy.hh" @@ -37,7 +39,7 @@ static std::string currentLoad; static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot) { - return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true); + return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri.render()), slot), true); } static bool allSupportedLocally(Store & store, const std::set& requiredFeatures) { @@ -135,7 +137,7 @@ static int main_build_remote(int argc, char * * argv) Machine * bestMachine = nullptr; uint64_t bestLoad = 0; for (auto & m : machines) { - debug("considering building on remote machine '%s'", m.storeUri); + debug("considering building on remote machine '%s'", m.storeUri.render()); if (m.enabled && m.systemSupported(neededSystem) && @@ -202,7 +204,7 @@ static int main_build_remote(int argc, char * * argv) else drvstr = ""; - auto error = HintFmt(errorText); + auto error = HintFmt::fromFormatString(errorText); error % drvstr % neededSystem @@ -232,17 +234,16 @@ static int main_build_remote(int argc, char * * argv) lock = -1; try { + storeUri = bestMachine->storeUri.render(); - Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri)); + Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", storeUri)); sshStore = bestMachine->openStore(); sshStore->connect(); - storeUri = bestMachine->storeUri; - } catch (std::exception & e) { auto msg = chomp(drainFD(5, false)); printError("cannot build on '%s': %s%s", - bestMachine->storeUri, e.what(), + storeUri, e.what(), msg.empty() ? "" : ": " + msg); bestMachine->enabled = false; continue; @@ -262,7 +263,20 @@ connected: auto inputs = readStrings(source); auto wantedOutputs = readStrings(source); - AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true); + AutoCloseFD uploadLock; + { + auto setUpdateLock = [&](auto && fileName){ + uploadLock = openLockFile(currentLoad + "/" + escapeUri(fileName) + ".upload-lock", true); + }; + try { + setUpdateLock(storeUri); + } catch (SysError & e) { + if (e.errNo != ENAMETOOLONG) throw; + // Try again hashing the store URL so we have a shorter path + auto h = hashString(HashAlgorithm::MD5, storeUri); + setUpdateLock(h.to_string(HashFormat::Base64, false)); + } + } { Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri)); diff --git a/doc/internal-api/.gitignore b/src/external-api-docs/.gitignore similarity index 100% rename from doc/internal-api/.gitignore rename to src/external-api-docs/.gitignore diff --git a/src/external-api-docs/.version b/src/external-api-docs/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/external-api-docs/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/external-api-docs/README.md b/src/external-api-docs/README.md new file mode 100644 index 000000000..8760ac88b --- /dev/null +++ b/src/external-api-docs/README.md @@ -0,0 +1,121 @@ +# Getting started + +> **Warning** These bindings are **experimental**, which means they can change +> at any time or be removed outright; nevertheless the plan is to provide a +> stable external C API to the Nix language and the Nix store. + +The language library allows evaluating Nix expressions and interacting with Nix +language values. The Nix store API is still rudimentary, and only allows +initialising and connecting to a store for the Nix language evaluator to +interact with. + +Currently there are two ways to interface with the Nix language evaluator +programmatically: + +1. Embedding the evaluator +2. Writing language plug-ins + +Embedding means you link the Nix C libraries in your program and use them from +there. Adding a plug-in means you make a library that gets loaded by the Nix +language evaluator, specified through a configuration option. + +Many of the components and mechanisms involved are not yet documented, therefore +please refer to the [Nix source code](https://github.com/NixOS/nix/) for +details. Additions to in-code documentation and the reference manual are highly +appreciated. + +The following examples, for simplicity, don't include error handling. See the +[Handling errors](@ref errors) section for more information. + +# Embedding the Nix Evaluator{#nix_evaluator_example} + +In this example we programmatically start the Nix language evaluator with a +dummy store (that has no store paths and cannot be written to), and evaluate the +Nix expression `builtins.nixVersion`. + +**main.c:** + +```C +#include +#include +#include +#include +#include +#include + +// NOTE: This example lacks all error handling. Production code must check for +// errors, as some return values will be undefined. + +void my_get_string_cb(const char * start, unsigned int n, void * user_data) +{ + *((char **) user_data) = strdup(start); +} + +int main() +{ + nix_libexpr_init(NULL); + + Store * store = nix_store_open(NULL, "dummy://", NULL); + EvalState * state = nix_state_create(NULL, NULL, store); // empty search path (NIX_PATH) + Value * value = nix_alloc_value(NULL, state); + + nix_expr_eval_from_string(NULL, state, "builtins.nixVersion", ".", value); + nix_value_force(NULL, state, value); + + char * version; + nix_get_string(NULL, value, my_get_string_cb, &version); + printf("Nix version: %s\n", version); + + free(version); + nix_gc_decref(NULL, value); + nix_state_free(state); + nix_store_free(store); + return 0; +} +``` + +**Usage:** + +```ShellSession +$ gcc main.c $(pkg-config nix-expr-c --libs --cflags) -o main +$ ./main +Nix version: 2.17 +``` + +# Writing a Nix language plug-in + +In this example we add a custom primitive operation (_primop_) to `builtins`. It +will increment the argument if it is an integer and throw an error otherwise. + +**plugin.c:** + +```C +#include +#include +#include + +void increment(void* user_data, nix_c_context* ctx, EvalState* state, Value** args, Value* v) { + nix_value_force(NULL, state, args[0]); + if (nix_get_type(NULL, args[0]) == NIX_TYPE_INT) { + nix_init_int(NULL, v, nix_get_int(NULL, args[0]) + 1); + } else { + nix_set_err_msg(ctx, NIX_ERR_UNKNOWN, "First argument should be an integer."); + } +} + +void nix_plugin_entry() { + const char* args[] = {"n", NULL}; + PrimOp *p = nix_alloc_primop(NULL, increment, 1, "increment", args, "Example custom built-in function: increments an integer", NULL); + nix_register_primop(NULL, p); + nix_gc_decref(NULL, p); +} +``` + +**Usage:** + +```ShellSession +$ gcc plugin.c $(pkg-config nix-expr-c --libs --cflags) -shared -o plugin.so +$ nix --plugin-files ./plugin.so repl +nix-repl> builtins.increment 1 +2 +``` diff --git a/src/external-api-docs/doxygen.cfg.in b/src/external-api-docs/doxygen.cfg.in new file mode 100644 index 000000000..1be71d895 --- /dev/null +++ b/src/external-api-docs/doxygen.cfg.in @@ -0,0 +1,58 @@ +# Doxyfile 1.9.5 + +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by +# double-quotes, unless you are using Doxywizard) that should identify the +# project for which the documentation is generated. This name is used in the +# title of most generated pages and in a few other places. +# The default value is: My Project. + +PROJECT_NAME = "Nix" + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. This +# could be handy for archiving the generated documentation or if some version +# control system is used. + +PROJECT_NUMBER = @PROJECT_NUMBER@ + +OUTPUT_DIRECTORY = @OUTPUT_DIRECTORY@ + +# Using the PROJECT_BRIEF tag one can provide an optional one line description +# for a project that appears at the top of each page and should give viewer a +# quick idea about the purpose of the project. Keep the description short. + +PROJECT_BRIEF = "Nix, the purely functional package manager: C API (experimental)" + +# If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output. +# The default value is: YES. + +GENERATE_LATEX = NO + +# The INPUT tag is used to specify the files and/or directories that contain +# documented source files. You may enter file names like myfile.cpp or +# directories like /usr/src/myproject. Separate the files or directories with +# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING +# Note: If this tag is empty the current directory is searched. + +# FIXME Make this list more maintainable somehow. We could maybe generate this +# in the Makefile, but we would need to change how `.in` files are preprocessed +# so they can expand variables despite configure variables. + +INPUT = \ + @src@/src/libutil-c \ + @src@/src/libexpr-c \ + @src@/src/libstore-c \ + @src@/doc/external-api/README.md + +FILE_PATTERNS = nix_api_*.h *.md + +# The INCLUDE_PATH tag can be used to specify one or more directories that +# contain include files that are not input files but should be processed by the +# preprocessor. Note that the INCLUDE_PATH is not recursive, so the setting of +# RECURSIVE has no effect here. +# This tag requires that the tag SEARCH_INCLUDES is set to YES. + +EXCLUDE_PATTERNS = *_internal.h +GENERATE_TREEVIEW = YES +OPTIMIZE_OUTPUT_FOR_C = YES + +USE_MDFILE_AS_MAINPAGE = doc/external-api/README.md diff --git a/src/external-api-docs/meson.build b/src/external-api-docs/meson.build new file mode 100644 index 000000000..62474ffe4 --- /dev/null +++ b/src/external-api-docs/meson.build @@ -0,0 +1,31 @@ +project('nix-external-api-docs', + version : files('.version'), + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +fs = import('fs') + +doxygen_cfg = configure_file( + input : 'doxygen.cfg.in', + output : 'doxygen.cfg', + configuration : { + 'PROJECT_NUMBER': meson.project_version(), + 'OUTPUT_DIRECTORY' : meson.current_build_dir(), + 'src' : fs.parent(fs.parent(meson.project_source_root())), + }, +) + +doxygen = find_program('doxygen', native : true, required : true) + +custom_target( + 'external-api-docs', + command : [ doxygen , doxygen_cfg ], + input : [ + doxygen_cfg, + ], + output : 'html', + install : true, + install_dir : get_option('datadir') / 'doc/nix/external-api', + build_always_stale : true, +) diff --git a/src/external-api-docs/package.nix b/src/external-api-docs/package.nix new file mode 100644 index 000000000..743b3e9b7 --- /dev/null +++ b/src/external-api-docs/package.nix @@ -0,0 +1,59 @@ +{ lib +, mkMesonDerivation + +, meson +, ninja +, doxygen + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-external-api-docs"; + inherit version; + + workDir = ./.; + fileset = + let + cpp = fileset.fileFilter (file: file.hasExt "cc" || file.hasExt "h"); + in + fileset.unions [ + ./.version + ../../.version + ./meson.build + ./doxygen.cfg.in + ./README.md + # Source is not compiled, but still must be available for Doxygen + # to gather comments. + (cpp ../libexpr-c) + (cpp ../libstore-c) + (cpp ../libutil-c) + ]; + + nativeBuildInputs = [ + meson + ninja + doxygen + ]; + + preConfigure = + '' + chmod u+w ./.version + echo ${finalAttrs.version} > ./.version + ''; + + postInstall = '' + mkdir -p ''${!outputDoc}/nix-support + echo "doc external-api-docs $out/share/doc/nix/external-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products + ''; + + meta = { + platforms = lib.platforms.all; + }; +}) diff --git a/src/internal-api-docs/.gitignore b/src/internal-api-docs/.gitignore new file mode 100644 index 000000000..dab28b6b0 --- /dev/null +++ b/src/internal-api-docs/.gitignore @@ -0,0 +1,3 @@ +/doxygen.cfg +/html +/latex diff --git a/src/internal-api-docs/.version b/src/internal-api-docs/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/internal-api-docs/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/doc/internal-api/doxygen.cfg.in b/src/internal-api-docs/doxygen.cfg.in similarity index 84% rename from doc/internal-api/doxygen.cfg.in rename to src/internal-api-docs/doxygen.cfg.in index 6c6c325bd..f1ef75b38 100644 --- a/doc/internal-api/doxygen.cfg.in +++ b/src/internal-api-docs/doxygen.cfg.in @@ -12,7 +12,9 @@ PROJECT_NAME = "Nix" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = @PACKAGE_VERSION@ +PROJECT_NUMBER = @PROJECT_NUMBER@ + +OUTPUT_DIRECTORY = @OUTPUT_DIRECTORY@ # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a @@ -36,27 +38,27 @@ GENERATE_LATEX = NO # so they can expand variables despite configure variables. INPUT = \ - src/libcmd \ - src/libexpr \ - src/libexpr/flake \ - tests/unit/libexpr \ - tests/unit/libexpr/value \ - tests/unit/libexpr/test \ - tests/unit/libexpr/test/value \ - src/libexpr/value \ - src/libfetchers \ - src/libmain \ - src/libstore \ - src/libstore/build \ - src/libstore/builtins \ - tests/unit/libstore \ - tests/unit/libstore/test \ - src/libutil \ - tests/unit/libutil \ - tests/unit/libutil/test \ - src/nix \ - src/nix-env \ - src/nix-store + @src@/libcmd \ + @src@/libexpr \ + @src@/libexpr/flake \ + @src@/nix-expr-tests \ + @src@/nix-expr-tests/value \ + @src@/nix-expr-test-support/test \ + @src@/nix-expr-test-support/test/value \ + @src@/libexpr/value \ + @src@/libfetchers \ + @src@/libmain \ + @src@/libstore \ + @src@/libstore/build \ + @src@/libstore/builtins \ + @src@/nix-store-tests \ + @src@/nix-store-test-support/test \ + @src@/libutil \ + @src@/nix-util-tests \ + @src@/nix-util-test-support/test \ + @src@/nix \ + @src@/nix-env \ + @src@/nix-store # If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names # in the source code. If set to NO, only conditional compilation will be diff --git a/src/internal-api-docs/meson.build b/src/internal-api-docs/meson.build new file mode 100644 index 000000000..54eb7e5dd --- /dev/null +++ b/src/internal-api-docs/meson.build @@ -0,0 +1,31 @@ +project('nix-internal-api-docs', + version : files('.version'), + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +fs = import('fs') + +doxygen_cfg = configure_file( + input : 'doxygen.cfg.in', + output : 'doxygen.cfg', + configuration : { + 'PROJECT_NUMBER': meson.project_version(), + 'OUTPUT_DIRECTORY' : meson.current_build_dir(), + 'src' : fs.parent(fs.parent(meson.project_source_root())) / 'src', + }, +) + +doxygen = find_program('doxygen', native : true, required : true) + +custom_target( + 'internal-api-docs', + command : [ doxygen , doxygen_cfg ], + input : [ + doxygen_cfg, + ], + output : 'html', + install : true, + install_dir : get_option('datadir') / 'doc/nix/internal-api', + build_always_stale : true, +) diff --git a/src/internal-api-docs/package.nix b/src/internal-api-docs/package.nix new file mode 100644 index 000000000..07ca6d4d9 --- /dev/null +++ b/src/internal-api-docs/package.nix @@ -0,0 +1,54 @@ +{ lib +, mkMesonDerivation + +, meson +, ninja +, doxygen + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-internal-api-docs"; + inherit version; + + workDir = ./.; + fileset = let + cpp = fileset.fileFilter (file: file.hasExt "cc" || file.hasExt "hh"); + in fileset.unions [ + ./.version + ../../.version + ./meson.build + ./doxygen.cfg.in + # Source is not compiled, but still must be available for Doxygen + # to gather comments. + (cpp ../.) + ]; + + nativeBuildInputs = [ + meson + ninja + doxygen + ]; + + preConfigure = + '' + chmod u+w ./.version + echo ${finalAttrs.version} > ./.version + ''; + + postInstall = '' + mkdir -p ''${!outputDoc}/nix-support + echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products + ''; + + meta = { + platforms = lib.platforms.all; + }; +}) diff --git a/src/libcmd/.version b/src/libcmd/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libcmd/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libcmd/build-utils-meson b/src/libcmd/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libcmd/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc index c5eb93c5d..905e70f32 100644 --- a/src/libcmd/built-path.cc +++ b/src/libcmd/built-path.cc @@ -1,6 +1,7 @@ #include "built-path.hh" #include "derivations.hh" #include "store-api.hh" +#include "comparator.hh" #include @@ -8,30 +9,24 @@ namespace nix { -#define CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, COMPARATOR) \ - bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \ - { \ - const MY_TYPE* me = this; \ - auto fields1 = std::tie(*me->drvPath, me->FIELD); \ - me = &other; \ - auto fields2 = std::tie(*me->drvPath, me->FIELD); \ - return fields1 COMPARATOR fields2; \ - } -#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, ==) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <) +// Custom implementation to avoid `ref` ptr equality +GENERATE_CMP_EXT( + , + std::strong_ordering, + SingleBuiltPathBuilt, + *me->drvPath, + me->output); -#define FIELD_TYPE std::pair -CMP(SingleBuiltPath, SingleBuiltPathBuilt, output) -#undef FIELD_TYPE +// Custom implementation to avoid `ref` ptr equality -#define FIELD_TYPE std::map -CMP(SingleBuiltPath, BuiltPathBuilt, outputs) -#undef FIELD_TYPE - -#undef CMP -#undef CMP_ONE +// TODO no `GENERATE_CMP_EXT` because no `std::set::operator<=>` on +// Darwin, per header. +GENERATE_EQUAL( + , + BuiltPathBuilt ::, + BuiltPathBuilt, + *me->drvPath, + me->outputs); StorePath SingleBuiltPath::outPath() const { diff --git a/src/libcmd/built-path.hh b/src/libcmd/built-path.hh index 99917e0ee..dc78d3e59 100644 --- a/src/libcmd/built-path.hh +++ b/src/libcmd/built-path.hh @@ -18,7 +18,8 @@ struct SingleBuiltPathBuilt { static SingleBuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view); nlohmann::json toJSON(const StoreDirConfig & store) const; - DECLARE_CMP(SingleBuiltPathBuilt); + bool operator ==(const SingleBuiltPathBuilt &) const noexcept; + std::strong_ordering operator <=>(const SingleBuiltPathBuilt &) const noexcept; }; using _SingleBuiltPathRaw = std::variant< @@ -33,6 +34,9 @@ struct SingleBuiltPath : _SingleBuiltPathRaw { using Opaque = DerivedPathOpaque; using Built = SingleBuiltPathBuilt; + bool operator == (const SingleBuiltPath &) const = default; + auto operator <=> (const SingleBuiltPath &) const = default; + inline const Raw & raw() const { return static_cast(*this); } @@ -59,11 +63,13 @@ struct BuiltPathBuilt { ref drvPath; std::map outputs; + bool operator == (const BuiltPathBuilt &) const noexcept; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //std::strong_ordering operator <=> (const BuiltPathBuilt &) const noexcept; + std::string to_string(const StoreDirConfig & store) const; static BuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view); nlohmann::json toJSON(const StoreDirConfig & store) const; - - DECLARE_CMP(BuiltPathBuilt); }; using _BuiltPathRaw = std::variant< @@ -82,6 +88,10 @@ struct BuiltPath : _BuiltPathRaw { using Opaque = DerivedPathOpaque; using Built = BuiltPathBuilt; + bool operator == (const BuiltPath &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=> (const BuiltPath &) const = default; + inline const Raw & raw() const { return static_cast(*this); } diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 369fa6004..67fef1909 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -1,3 +1,5 @@ +#include + #include "command.hh" #include "markdown.hh" #include "store-api.hh" @@ -6,8 +8,7 @@ #include "nixexpr.hh" #include "profiles.hh" #include "repl.hh" - -#include +#include "strings.hh" extern char * * environ __attribute__((weak)); @@ -127,12 +128,12 @@ ref EvalCommand::getEvalState() if (!evalState) { evalState = #if HAVE_BOEHMGC - std::allocate_shared(traceable_allocator(), - searchPath, getEvalStore(), getStore()) + std::allocate_shared( + traceable_allocator(), #else std::make_shared( - searchPath, getEvalStore(), getStore()) #endif + lookupPath, getEvalStore(), fetchSettings, evalSettings, getStore()) ; evalState->repair = repair; @@ -148,7 +149,7 @@ MixOperateOnOptions::MixOperateOnOptions() { addFlag({ .longName = "derivation", - .description = "Operate on the [store derivation](../../glossary.md#gloss-store-derivation) rather than its outputs.", + .description = "Operate on the [store derivation](@docroot@/glossary.md#gloss-store-derivation) rather than its outputs.", .category = installablesCategory, .handler = {&operateOn, OperateOn::Derivation}, }); diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 444ff81c9..fcef92487 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -1,18 +1,57 @@ +#include "fetch-settings.hh" #include "eval-settings.hh" #include "common-eval-args.hh" #include "shared.hh" +#include "config-global.hh" #include "filetransfer.hh" #include "eval.hh" #include "fetchers.hh" #include "registry.hh" #include "flake/flakeref.hh" +#include "flake/settings.hh" #include "store-api.hh" #include "command.hh" #include "tarball.hh" #include "fetch-to-store.hh" +#include "compatibility-settings.hh" +#include "eval-settings.hh" namespace nix { +fetchers::Settings fetchSettings; + +static GlobalConfig::Register rFetchSettings(&fetchSettings); + +EvalSettings evalSettings { + settings.readOnlyMode, + { + { + "flake", + [](ref store, std::string_view rest) { + experimentalFeatureSettings.require(Xp::Flakes); + // FIXME `parseFlakeRef` should take a `std::string_view`. + auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false); + debug("fetching flake search path element '%s''", rest); + auto storePath = flakeRef.resolve(store).fetchTree(store).first; + return store->toRealPath(storePath); + }, + }, + }, +}; + +static GlobalConfig::Register rEvalSettings(&evalSettings); + + +flake::Settings flakeSettings; + +static GlobalConfig::Register rFlakeSettings(&flakeSettings); + + +CompatibilitySettings compatibilitySettings {}; + +static GlobalConfig::Register rCompatibilitySettings(&compatibilitySettings); + + MixEvalArgs::MixEvalArgs() { addFlag({ @@ -20,7 +59,7 @@ MixEvalArgs::MixEvalArgs() .description = "Pass the value *expr* as the argument *name* to Nix functions.", .category = category, .labels = {"name", "expr"}, - .handler = {[&](std::string name, std::string expr) { autoArgs[name] = 'E' + expr; }} + .handler = {[&](std::string name, std::string expr) { autoArgs.insert_or_assign(name, AutoArg{AutoArgExpr{expr}}); }} }); addFlag({ @@ -28,87 +67,40 @@ MixEvalArgs::MixEvalArgs() .description = "Pass the string *string* as the argument *name* to Nix functions.", .category = category, .labels = {"name", "string"}, - .handler = {[&](std::string name, std::string s) { autoArgs[name] = 'S' + s; }}, + .handler = {[&](std::string name, std::string s) { autoArgs.insert_or_assign(name, AutoArg{AutoArgString{s}}); }}, + }); + + addFlag({ + .longName = "arg-from-file", + .description = "Pass the contents of file *path* as the argument *name* to Nix functions.", + .category = category, + .labels = {"name", "path"}, + .handler = {[&](std::string name, std::string path) { autoArgs.insert_or_assign(name, AutoArg{AutoArgFile{path}}); }}, + .completer = completePath + }); + + addFlag({ + .longName = "arg-from-stdin", + .description = "Pass the contents of stdin as the argument *name* to Nix functions.", + .category = category, + .labels = {"name"}, + .handler = {[&](std::string name) { autoArgs.insert_or_assign(name, AutoArg{AutoArgStdin{}}); }}, }); addFlag({ .longName = "include", .shortName = 'I', .description = R"( - Add *path* to the Nix search path. The Nix search path is - initialized from the colon-separated [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) environment - variable, and is used to look up the location of Nix expressions using [paths](@docroot@/language/values.md#type-path) enclosed in angle - brackets (i.e., ``). + Add *path* to search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md) - For instance, passing + This option may be given multiple times. - ``` - -I /home/eelco/Dev - -I /etc/nixos - ``` - - will cause Nix to look for paths relative to `/home/eelco/Dev` and - `/etc/nixos`, in that order. This is equivalent to setting the - `NIX_PATH` environment variable to - - ``` - /home/eelco/Dev:/etc/nixos - ``` - - It is also possible to match paths against a prefix. For example, - passing - - ``` - -I nixpkgs=/home/eelco/Dev/nixpkgs-branch - -I /etc/nixos - ``` - - will cause Nix to search for `` in - `/home/eelco/Dev/nixpkgs-branch/path` and `/etc/nixos/nixpkgs/path`. - - If a path in the Nix search path starts with `http://` or `https://`, - it is interpreted as the URL of a tarball that will be downloaded and - unpacked to a temporary location. The tarball must consist of a single - top-level directory. For example, passing - - ``` - -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/master.tar.gz - ``` - - tells Nix to download and use the current contents of the `master` - branch in the `nixpkgs` repository. - - The URLs of the tarballs from the official `nixos.org` channels - (see [the manual page for `nix-channel`](../nix-channel.md)) can be - abbreviated as `channel:`. For instance, the - following two flags are equivalent: - - ``` - -I nixpkgs=channel:nixos-21.05 - -I nixpkgs=https://nixos.org/channels/nixos-21.05/nixexprs.tar.xz - ``` - - You can also fetch source trees using [flake URLs](./nix3-flake.md#url-like-syntax) and add them to the - search path. For instance, - - ``` - -I nixpkgs=flake:nixpkgs - ``` - - specifies that the prefix `nixpkgs` shall refer to the source tree - downloaded from the `nixpkgs` entry in the flake registry. Similarly, - - ``` - -I nixpkgs=flake:github:NixOS/nixpkgs/nixos-22.05 - ``` - - makes `` refer to a particular branch of the - `NixOS/nixpkgs` repository on GitHub. + Paths added through `-I` take precedence over the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path) and the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH). )", .category = category, .labels = {"path"}, .handler = {[&](std::string s) { - searchPath.elements.emplace_back(SearchPath::Elem::parse(s)); + lookupPath.elements.emplace_back(LookupPath::Elem::parse(s)); }} }); @@ -127,8 +119,8 @@ MixEvalArgs::MixEvalArgs() .category = category, .labels = {"original-ref", "resolved-ref"}, .handler = {[&](std::string _from, std::string _to) { - auto from = parseFlakeRef(_from, absPath(".")); - auto to = parseFlakeRef(_to, absPath(".")); + auto from = parseFlakeRef(fetchSettings, _from, absPath(".")); + auto to = parseFlakeRef(fetchSettings, _to, absPath(".")); fetchers::Attrs extraAttrs; if (to.subdir != "") extraAttrs["dir"] = to.subdir; fetchers::overrideRegistry(from.input, to.input, extraAttrs); @@ -154,13 +146,23 @@ MixEvalArgs::MixEvalArgs() Bindings * MixEvalArgs::getAutoArgs(EvalState & state) { auto res = state.buildBindings(autoArgs.size()); - for (auto & i : autoArgs) { + for (auto & [name, arg] : autoArgs) { auto v = state.allocValue(); - if (i.second[0] == 'E') - state.mkThunk_(*v, state.parseExprFromString(i.second.substr(1), state.rootPath("."))); - else - v->mkString(((std::string_view) i.second).substr(1)); - res.insert(state.symbols.create(i.first), v); + std::visit(overloaded { + [&](const AutoArgExpr & arg) { + state.mkThunk_(*v, state.parseExprFromString(arg.expr, compatibilitySettings.nixShellShebangArgumentsRelativeToScript ? state.rootPath(absPath(getCommandBaseDir())) : state.rootPath("."))); + }, + [&](const AutoArgString & arg) { + v->mkString(arg.s); + }, + [&](const AutoArgFile & arg) { + v->mkString(readFile(arg.path.string())); + }, + [&](const AutoArgStdin & arg) { + v->mkString(readFile(STDIN_FILENO)); + } + }, arg); + res.insert(state.symbols.create(name), v); } return res.finish(); } @@ -176,7 +178,7 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas else if (hasPrefix(s, "flake:")) { experimentalFeatureSettings.require(Xp::Flakes); - auto flakeRef = parseFlakeRef(std::string(s.substr(6)), {}, true, false); + auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false); auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first; return state.rootPath(CanonPath(state.store->toRealPath(storePath))); } diff --git a/src/libcmd/common-eval-args.hh b/src/libcmd/common-eval-args.hh index 2eb63e15d..c62365b32 100644 --- a/src/libcmd/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh @@ -6,13 +6,42 @@ #include "common-args.hh" #include "search-path.hh" +#include + namespace nix { class Store; + +namespace fetchers { struct Settings; } + class EvalState; +struct EvalSettings; +struct CompatibilitySettings; class Bindings; struct SourcePath; +namespace flake { struct Settings; } + +/** + * @todo Get rid of global setttings variables + */ +extern fetchers::Settings fetchSettings; + +/** + * @todo Get rid of global setttings variables + */ +extern EvalSettings evalSettings; + +/** + * @todo Get rid of global setttings variables + */ +extern flake::Settings flakeSettings; + +/** + * Settings that control behaviors that have changed since Nix 2.3. + */ +extern CompatibilitySettings compatibilitySettings; + struct MixEvalArgs : virtual Args, virtual MixRepair { static constexpr auto category = "Common evaluation options"; @@ -21,14 +50,24 @@ struct MixEvalArgs : virtual Args, virtual MixRepair Bindings * getAutoArgs(EvalState & state); - SearchPath searchPath; + LookupPath lookupPath; std::optional evalStoreUrl; private: - std::map autoArgs; + struct AutoArgExpr { std::string expr; }; + struct AutoArgString { std::string s; }; + struct AutoArgFile { std::filesystem::path path; }; + struct AutoArgStdin { }; + + using AutoArg = std::variant; + + std::map autoArgs; }; +/** + * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) + */ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * baseDir = nullptr); } diff --git a/src/libcmd/compatibility-settings.hh b/src/libcmd/compatibility-settings.hh new file mode 100644 index 000000000..a129a957a --- /dev/null +++ b/src/libcmd/compatibility-settings.hh @@ -0,0 +1,36 @@ +#pragma once +#include "config.hh" + +namespace nix { +struct CompatibilitySettings : public Config +{ + + CompatibilitySettings() = default; + + // Added in Nix 2.24, July 2024. + Setting nixShellAlwaysLooksForShellNix{this, true, "nix-shell-always-looks-for-shell-nix", R"( + Before Nix 2.24, [`nix-shell`](@docroot@/command-ref/nix-shell.md) would only look at `shell.nix` if it was in the working directory - when no file was specified. + + Since Nix 2.24, `nix-shell` always looks for a `shell.nix`, whether that's in the working directory, or in a directory that was passed as an argument. + + You may set this to `false` to temporarily revert to the behavior of Nix 2.23 and older. + + Using this setting is not recommended. + It will be deprecated and removed. + )"}; + + // Added in Nix 2.24, July 2024. + Setting nixShellShebangArgumentsRelativeToScript{ + this, true, "nix-shell-shebang-arguments-relative-to-script", R"( + Before Nix 2.24, relative file path expressions in arguments in a `nix-shell` shebang were resolved relative to the working directory. + + Since Nix 2.24, `nix-shell` resolves these paths in a manner that is relative to the [base directory](@docroot@/glossary.md#gloss-base-directory), defined as the script's directory. + + You may set this to `false` to temporarily revert to the behavior of Nix 2.23 and older. + + Using this setting is not recommended. + It will be deprecated and removed. + )"}; +}; + +}; diff --git a/src/libcmd/installable-attr-path.cc b/src/libcmd/installable-attr-path.cc index 3ec1c1614..8917e7a01 100644 --- a/src/libcmd/installable-attr-path.cc +++ b/src/libcmd/installable-attr-path.cc @@ -75,6 +75,8 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths() std::set outputsToInstall; for (auto & output : packageInfo.queryOutputs(false, true)) outputsToInstall.insert(output.first); + if (outputsToInstall.empty()) + outputsToInstall.insert("out"); return OutputsSpec::Names { std::move(outputsToInstall) }; }, [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec { diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index ddec7537b..8796ad5ba 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -43,20 +43,6 @@ std::vector InstallableFlake::getActualAttrPaths() return res; } -Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake) -{ - auto vFlake = state.allocValue(); - - callFlake(state, lockedFlake, *vFlake); - - auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); - assert(aOutputs); - - state.forceValue(*aOutputs->value, aOutputs->value->determinePos(noPos)); - - return aOutputs->value; -} - static std::string showAttrPaths(const std::vector & paths) { std::string s; @@ -106,9 +92,14 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() fmt("while evaluating the flake output attribute '%s'", attrPath))) { return { *derivedPathWithInfo }; + } else { + throw Error( + "expected flake output attribute '%s' to be a derivation or path but found %s: %s", + attrPath, + showType(v), + ValuePrinter(*this->state, v, errorPrintOptions) + ); } - else - throw Error("flake output attribute '%s' is not a derivation or path", attrPath); } auto drvPath = attr->forceDerivation(); @@ -205,7 +196,8 @@ std::shared_ptr InstallableFlake::getLockedFlake() const flake::LockFlags lockFlagsApplyConfig = lockFlags; // FIXME why this side effect? lockFlagsApplyConfig.applyNixConfig = true; - _lockedFlake = std::make_shared(lockFlake(*state, flakeRef, lockFlagsApplyConfig)); + _lockedFlake = std::make_shared(lockFlake( + flakeSettings, *state, flakeRef, lockFlagsApplyConfig)); } return _lockedFlake; } diff --git a/src/libcmd/installable-flake.hh b/src/libcmd/installable-flake.hh index 314918c14..8e0a232ef 100644 --- a/src/libcmd/installable-flake.hh +++ b/src/libcmd/installable-flake.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "common-eval-args.hh" #include "installable-value.hh" namespace nix { @@ -52,8 +53,6 @@ struct InstallableFlake : InstallableValue std::vector getActualAttrPaths(); - Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake); - DerivedPathsWithInfo toDerivedPaths() override; std::pair toValue(EvalState & state) override; @@ -80,7 +79,7 @@ struct InstallableFlake : InstallableValue */ static inline FlakeRef defaultNixpkgsFlakeRef() { - return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}}); + return FlakeRef::fromAttrs(fetchSettings, {{"type","indirect"}, {"id", "nixpkgs"}}); } ref openEvalCache( diff --git a/src/libcmd/installable-value.hh b/src/libcmd/installable-value.hh index f300d392b..798cb5e1a 100644 --- a/src/libcmd/installable-value.hh +++ b/src/libcmd/installable-value.hh @@ -66,7 +66,7 @@ struct ExtraPathInfoValue : ExtraPathInfo }; /** - * An Installable which corresponds a Nix langauge value, in addition to + * An Installable which corresponds a Nix language value, in addition to * a collection of \ref DerivedPath "derived paths". */ struct InstallableValue : Installable diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 16d25d3cf..0fe956ec0 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -27,6 +27,8 @@ #include +#include "strings-inline.hh" + namespace nix { void completeFlakeInputPath( @@ -129,7 +131,7 @@ MixFlakeOptions::MixFlakeOptions() lockFlags.writeLockFile = false; lockFlags.inputOverrides.insert_or_assign( flake::parseInputPath(inputPath), - parseFlakeRef(flakeRef, absPath(getCommandBaseDir()), true)); + parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir()), true)); }}, .completer = {[&](AddCompletions & completions, size_t n, std::string_view prefix) { if (n == 0) { @@ -146,7 +148,7 @@ MixFlakeOptions::MixFlakeOptions() .category = category, .labels = {"flake-lock-path"}, .handler = {[&](std::string lockFilePath) { - lockFlags.referenceLockFilePath = lockFilePath; + lockFlags.referenceLockFilePath = {getFSSourceAccessor(), CanonPath(absPath(lockFilePath))}; }}, .completer = completePath }); @@ -170,14 +172,15 @@ MixFlakeOptions::MixFlakeOptions() .handler = {[&](std::string flakeRef) { auto evalState = getEvalState(); auto flake = flake::lockFlake( + flakeSettings, *evalState, - parseFlakeRef(flakeRef, absPath(getCommandBaseDir())), + parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir())), { .writeLockFile = false }); for (auto & [inputName, input] : flake.lockFile.root->inputs) { auto input2 = flake.lockFile.findInput({inputName}); // resolve 'follows' nodes if (auto input3 = std::dynamic_pointer_cast(input2)) { overrideRegistry( - fetchers::Input::fromAttrs({{"type","indirect"}, {"id", inputName}}), + fetchers::Input::fromAttrs(fetchSettings, {{"type","indirect"}, {"id", inputName}}), input3->lockedRef.input, {}); } @@ -288,11 +291,11 @@ void SourceExprCommand::completeInstallable(AddCompletions & completions, std::s state->autoCallFunction(*autoArgs, v1, v2); if (v2.type() == nAttrs) { - for (auto & i : *v2.attrs) { - std::string name = state->symbols[i.name]; + for (auto & i : *v2.attrs()) { + std::string_view name = state->symbols[i.name]; if (name.find(searchWord) == 0) { if (prefix_ == "") - completions.add(name); + completions.add(std::string(name)); else completions.add(prefix_ + "." + name); } @@ -338,10 +341,11 @@ void completeFlakeRefWithFragment( auto flakeRefS = std::string(prefix.substr(0, hash)); // TODO: ideally this would use the command base directory instead of assuming ".". - auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath(".")); + auto flakeRef = parseFlakeRef(fetchSettings, expandTilde(flakeRefS), absPath(".")); auto evalCache = openEvalCache(*evalState, - std::make_shared(lockFlake(*evalState, flakeRef, lockFlags))); + std::make_shared(lockFlake( + flakeSettings, *evalState, flakeRef, lockFlags))); auto root = evalCache->getRoot(); @@ -372,6 +376,7 @@ void completeFlakeRefWithFragment( auto attrPath2 = (*attr)->getAttrPath(attr2); /* Strip the attrpath prefix. */ attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size()); + // FIXME: handle names with dots completions.add(flakeRefS + "#" + prefixRoot + concatStringsSep(".", evalState->symbols.resolve(attrPath2))); } } @@ -403,7 +408,7 @@ void completeFlakeRef(AddCompletions & completions, ref store, std::strin Args::completeDir(completions, 0, prefix); /* Look for registry entries that match the prefix. */ - for (auto & registry : fetchers::getRegistries(store)) { + for (auto & registry : fetchers::getRegistries(fetchSettings, store)) { for (auto & entry : registry->entries) { auto from = entry.from.to_string(); if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) { @@ -442,13 +447,10 @@ ref openEvalCache( EvalState & state, std::shared_ptr lockedFlake) { - auto fingerprint = lockedFlake->getFingerprint(); - return make_ref( - evalSettings.useEvalCache && evalSettings.pureEval - ? std::optional { std::cref(fingerprint) } - : std::nullopt, - state, - [&state, lockedFlake]() + auto fingerprint = evalSettings.useEvalCache && evalSettings.pureEval + ? lockedFlake->getFingerprint(state.store) + : std::nullopt; + auto rootLoader = [&state, lockedFlake]() { /* For testing whether the evaluation cache is complete. */ @@ -460,11 +462,21 @@ ref openEvalCache( state.forceAttrs(*vFlake, noPos, "while parsing cached flake data"); - auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); + auto aOutputs = vFlake->attrs()->get(state.symbols.create("outputs")); assert(aOutputs); return aOutputs->value; - }); + }; + + if (fingerprint) { + auto search = state.evalCaches.find(fingerprint.value()); + if (search == state.evalCaches.end()) { + search = state.evalCaches.emplace(fingerprint.value(), make_ref(fingerprint, state, rootLoader)).first; + } + return search->second; + } else { + return make_ref(std::nullopt, state, rootLoader); + } } Installables SourceExprCommand::parseInstallables( @@ -527,7 +539,8 @@ Installables SourceExprCommand::parseInstallables( } try { - auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, absPath(getCommandBaseDir())); + auto [flakeRef, fragment] = parseFlakeRefWithFragment( + fetchSettings, std::string { prefix }, absPath(getCommandBaseDir())); result.push_back(make_ref( this, getEvalState(), @@ -594,6 +607,37 @@ std::vector Installable::build( return res; } +static void throwBuildErrors( + std::vector & buildResults, + const Store & store) +{ + std::vector failed; + for (auto & buildResult : buildResults) { + if (!buildResult.success()) { + failed.push_back(buildResult); + } + } + + auto failedResult = failed.begin(); + if (failedResult != failed.end()) { + if (failed.size() == 1) { + failedResult->rethrow(); + } else { + StringSet failedPaths; + for (; failedResult != failed.end(); failedResult++) { + if (!failedResult->errorMsg.empty()) { + logError(ErrorInfo{ + .level = lvlError, + .msg = failedResult->errorMsg, + }); + } + failedPaths.insert(failedResult->path.to_string(store)); + } + throw Error("build of %s failed", concatStringsSep(", ", quoteStrings(failedPaths))); + } + } +} + std::vector, BuiltPathWithResult>> Installable::build2( ref evalStore, ref store, @@ -655,10 +699,9 @@ std::vector, BuiltPathWithResult>> Installable::build if (settings.printMissing) printMissing(store, pathsToBuild, lvlInfo); - for (auto & buildResult : store->buildPathsWithResults(pathsToBuild, bMode, evalStore)) { - if (!buildResult.success()) - buildResult.rethrow(); - + auto buildResults = store->buildPathsWithResults(pathsToBuild, bMode, evalStore); + throwBuildErrors(buildResults, *store); + for (auto & buildResult : buildResults) { for (auto & aux : backmap[buildResult.path]) { std::visit(overloaded { [&](const DerivedPath::Built & bfd) { @@ -814,6 +857,7 @@ std::vector RawInstallablesCommand::getFlakeRefsForCompletion() std::vector res; for (auto i : rawInstallables) res.push_back(parseFlakeRefWithFragment( + fetchSettings, expandTilde(i), absPath(getCommandBaseDir())).first); return res; @@ -836,6 +880,7 @@ std::vector InstallableCommand::getFlakeRefsForCompletion() { return { parseFlakeRefWithFragment( + fetchSettings, expandTilde(_installable), absPath(getCommandBaseDir())).first }; diff --git a/src/libcmd/local.mk b/src/libcmd/local.mk index abb7459a7..a270333f4 100644 --- a/src/libcmd/local.mk +++ b/src/libcmd/local.mk @@ -6,10 +6,10 @@ libcmd_DIR := $(d) libcmd_SOURCES := $(wildcard $(d)/*.cc) -libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers +libcmd_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) $(INCLUDE_libflake) $(INCLUDE_libmain) libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) $(THREAD_LDFLAGS) -libcmd_LIBS = libstore libutil libexpr libmain libfetchers +libcmd_LIBS = libutil libstore libfetchers libflake libexpr libmain $(eval $(call install-file-in, $(buildprefix)$(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644)) diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc index a4e3c5a77..6a0d05d9f 100644 --- a/src/libcmd/markdown.cc +++ b/src/libcmd/markdown.cc @@ -1,21 +1,23 @@ #include "markdown.hh" -#include "util.hh" +#include "environment-variables.hh" +#include "error.hh" #include "finally.hh" #include "terminal.hh" -#include #if HAVE_LOWDOWN -#include +# include +# include #endif namespace nix { -std::string renderMarkdownToTerminal(std::string_view markdown) -{ #if HAVE_LOWDOWN +static std::string doRenderMarkdownToTerminal(std::string_view markdown) +{ int windowWidth = getWindowSize().second; - struct lowdown_opts opts { + struct lowdown_opts opts + { .type = LOWDOWN_TERM, .maxdepth = 20, .cols = (size_t) std::max(windowWidth - 5, 60), @@ -50,10 +52,22 @@ std::string renderMarkdownToTerminal(std::string_view markdown) if (!rndr_res) throw Error("allocation error while rendering Markdown"); - return filterANSIEscapes(std::string(buf->data, buf->size), !shouldANSI()); -#else - return std::string(markdown); -#endif + return filterANSIEscapes(std::string(buf->data, buf->size), !isTTY()); } +std::string renderMarkdownToTerminal(std::string_view markdown) +{ + if (auto e = getEnv("_NIX_TEST_RAW_MARKDOWN"); e && *e == "1") + return std::string(markdown); + else + return doRenderMarkdownToTerminal(markdown); } + +#else +std::string renderMarkdownToTerminal(std::string_view markdown) +{ + return std::string(markdown); +} +#endif + +} // namespace nix diff --git a/src/libcmd/markdown.hh b/src/libcmd/markdown.hh index a04d32a4f..66db1736c 100644 --- a/src/libcmd/markdown.hh +++ b/src/libcmd/markdown.hh @@ -1,10 +1,17 @@ #pragma once ///@file -#include "types.hh" +#include namespace nix { +/** + * Render the given Markdown text to the terminal. + * + * If Nix is compiled without Markdown support, this function will return the input text as-is. + * + * The renderer takes into account the terminal width, and wraps text accordingly. + */ std::string renderMarkdownToTerminal(std::string_view markdown); } diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build new file mode 100644 index 000000000..c484cf998 --- /dev/null +++ b/src/libcmd/meson.build @@ -0,0 +1,130 @@ +project('nix-cmd', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ +] +deps_public_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), + dependency('nix-fetchers'), + dependency('nix-expr'), + dependency('nix-flake'), + dependency('nix-main'), +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') +deps_public += nlohmann_json + +lowdown = dependency('lowdown', version : '>= 0.9.0', required : get_option('markdown')) +deps_private += lowdown +configdata.set('HAVE_LOWDOWN', lowdown.found().to_int()) + +readline_flavor = get_option('readline-flavor') +if readline_flavor == 'editline' + editline = dependency('libeditline', 'editline', version : '>=1.14') + deps_private += editline +elif readline_flavor == 'readline' + readline = dependency('readline') + deps_private += readline + configdata.set( + 'USE_READLINE', + 1, + description: 'Use readline instead of editline', + ) +else + error('illegal editline flavor', readline_flavor) +endif + +config_h = configure_file( + configuration : configdata, + output : 'config-cmd.hh', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + # '-include', 'config-fetchers.h', + '-include', 'config-expr.hh', + '-include', 'config-main.hh', + '-include', 'config-cmd.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'built-path.cc', + 'command-installable-value.cc', + 'command.cc', + 'common-eval-args.cc', + 'editor-for.cc', + 'installable-attr-path.cc', + 'installable-derived-path.cc', + 'installable-flake.cc', + 'installable-value.cc', + 'installables.cc', + 'legacy.cc', + 'markdown.cc', + 'misc-store-flags.cc', + 'network-proxy.cc', + 'repl-interacter.cc', + 'repl.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'built-path.hh', + 'command-installable-value.hh', + 'command.hh', + 'common-eval-args.hh', + 'compatibility-settings.hh', + 'editor-for.hh', + 'installable-attr-path.hh', + 'installable-derived-path.hh', + 'installable-flake.hh', + 'installable-value.hh', + 'installables.hh', + 'legacy.hh', + 'markdown.hh', + 'misc-store-flags.hh', + 'network-proxy.hh', + 'repl-interacter.hh', + 'repl.hh', +) + +this_library = library( + 'nixcmd', + sources, + dependencies : deps_public + deps_private + deps_other, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libcmd/meson.options b/src/libcmd/meson.options new file mode 100644 index 000000000..79ae4fa55 --- /dev/null +++ b/src/libcmd/meson.options @@ -0,0 +1,15 @@ +# vim: filetype=meson + +option( + 'markdown', + type: 'feature', + description: 'Enable Markdown rendering in the Nix binary (requires lowdown)', +) + +option( + 'readline-flavor', + type : 'combo', + choices : ['editline', 'readline'], + value : 'editline', + description : 'Which library to use for nice line editing with the Nix language REPL', +) diff --git a/src/libcmd/misc-store-flags.cc b/src/libcmd/misc-store-flags.cc index e66d3f63b..06552c032 100644 --- a/src/libcmd/misc-store-flags.cc +++ b/src/libcmd/misc-store-flags.cc @@ -81,9 +81,15 @@ Args::Flag fileIngestionMethod(FileIngestionMethod * method) How to compute the hash of the input. One of: - - `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function. + - `nar` (the default): + Serialises the input as a + [Nix Archive](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) + and passes that to the hash function. - - `flat`: Assumes that the input is a single file and directly passes it to the hash function; + - `flat`: + Assumes that the input is a single file and + [directly passes](@docroot@/store/file-system-object/content-address.md#serial-flat) + it to the hash function. )", .labels = {"file-ingestion-method"}, .handler = {[method](std::string s) { @@ -101,15 +107,23 @@ Args::Flag contentAddressMethod(ContentAddressMethod * method) How to compute the content-address of the store object. One of: - - `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function. + - [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive) + (the default): + Serialises the input as a + [Nix Archive](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) + and passes that to the hash function. - - `flat`: Assumes that the input is a single file and directly passes it to the hash function; + - [`flat`](@docroot@/store/store-object/content-address.md#method-flat): + Assumes that the input is a single file and + [directly passes](@docroot@/store/file-system-object/content-address.md#serial-flat) + it to the hash function. - - `text`: Like `flat`, but used for - [derivations](@docroot@/glossary.md#store-derivation) serialized in store object and + - [`text`](@docroot@/store/store-object/content-address.md#method-text): + Like `flat`, but used for + [derivations](@docroot@/glossary.md#store-derivation) serialized in store object and [`builtins.toFile`](@docroot@/language/builtins.html#builtins-toFile). For advanced use-cases only; - for regular usage prefer `nar` and `flat. + for regular usage prefer `nar` and `flat`. )", .labels = {"content-address-method"}, .handler = {[method](std::string s) { diff --git a/src/libcmd/network-proxy.cc b/src/libcmd/network-proxy.cc new file mode 100644 index 000000000..738bf6147 --- /dev/null +++ b/src/libcmd/network-proxy.cc @@ -0,0 +1,50 @@ +#include "network-proxy.hh" + +#include + +#include "environment-variables.hh" + +namespace nix { + +static const StringSet lowercaseVariables{"http_proxy", "https_proxy", "ftp_proxy", "all_proxy", "no_proxy"}; + +static StringSet getAllVariables() +{ + StringSet variables = lowercaseVariables; + for (const auto & variable : lowercaseVariables) { + std::string upperVariable; + std::transform( + variable.begin(), variable.end(), upperVariable.begin(), [](unsigned char c) { return std::toupper(c); }); + variables.insert(std::move(upperVariable)); + } + return variables; +} + +const StringSet networkProxyVariables = getAllVariables(); + +static StringSet getExcludingNoProxyVariables() +{ + static const StringSet excludeVariables{"no_proxy", "NO_PROXY"}; + StringSet variables; + std::set_difference( + networkProxyVariables.begin(), + networkProxyVariables.end(), + excludeVariables.begin(), + excludeVariables.end(), + std::inserter(variables, variables.begin())); + return variables; +} + +static const StringSet excludingNoProxyVariables = getExcludingNoProxyVariables(); + +bool haveNetworkProxyConnection() +{ + for (const auto & variable : excludingNoProxyVariables) { + if (getEnv(variable).has_value()) { + return true; + } + } + return false; +} + +} diff --git a/src/libcmd/network-proxy.hh b/src/libcmd/network-proxy.hh new file mode 100644 index 000000000..0b6856acb --- /dev/null +++ b/src/libcmd/network-proxy.hh @@ -0,0 +1,22 @@ +#pragma once +///@file + +#include "types.hh" + +namespace nix { + +/** + * Environment variables relating to network proxying. These are used by + * a few misc commands. + * + * See the Environment section of https://curl.se/docs/manpage.html for details. + */ +extern const StringSet networkProxyVariables; + +/** + * Heuristically check if there is a proxy connection by checking for defined + * proxy variables. + */ +bool haveNetworkProxyConnection(); + +} diff --git a/src/libcmd/package.nix b/src/libcmd/package.nix new file mode 100644 index 000000000..cde494901 --- /dev/null +++ b/src/libcmd/package.nix @@ -0,0 +1,104 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-util +, nix-store +, nix-fetchers +, nix-expr +, nix-flake +, nix-main +, editline +, readline +, lowdown +, nlohmann_json + +# Configuration Options + +, version + +# Whether to enable Markdown rendering in the Nix binary. +, enableMarkdown ? !stdenv.hostPlatform.isWindows + +# Which interactive line editor library to use for Nix's repl. +# +# Currently supported choices are: +# +# - editline (default) +# - readline +, readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline" +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-cmd"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + ({ inherit editline readline; }.${readlineFlavor}) + ] ++ lib.optional enableMarkdown lowdown; + + propagatedBuildInputs = [ + nix-util + nix-store + nix-fetchers + nix-expr + nix-flake + nix-main + nlohmann_json + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + mesonFlags = [ + (lib.mesonEnable "markdown" enableMarkdown) + (lib.mesonOption "readline-flavor" readlineFlavor) + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libcmd/repl-interacter.cc b/src/libcmd/repl-interacter.cc new file mode 100644 index 000000000..187af46ea --- /dev/null +++ b/src/libcmd/repl-interacter.cc @@ -0,0 +1,206 @@ +#include + +#ifdef USE_READLINE +#include +#include +#else +// editline < 1.15.2 don't wrap their API for C++ usage +// (added in https://github.com/troglobit/editline/commit/91398ceb3427b730995357e9d120539fb9bb7461). +// This results in linker errors due to to name-mangling of editline C symbols. +// For compatibility with these versions, we wrap the API here +// (wrapping multiple times on newer versions is no problem). +extern "C" { +#include +} +#endif + +#include "signals.hh" +#include "finally.hh" +#include "repl-interacter.hh" +#include "file-system.hh" +#include "repl.hh" +#include "environment-variables.hh" + +namespace nix { + +namespace { +// Used to communicate to NixRepl::getLine whether a signal occurred in ::readline. +volatile sig_atomic_t g_signal_received = 0; + +void sigintHandler(int signo) +{ + g_signal_received = signo; +} +}; + +static detail::ReplCompleterMixin * curRepl; // ugly + +#ifndef USE_READLINE +static char * completionCallback(char * s, int * match) +{ + auto possible = curRepl->completePrefix(s); + if (possible.size() == 1) { + *match = 1; + auto * res = strdup(possible.begin()->c_str() + strlen(s)); + if (!res) + throw Error("allocation failure"); + return res; + } else if (possible.size() > 1) { + auto checkAllHaveSameAt = [&](size_t pos) { + auto & first = *possible.begin(); + for (auto & p : possible) { + if (p.size() <= pos || p[pos] != first[pos]) + return false; + } + return true; + }; + size_t start = strlen(s); + size_t len = 0; + while (checkAllHaveSameAt(start + len)) + ++len; + if (len > 0) { + *match = 1; + auto * res = strdup(std::string(*possible.begin(), start, len).c_str()); + if (!res) + throw Error("allocation failure"); + return res; + } + } + + *match = 0; + return nullptr; +} + +static int listPossibleCallback(char * s, char *** avp) +{ + auto possible = curRepl->completePrefix(s); + + if (possible.size() > (std::numeric_limits::max() / sizeof(char *))) + throw Error("too many completions"); + + int ac = 0; + char ** vp = nullptr; + + auto check = [&](auto * p) { + if (!p) { + if (vp) { + while (--ac >= 0) + free(vp[ac]); + free(vp); + } + throw Error("allocation failure"); + } + return p; + }; + + vp = check((char **) malloc(possible.size() * sizeof(char *))); + + for (auto & p : possible) + vp[ac++] = check(strdup(p.c_str())); + + *avp = vp; + + return ac; +} +#endif + +ReadlineLikeInteracter::Guard ReadlineLikeInteracter::init(detail::ReplCompleterMixin * repl) +{ + // Allow nix-repl specific settings in .inputrc + rl_readline_name = "nix-repl"; + try { + createDirs(dirOf(historyFile)); + } catch (SystemError & e) { + logWarning(e.info()); + } +#ifndef USE_READLINE + el_hist_size = 1000; +#endif + read_history(historyFile.c_str()); + auto oldRepl = curRepl; + curRepl = repl; + Guard restoreRepl([oldRepl] { curRepl = oldRepl; }); +#ifndef USE_READLINE + rl_set_complete_func(completionCallback); + rl_set_list_possib_func(listPossibleCallback); +#endif + return restoreRepl; +} + +static constexpr const char * promptForType(ReplPromptType promptType) +{ + switch (promptType) { + case ReplPromptType::ReplPrompt: + return "nix-repl> "; + case ReplPromptType::ContinuationPrompt: + return " "; + } + assert(false); +} + +bool ReadlineLikeInteracter::getLine(std::string & input, ReplPromptType promptType) +{ +#ifndef _WIN32 // TODO use more signals.hh for this + struct sigaction act, old; + sigset_t savedSignalMask, set; + + auto setupSignals = [&]() { + act.sa_handler = sigintHandler; + sigfillset(&act.sa_mask); + act.sa_flags = 0; + if (sigaction(SIGINT, &act, &old)) + throw SysError("installing handler for SIGINT"); + + sigemptyset(&set); + sigaddset(&set, SIGINT); + if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask)) + throw SysError("unblocking SIGINT"); + }; + auto restoreSignals = [&]() { + if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) + throw SysError("restoring signals"); + + if (sigaction(SIGINT, &old, 0)) + throw SysError("restoring handler for SIGINT"); + }; + + setupSignals(); +#endif + char * s = readline(promptForType(promptType)); + Finally doFree([&]() { free(s); }); +#ifndef _WIN32 // TODO use more signals.hh for this + restoreSignals(); +#endif + + if (g_signal_received) { + g_signal_received = 0; + input.clear(); + return true; + } + + // editline doesn't echo the input to the output when non-interactive, unlike readline + // this results in a different behavior when running tests. The echoing is + // quite useful for reading the test output, so we add it here. + if (auto e = getEnv("_NIX_TEST_REPL_ECHO"); s && e && *e == "1") + { +#ifndef USE_READLINE + // This is probably not right for multi-line input, but we don't use that + // in the characterisation tests, so it's fine. + std::cout << promptForType(promptType) << s << std::endl; +#endif + } + + if (!s) + return false; + input += s; + input += '\n'; + + return true; +} + +ReadlineLikeInteracter::~ReadlineLikeInteracter() +{ + write_history(historyFile.c_str()); +} + +}; diff --git a/src/libcmd/repl-interacter.hh b/src/libcmd/repl-interacter.hh new file mode 100644 index 000000000..cc70efd07 --- /dev/null +++ b/src/libcmd/repl-interacter.hh @@ -0,0 +1,48 @@ +#pragma once +/// @file + +#include "finally.hh" +#include "types.hh" +#include +#include + +namespace nix { + +namespace detail { +/** Provides the completion hooks for the repl, without exposing its complete + * internals. */ +struct ReplCompleterMixin { + virtual StringSet completePrefix(const std::string & prefix) = 0; +}; +}; + +enum class ReplPromptType { + ReplPrompt, + ContinuationPrompt, +}; + +class ReplInteracter +{ +public: + using Guard = Finally>; + + virtual Guard init(detail::ReplCompleterMixin * repl) = 0; + /** Returns a boolean of whether the interacter got EOF */ + virtual bool getLine(std::string & input, ReplPromptType promptType) = 0; + virtual ~ReplInteracter(){}; +}; + +class ReadlineLikeInteracter : public virtual ReplInteracter +{ + std::string historyFile; +public: + ReadlineLikeInteracter(std::string historyFile) + : historyFile(historyFile) + { + } + virtual Guard init(detail::ReplCompleterMixin * repl) override; + virtual bool getLine(std::string & input, ReplPromptType promptType) override; + virtual ~ReadlineLikeInteracter() override; +}; + +}; diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 8b83608fa..efc04b029 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -1,34 +1,17 @@ #include #include #include -#include - -#include - -#ifdef USE_READLINE -#include -#include -#else -// editline < 1.15.2 don't wrap their API for C++ usage -// (added in https://github.com/troglobit/editline/commit/91398ceb3427b730995357e9d120539fb9bb7461). -// This results in linker errors due to to name-mangling of editline C symbols. -// For compatibility with these versions, we wrap the API here -// (wrapping multiple times on newer versions is no problem). -extern "C" { -#include -} -#endif +#include "repl-interacter.hh" #include "repl.hh" #include "ansicolor.hh" -#include "signals.hh" #include "shared.hh" +#include "config-global.hh" #include "eval.hh" -#include "eval-cache.hh" -#include "eval-inline.hh" #include "eval-settings.hh" #include "attr-path.hh" +#include "signals.hh" #include "store-api.hh" #include "log-store.hh" #include "common-eval-args.hh" @@ -38,18 +21,21 @@ extern "C" { #include "flake/flake.hh" #include "flake/lockfile.hh" #include "users.hh" -#include "terminal.hh" #include "editor-for.hh" #include "finally.hh" #include "markdown.hh" #include "local-fs-store.hh" #include "print.hh" +#include "ref.hh" +#include "value.hh" #if HAVE_BOEHMGC #define GC_INCLUDE_NEW #include #endif +#include "strings.hh" + namespace nix { /** @@ -75,6 +61,7 @@ enum class ProcessLineResult { struct NixRepl : AbstractNixRepl + , detail::ReplCompleterMixin #if HAVE_BOEHMGC , gc #endif @@ -90,17 +77,16 @@ struct NixRepl int displ; StringSet varNames; - const Path historyFile; + std::unique_ptr interacter; - NixRepl(const SearchPath & searchPath, nix::ref store,ref state, + NixRepl(const LookupPath & lookupPath, nix::ref store,ref state, std::function getValues); - virtual ~NixRepl(); + virtual ~NixRepl() = default; ReplExitStatus mainLoop() override; void initEnv() override; - StringSet completePrefix(const std::string & prefix); - bool getLine(std::string & input, const std::string & prompt); + virtual StringSet completePrefix(const std::string & prefix) override; StorePath getDerivationPath(Value & v); ProcessLineResult processLine(std::string line); @@ -123,7 +109,8 @@ struct NixRepl .force = true, .derivationPaths = true, .maxDepth = maxDepth, - .prettyIndent = 2 + .prettyIndent = 2, + .errors = ErrorPrintBehavior::ThrowTopLevel, }); } }; @@ -137,111 +124,33 @@ std::string removeWhitespace(std::string s) } -NixRepl::NixRepl(const SearchPath & searchPath, nix::ref store, ref state, +NixRepl::NixRepl(const LookupPath & lookupPath, nix::ref store, ref state, std::function getValues) : AbstractNixRepl(state) , debugTraceIndex(0) , getValues(getValues) , staticEnv(new StaticEnv(nullptr, state->staticBaseEnv.get())) - , historyFile(getDataDir() + "/nix/repl-history") + , interacter(make_unique(getDataDir() + "/nix/repl-history")) { } - -NixRepl::~NixRepl() -{ - write_history(historyFile.c_str()); -} - void runNix(Path program, const Strings & args, const std::optional & input = {}) { auto subprocessEnv = getEnv(); subprocessEnv["NIX_CONFIG"] = globalConfig.toKeyValue(); - + //isInteractive avoid grabling interactive commands runProgram2(RunOptions { .program = settings.nixBinDir+ "/" + program, .args = args, .environment = subprocessEnv, .input = input, + .isInteractive = true, }); return; } -static NixRepl * curRepl; // ugly - -static char * completionCallback(char * s, int *match) { - auto possible = curRepl->completePrefix(s); - if (possible.size() == 1) { - *match = 1; - auto *res = strdup(possible.begin()->c_str() + strlen(s)); - if (!res) throw Error("allocation failure"); - return res; - } else if (possible.size() > 1) { - auto checkAllHaveSameAt = [&](size_t pos) { - auto &first = *possible.begin(); - for (auto &p : possible) { - if (p.size() <= pos || p[pos] != first[pos]) - return false; - } - return true; - }; - size_t start = strlen(s); - size_t len = 0; - while (checkAllHaveSameAt(start + len)) ++len; - if (len > 0) { - *match = 1; - auto *res = strdup(std::string(*possible.begin(), start, len).c_str()); - if (!res) throw Error("allocation failure"); - return res; - } - } - - *match = 0; - return nullptr; -} - -static int listPossibleCallback(char *s, char ***avp) { - auto possible = curRepl->completePrefix(s); - - if (possible.size() > (INT_MAX / sizeof(char*))) - throw Error("too many completions"); - - int ac = 0; - char **vp = nullptr; - - auto check = [&](auto *p) { - if (!p) { - if (vp) { - while (--ac >= 0) - free(vp[ac]); - free(vp); - } - throw Error("allocation failure"); - } - return p; - }; - - vp = check((char **)malloc(possible.size() * sizeof(char*))); - - for (auto & p : possible) - vp[ac++] = check(strdup(p.c_str())); - - *avp = vp; - - return ac; -} - -namespace { - // Used to communicate to NixRepl::getLine whether a signal occurred in ::readline. - volatile sig_atomic_t g_signal_received = 0; - - void sigintHandler(int signo) { - g_signal_received = signo; - } -} - static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positions, const DebugTrace & dt) { if (dt.isError) @@ -281,24 +190,7 @@ ReplExitStatus NixRepl::mainLoop() loadFiles(); - // Allow nix-repl specific settings in .inputrc - rl_readline_name = "nix-repl"; - try { - createDirs(dirOf(historyFile)); - } catch (SystemError & e) { - logWarning(e.info()); - } -#ifndef USE_READLINE - el_hist_size = 1000; -#endif - read_history(historyFile.c_str()); - auto oldRepl = curRepl; - curRepl = this; - Finally restoreRepl([&] { curRepl = oldRepl; }); -#ifndef USE_READLINE - rl_set_complete_func(completionCallback); - rl_set_list_possib_func(listPossibleCallback); -#endif + auto _guard = interacter->init(static_cast(this)); std::string input; @@ -307,7 +199,7 @@ ReplExitStatus NixRepl::mainLoop() logger->pause(); // When continuing input from previous lines, don't print a prompt, just align to the same // number of chars as the prompt. - if (!getLine(input, input.empty() ? "nix-repl> " : " ")) { + if (!interacter->getLine(input, input.empty() ? ReplPromptType::ReplPrompt : ReplPromptType::ContinuationPrompt)) { // Ctrl-D should exit the debugger. state->debugStop = false; logger->cout(""); @@ -325,7 +217,7 @@ ReplExitStatus NixRepl::mainLoop() case ProcessLineResult::PromptAgain: break; default: - abort(); + unreachable(); } } catch (ParseError & e) { if (e.msg().find("unexpected end of file") != std::string::npos) { @@ -336,13 +228,7 @@ ReplExitStatus NixRepl::mainLoop() printMsg(lvlError, e.msg()); } } catch (EvalError & e) { - // in debugger mode, an EvalError should trigger another repl session. - // when that session returns the exception will land here. No need to show it again; - // show the error for this repl session instead. - if (state->debugRepl && !state->debugTraces.empty()) - showDebugTrace(std::cout, state->positions, state->debugTraces.front()); - else - printMsg(lvlError, e.msg()); + printMsg(lvlError, e.msg()); } catch (Error & e) { printMsg(lvlError, e.msg()); } catch (Interrupted & e) { @@ -356,51 +242,6 @@ ReplExitStatus NixRepl::mainLoop() } } - -bool NixRepl::getLine(std::string & input, const std::string & prompt) -{ - struct sigaction act, old; - sigset_t savedSignalMask, set; - - auto setupSignals = [&]() { - act.sa_handler = sigintHandler; - sigfillset(&act.sa_mask); - act.sa_flags = 0; - if (sigaction(SIGINT, &act, &old)) - throw SysError("installing handler for SIGINT"); - - sigemptyset(&set); - sigaddset(&set, SIGINT); - if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask)) - throw SysError("unblocking SIGINT"); - }; - auto restoreSignals = [&]() { - if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) - throw SysError("restoring signals"); - - if (sigaction(SIGINT, &old, 0)) - throw SysError("restoring handler for SIGINT"); - }; - - setupSignals(); - char * s = readline(prompt.c_str()); - Finally doFree([&]() { free(s); }); - restoreSignals(); - - if (g_signal_received) { - g_signal_received = 0; - input.clear(); - return true; - } - - if (!s) - return false; - input += s; - input += '\n'; - return true; -} - - StringSet NixRepl::completePrefix(const std::string & prefix) { StringSet completions; @@ -421,11 +262,14 @@ StringSet NixRepl::completePrefix(const std::string & prefix) try { auto dir = std::string(cur, 0, slash); auto prefix2 = std::string(cur, slash + 1); - for (auto & entry : readDirectory(dir == "" ? "/" : dir)) { - if (entry.name[0] != '.' && hasPrefix(entry.name, prefix2)) - completions.insert(prev + dir + "/" + entry.name); + for (auto & entry : std::filesystem::directory_iterator{dir == "" ? "/" : dir}) { + checkInterrupt(); + auto name = entry.path().filename().string(); + if (name[0] != '.' && hasPrefix(name, prefix2)) + completions.insert(prev + entry.path().string()); } } catch (Error &) { + } catch (std::filesystem::filesystem_error &) { } } else if ((dot = cur.rfind('.')) == std::string::npos) { /* This is a variable name; look it up in the current scope. */ @@ -452,7 +296,7 @@ StringSet NixRepl::completePrefix(const std::string & prefix) e->eval(*state, *env, v); state->forceAttrs(v, noPos, "while evaluating an attrset for the purpose of completion (this error should not be displayed; file an issue?)"); - for (auto & i : *v.attrs) { + for (auto & i : *v.attrs()) { std::string_view name = state->symbols[i.name]; if (name.substr(0, cur2.size()) != cur2) continue; completions.insert(concatStrings(prev, expr, ".", name)); @@ -464,6 +308,8 @@ StringSet NixRepl::completePrefix(const std::string & prefix) // Quietly ignore evaluation errors. } catch (BadURL & e) { // Quietly ignore BadURL flake-related errors. + } catch (FileNotFound & e) { + // Quietly ignore non-existent file beeing `import`-ed. } } @@ -519,7 +365,7 @@ ProcessLineResult NixRepl::processLine(std::string line) if (line.empty()) return ProcessLineResult::PromptAgain; - _isInterrupted = false; + setInterrupted(false); std::string command, arg; @@ -548,6 +394,7 @@ ProcessLineResult NixRepl::processLine(std::string line) << " :l, :load Load Nix expression and add it to scope\n" << " :lf, :load-flake Load Nix flake and add it to scope\n" << " :p, :print Evaluate and print expression recursively\n" + << " Strings are printed directly, without escaping.\n" << " :q, :quit Exit nix-repl\n" << " :r, :reload Reload all files\n" << " :sh Build dependencies of derivation, then start\n" @@ -651,7 +498,7 @@ ProcessLineResult NixRepl::processLine(std::string line) auto path = state->coerceToPath(noPos, v, context, "while evaluating the filename to edit"); return {path, 0}; } else if (v.isLambda()) { - auto pos = state->positions[v.lambda.fun->pos]; + auto pos = state->positions[v.payload.lambda.fun->pos]; if (auto path = std::get_if(&pos.origin)) return {*path, pos.line}; else @@ -669,7 +516,7 @@ ProcessLineResult NixRepl::processLine(std::string line) // runProgram redirects stdout to a StringSink, // using runProgram2 to allow editors to display their UI - runProgram2(RunOptions { .program = editor, .searchPath = true, .args = args }); + runProgram2(RunOptions { .program = editor, .lookupPath = true, .args = args , .isInteractive = true }); // Reload right after exiting the editor state->resetFileCache(); @@ -755,7 +602,11 @@ ProcessLineResult NixRepl::processLine(std::string line) else if (command == ":p" || command == ":print") { Value v; evalString(arg, v); - printValue(std::cout, v); + if (v.type() == nString) { + std::cout << v.string_view(); + } else { + printValue(std::cout, v); + } std::cout << std::endl; } @@ -766,6 +617,35 @@ ProcessLineResult NixRepl::processLine(std::string line) else if (command == ":doc") { Value v; + + auto expr = parseString(arg); + std::string fallbackName; + PosIdx fallbackPos; + DocComment fallbackDoc; + if (auto select = dynamic_cast(expr)) { + Value vAttrs; + auto name = select->evalExceptFinalSelect(*state, *env, vAttrs); + fallbackName = state->symbols[name]; + + state->forceAttrs(vAttrs, noPos, "while evaluating an attribute set to look for documentation"); + auto attrs = vAttrs.attrs(); + assert(attrs); + auto attr = attrs->get(name); + if (!attr) { + // When missing, trigger the normal exception + // e.g. :doc builtins.foo + // behaves like + // nix-repl> builtins.foo + // error: attribute 'foo' missing + evalString(arg, v); + assert(false); + } + if (attr->pos) { + fallbackPos = attr->pos; + fallbackDoc = state->getDocCommentForPos(fallbackPos); + } + } + evalString(arg, v); if (auto doc = state->getDoc(v)) { std::string markdown; @@ -783,6 +663,19 @@ ProcessLineResult NixRepl::processLine(std::string line) markdown += stripIndentation(doc->doc); logger->cout(trim(renderMarkdownToTerminal(markdown))); + } else if (fallbackPos) { + std::stringstream ss; + ss << "Attribute `" << fallbackName << "`\n\n"; + ss << " … defined at " << state->positions[fallbackPos] << "\n\n"; + if (fallbackDoc) { + ss << fallbackDoc.getInnerText(state->positions); + } else { + ss << "No documentation found.\n\n"; + } + + auto markdown = ss.str(); + logger->cout(trim(renderMarkdownToTerminal(markdown))); + } else throw Error("value does not have documentation"); } @@ -840,14 +733,14 @@ void NixRepl::loadFlake(const std::string & flakeRefS) if (flakeRefS.empty()) throw Error("cannot use ':load-flake' without a path specified. (Use '.' for the current working directory.)"); - auto flakeRef = parseFlakeRef(flakeRefS, absPath("."), true); + auto flakeRef = parseFlakeRef(fetchSettings, flakeRefS, absPath("."), true); if (evalSettings.pureEval && !flakeRef.input.isLocked()) throw Error("cannot use ':load-flake' on locked flake reference '%s' (use --impure to override)", flakeRefS); Value v; flake::callFlake(*state, - flake::lockFlake(*state, flakeRef, + flake::lockFlake(flakeSettings, *state, flakeRef, flake::LockFlags { .updateLockFile = false, .useRegistries = !evalSettings.pureEval, @@ -899,17 +792,17 @@ void NixRepl::loadFiles() void NixRepl::addAttrsToScope(Value & attrs) { state->forceAttrs(attrs, [&]() { return attrs.determinePos(noPos); }, "while evaluating an attribute set to be merged in the global scope"); - if (displ + attrs.attrs->size() >= envSize) + if (displ + attrs.attrs()->size() >= envSize) throw Error("environment full; cannot add more variables"); - for (auto & i : *attrs.attrs) { + for (auto & i : *attrs.attrs()) { staticEnv->vars.emplace_back(i.name, displ); env->values[displ++] = i.value; varNames.emplace(state->symbols[i.name]); } staticEnv->sort(); staticEnv->deduplicate(); - notice("Added %1% variables.", attrs.attrs->size()); + notice("Added %1% variables.", attrs.attrs()->size()); } @@ -941,11 +834,11 @@ void NixRepl::evalString(std::string s, Value & v) std::unique_ptr AbstractNixRepl::create( - const SearchPath & searchPath, nix::ref store, ref state, + const LookupPath & lookupPath, nix::ref store, ref state, std::function getValues) { return std::make_unique( - searchPath, + lookupPath, openStore(), state, getValues @@ -961,9 +854,9 @@ ReplExitStatus AbstractNixRepl::runSimple( NixRepl::AnnotatedValues values; return values; }; - SearchPath searchPath = {}; + LookupPath lookupPath = {}; auto repl = std::make_unique( - searchPath, + lookupPath, openStore(), evalState, getValues diff --git a/src/libcmd/repl.hh b/src/libcmd/repl.hh index 21aa8bfc7..3fd4b2c39 100644 --- a/src/libcmd/repl.hh +++ b/src/libcmd/repl.hh @@ -3,11 +3,6 @@ #include "eval.hh" -#if HAVE_BOEHMGC -#define GC_INCLUDE_NEW -#include -#endif - namespace nix { struct AbstractNixRepl @@ -25,7 +20,7 @@ struct AbstractNixRepl typedef std::vector> AnnotatedValues; static std::unique_ptr create( - const SearchPath & searchPath, nix::ref store, ref state, + const LookupPath & lookupPath, nix::ref store, ref state, std::function getValues); static ReplExitStatus runSimple( diff --git a/src/libexpr-c/.version b/src/libexpr-c/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libexpr-c/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libexpr-c/build-utils-meson b/src/libexpr-c/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libexpr-c/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libexpr-c/local.mk b/src/libexpr-c/local.mk new file mode 100644 index 000000000..227a4095b --- /dev/null +++ b/src/libexpr-c/local.mk @@ -0,0 +1,25 @@ +libraries += libexprc + +libexprc_NAME = libnixexprc + +libexprc_DIR := $(d) + +libexprc_SOURCES := \ + $(wildcard $(d)/*.cc) \ + +# Not just for this library itself, but also for downstream libraries using this library + +INCLUDE_libexprc := -I $(d) +libexprc_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libutilc) \ + $(INCLUDE_libfetchers) \ + $(INCLUDE_libstore) $(INCLUDE_libstorec) \ + $(INCLUDE_libexpr) $(INCLUDE_libexprc) + +libexprc_LIBS = libutil libutilc libstore libstorec libfetchers libexpr + +libexprc_LDFLAGS += $(THREAD_LDFLAGS) + +$(eval $(call install-file-in, $(d)/nix-expr-c.pc, $(libdir)/pkgconfig, 0644)) + +libexprc_FORCE_INSTALL := 1 + diff --git a/src/libexpr-c/meson.build b/src/libexpr-c/meson.build new file mode 100644 index 000000000..6db5b83b8 --- /dev/null +++ b/src/libexpr-c/meson.build @@ -0,0 +1,93 @@ +project('nix-expr-c', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), + dependency('nix-expr'), +] +deps_public_maybe_subproject = [ + dependency('nix-util-c'), + dependency('nix-store-c'), +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +# TODO rename, because it will conflict with downstream projects +configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) + +config_h = configure_file( + configuration : configdata, + output : 'config-expr.h', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + + # From C++ libraries, only for internals + '-include', 'config-util.hh', + '-include', 'config-store.hh', + '-include', 'config-expr.hh', + + # From C libraries, for our public, installed headers too + '-include', 'config-util.h', + '-include', 'config-store.h', + '-include', 'config-expr.h', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'nix_api_expr.cc', + 'nix_api_external.cc', + 'nix_api_value.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'nix_api_expr.h', + 'nix_api_external.h', + 'nix_api_value.h', +) + +# TODO move this header to libexpr, maybe don't use it in tests? +headers += files('nix_api_expr_internal.h') + +subdir('build-utils-meson/export-all-symbols') + +this_library = library( + 'nixexprc', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + link_args: linker_export_flags, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libexpr-c/nix-expr-c.pc.in b/src/libexpr-c/nix-expr-c.pc.in new file mode 100644 index 000000000..06897064d --- /dev/null +++ b/src/libexpr-c/nix-expr-c.pc.in @@ -0,0 +1,10 @@ +prefix=@prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: Nix +Description: Nix Language Evaluator - C API +Version: @PACKAGE_VERSION@ +Requires: nix-store-c +Libs: -L${libdir} -lnixexprc +Cflags: -I${includedir}/nix diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc new file mode 100644 index 000000000..8f21d7022 --- /dev/null +++ b/src/libexpr-c/nix_api_expr.cc @@ -0,0 +1,213 @@ +#include +#include +#include + +#include "eval.hh" +#include "eval-gc.hh" +#include "globals.hh" +#include "eval-settings.hh" + +#include "nix_api_expr.h" +#include "nix_api_expr_internal.h" +#include "nix_api_store.h" +#include "nix_api_store_internal.h" +#include "nix_api_util.h" +#include "nix_api_util_internal.h" + +#if HAVE_BOEHMGC +# include +# define GC_INCLUDE_NEW 1 +# include "gc_cpp.h" +#endif + +nix_err nix_libexpr_init(nix_c_context * context) +{ + if (context) + context->last_err_code = NIX_OK; + { + auto ret = nix_libutil_init(context); + if (ret != NIX_OK) + return ret; + } + { + auto ret = nix_libstore_init(context); + if (ret != NIX_OK) + return ret; + } + try { + nix::initGC(); + } + NIXC_CATCH_ERRS +} + +nix_err nix_expr_eval_from_string( + nix_c_context * context, EvalState * state, const char * expr, const char * path, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + nix::Expr * parsedExpr = state->state.parseExprFromString(expr, state->state.rootPath(nix::CanonPath(path))); + state->state.eval(parsedExpr, value->value); + state->state.forceValue(value->value, nix::noPos); + } + NIXC_CATCH_ERRS +} + +nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, nix_value * arg, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + state->state.callFunction(fn->value, arg->value, value->value, nix::noPos); + state->state.forceValue(value->value, nix::noPos); + } + NIXC_CATCH_ERRS +} + +nix_err nix_value_call_multi(nix_c_context * context, EvalState * state, nix_value * fn, size_t nargs, nix_value ** args, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + state->state.callFunction(fn->value, nargs, (nix::Value * *)args, value->value, nix::noPos); + state->state.forceValue(value->value, nix::noPos); + } + NIXC_CATCH_ERRS +} + +nix_err nix_value_force(nix_c_context * context, EvalState * state, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + state->state.forceValue(value->value, nix::noPos); + } + NIXC_CATCH_ERRS +} + +nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + state->state.forceValueDeep(value->value); + } + NIXC_CATCH_ERRS +} + +EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c, Store * store) +{ + if (context) + context->last_err_code = NIX_OK; + try { + nix::Strings lookupPath; + if (lookupPath_c != nullptr) + for (size_t i = 0; lookupPath_c[i] != nullptr; i++) + lookupPath.push_back(lookupPath_c[i]); + + void * p = ::operator new( + sizeof(EvalState), + static_cast(alignof(EvalState))); + auto * p2 = static_cast(p); + new (p) EvalState { + .fetchSettings = nix::fetchers::Settings{}, + .settings = nix::EvalSettings{ + nix::settings.readOnlyMode, + }, + .state = nix::EvalState( + nix::LookupPath::parse(lookupPath), + store->ptr, + p2->fetchSettings, + p2->settings), + }; + loadConfFile(p2->settings); + return p2; + } + NIXC_CATCH_ERRS_NULL +} + +void nix_state_free(EvalState * state) +{ + delete state; +} + +#if HAVE_BOEHMGC +std::unordered_map< + const void *, + unsigned int, + std::hash, + std::equal_to, + traceable_allocator
; }' + > { a = ; } + > ``` + > + > For human-readable output, `nix eval` (experimental) is more informative: + > + > ```console + > $ nix-instantiate --eval --expr 'a: a' + > + > $ nix eval --expr 'a: a' + > «lambda @ «string»:1:1» + > ``` + > + > For machine-readable output, the `--xml` option produces unambiguous + > output: + > + > ```console + > $ nix-instantiate --eval --xml --expr '{ foo = ; }' + > + > + > + > + > + > + > + > + > ``` - - `--json`\ - When used with `--eval`, print the resulting value as an JSON - representation of the abstract syntax tree rather than as a Nix expression. +- `--find-file` - - `--xml`\ - When used with `--eval`, print the resulting value as an XML - representation of the abstract syntax tree rather than as a Nix expression. - The schema is the same as that used by the [`toXML` - built-in](../language/builtins.md). + Look up the given files in Nix’s search path (as specified by the + `NIX_PATH` environment variable). If found, print the corresponding + absolute paths on standard output. For instance, if `NIX_PATH` is + `nixpkgs=/home/alice/nixpkgs`, then `nix-instantiate --find-file + nixpkgs/default.nix` will print `/home/alice/nixpkgs/default.nix`. - - `--read-write-mode`\ - When used with `--eval`, perform evaluation in read/write mode so - nix language features that require it will still work (at the cost - of needing to do instantiation of every evaluated derivation). If - this option is not enabled, there may be uninstantiated store paths - in the final output. +- `--strict` + + When used with `--eval`, recursively evaluate list elements and + attributes. Normally, such sub-expressions are left unevaluated + (since the Nix language is lazy). + + > **Warning** + > + > This option can cause non-termination, because lazy data + > structures can be infinitely large. + +- `--json` + + When used with `--eval`, print the resulting value as an JSON + representation of the abstract syntax tree rather than as a Nix expression. + +- `--xml` + + When used with `--eval`, print the resulting value as an XML + representation of the abstract syntax tree rather than as a Nix expression. + The schema is the same as that used by the [`toXML` + built-in](../language/builtins.md). + +- `--read-write-mode` + + When used with `--eval`, perform evaluation in read/write mode so + nix language features that require it will still work (at the cost + of needing to do instantiation of every evaluated derivation). If + this option is not enabled, there may be uninstantiated store paths + in the final output. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-prefetch-url.md b/doc/manual/src/command-ref/nix-prefetch-url.md index 45ef01e02..ffab94b8a 100644 --- a/doc/manual/src/command-ref/nix-prefetch-url.md +++ b/doc/manual/src/command-ref/nix-prefetch-url.md @@ -39,27 +39,32 @@ the path of the downloaded file in the Nix store is also printed. # Options - - `--type` *hashAlgo*\ - Use the specified cryptographic hash algorithm, - which can be one of `md5`, `sha1`, `sha256`, and `sha512`. - The default is `sha256`. +- `--type` *hashAlgo* - - `--print-path`\ - Print the store path of the downloaded file on standard output. + Use the specified cryptographic hash algorithm, + which can be one of `md5`, `sha1`, `sha256`, and `sha512`. + The default is `sha256`. - - `--unpack`\ - Unpack the archive (which must be a tarball or zip file) and add the - result to the Nix store. The resulting hash can be used with - functions such as Nixpkgs’s `fetchzip` or `fetchFromGitHub`. +- `--print-path` - - `--executable`\ - Set the executable bit on the downloaded file. + Print the store path of the downloaded file on standard output. - - `--name` *name*\ - Override the name of the file in the Nix store. By default, this is - `hash-basename`, where *basename* is the last component of *url*. - Overriding the name is necessary when *basename* contains characters - that are not allowed in Nix store paths. +- `--unpack` + + Unpack the archive (which must be a tarball or zip file) and add the + result to the Nix store. The resulting hash can be used with + functions such as Nixpkgs’s `fetchzip` or `fetchFromGitHub`. + +- `--executable` + + Set the executable bit on the downloaded file. + +- `--name` *name* + + Override the name of the file in the Nix store. By default, this is + `hash-basename`, where *basename* is the last component of *url*. + Overriding the name is necessary when *basename* contains characters + that are not allowed in Nix store paths. # Examples diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index 1eaf3c36a..69a711bd5 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -60,55 +60,63 @@ All options not listed here are passed to `nix-store --realise`, except for `--arg` and `--attr` / `-A` which are passed to `nix-instantiate`. - - `--command` *cmd*\ - In the environment of the derivation, run the shell command *cmd*. - This command is executed in an interactive shell. (Use `--run` to - use a non-interactive shell instead.) However, a call to `exit` is - implicitly added to the command, so the shell will exit after - running the command. To prevent this, add `return` at the end; - e.g. `--command "echo Hello; return"` will print `Hello` and then - drop you into the interactive shell. This can be useful for doing - any additional initialisation. +- `--command` *cmd* - - `--run` *cmd*\ - Like `--command`, but executes the command in a non-interactive - shell. This means (among other things) that if you hit Ctrl-C while - the command is running, the shell exits. + In the environment of the derivation, run the shell command *cmd*. + This command is executed in an interactive shell. (Use `--run` to + use a non-interactive shell instead.) However, a call to `exit` is + implicitly added to the command, so the shell will exit after + running the command. To prevent this, add `return` at the end; + e.g. `--command "echo Hello; return"` will print `Hello` and then + drop you into the interactive shell. This can be useful for doing + any additional initialisation. - - `--exclude` *regexp*\ - Do not build any dependencies whose store path matches the regular - expression *regexp*. This option may be specified multiple times. +- `--run` *cmd* - - `--pure`\ - If this flag is specified, the environment is almost entirely - cleared before the interactive shell is started, so you get an - environment that more closely corresponds to the “real†Nix build. A - few variables, in particular `HOME`, `USER` and `DISPLAY`, are - retained. + Like `--command`, but executes the command in a non-interactive + shell. This means (among other things) that if you hit Ctrl-C while + the command is running, the shell exits. - - `--packages` / `-p` *packages*…\ - Set up an environment in which the specified packages are present. - The command line arguments are interpreted as attribute names inside - the Nix Packages collection. Thus, `nix-shell --packages libjpeg openjdk` - will start a shell in which the packages denoted by the attribute - names `libjpeg` and `openjdk` are present. +- `--exclude` *regexp* - - `-i` *interpreter*\ - The chained script interpreter to be invoked by `nix-shell`. Only - applicable in `#!`-scripts (described below). + Do not build any dependencies whose store path matches the regular + expression *regexp*. This option may be specified multiple times. - - `--keep` *name*\ - When a `--pure` shell is started, keep the listed environment - variables. +- `--pure` + + If this flag is specified, the environment is almost entirely + cleared before the interactive shell is started, so you get an + environment that more closely corresponds to the “real†Nix build. A + few variables, in particular `HOME`, `USER` and `DISPLAY`, are + retained. + +- `--packages` / `-p` *packages*… + + Set up an environment in which the specified packages are present. + The command line arguments are interpreted as attribute names inside + the Nix Packages collection. Thus, `nix-shell --packages libjpeg openjdk` + will start a shell in which the packages denoted by the attribute + names `libjpeg` and `openjdk` are present. + +- `-i` *interpreter* + + The chained script interpreter to be invoked by `nix-shell`. Only + applicable in `#!`-scripts (described below). + +- `--keep` *name* + + When a `--pure` shell is started, keep the listed environment + variables. {{#include ./opt-common.md}} # Environment variables - - `NIX_BUILD_SHELL`\ - Shell used to start the interactive environment. Defaults to the - `bash` found in ``, falling back to the `bash` found in - `PATH` if not found. +- `NIX_BUILD_SHELL` + + Shell used to start the interactive environment. Defaults to the + `bash` found in ``, falling back to the `bash` found in + `PATH` if not found. {{#include ./env-common.md}} @@ -202,14 +210,14 @@ For example, here is a Python script that depends on Python and the ```python #! /usr/bin/env nix-shell -#! nix-shell -i python --packages python pythonPackages.prettytable +#! nix-shell -i python3 --packages python3 python3Packages.prettytable import prettytable # Print a simple table. t = prettytable.PrettyTable(["N", "N^2"]) for n in range(1, 10): t.add_row([n, n * n]) -print t +print(t) ``` Similarly, the following is a Perl script that specifies that it @@ -289,3 +297,8 @@ with import {}; runCommand "dummy" { buildInputs = [ python pythonPackages.prettytable ]; } "" ``` + +The script's file name is passed as the first argument to the interpreter specified by the `-i` flag. + +Aside from the very first line, which is a directive to the operating system, the additional `#! nix-shell` lines do not need to be at the beginning of the file. +This allows wrapping them in block comments for languages where `#` does not start a comment, such as ECMAScript, Erlang, PHP, or Ruby. diff --git a/doc/manual/src/command-ref/nix-store/add-fixed.md b/doc/manual/src/command-ref/nix-store/add-fixed.md index d25db091c..bebf15026 100644 --- a/doc/manual/src/command-ref/nix-store/add-fixed.md +++ b/doc/manual/src/command-ref/nix-store/add-fixed.md @@ -16,9 +16,10 @@ public url or broke since the download expression was written. This operation has the following options: - - `--recursive`\ - Use recursive instead of flat hashing mode, used when adding - directories to the store. +- `--recursive` + + Use recursive instead of flat hashing mode, used when adding + directories to the store. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/dump.md b/doc/manual/src/command-ref/nix-store/dump.md index c2f3c42ef..3de0e27b0 100644 --- a/doc/manual/src/command-ref/nix-store/dump.md +++ b/doc/manual/src/command-ref/nix-store/dump.md @@ -1,6 +1,6 @@ # Name -`nix-store --dump` - write a single path to a Nix Archive +`nix-store --dump` - write a single path to a [Nix Archive] ## Synopsis @@ -8,7 +8,7 @@ ## Description -The operation `--dump` produces a NAR (Nix ARchive) file containing the +The operation `--dump` produces a [Nix archive](@docroot@/glossary.md#gloss-nar) (NAR) file containing the contents of the file system tree rooted at *path*. The archive is written to standard output. @@ -30,8 +30,9 @@ NAR archives support filenames of unlimited length and 64-bit file sizes. They can contain regular files, directories, and symbolic links, but not other types of files (such as device nodes). -A Nix archive can be unpacked using `nix-store ---restore`. +A Nix archive can be unpacked using [`nix-store --restore`](@docroot@/command-ref/nix-store/restore.md). + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/export.md b/doc/manual/src/command-ref/nix-store/export.md index 1bc46f53b..ba772eb43 100644 --- a/doc/manual/src/command-ref/nix-store/export.md +++ b/doc/manual/src/command-ref/nix-store/export.md @@ -1,6 +1,6 @@ # Name -`nix-store --export` - export store paths to a Nix Archive +`nix-store --export` - export store paths to a [Nix Archive] ## Synopsis @@ -8,16 +8,22 @@ ## Description -The operation `--export` writes a serialisation of the specified store -paths to standard output in a format that can be imported into another -Nix store with `nix-store --import`. This is like `nix-store ---dump`, except that the NAR archive produced by that command doesn’t -contain the necessary meta-information to allow it to be imported into -another Nix store (namely, the set of references of the path). +The operation `--export` writes a serialisation of the given [store objects](@docroot@/glossary.md#gloss-store-object) to standard output in a format that can be imported into another [Nix store](@docroot@/store/index.md) with [`nix-store --import`](./import.md). -This command does not produce a *closure* of the specified paths, so if -a store path references other store paths that are missing in the target -Nix store, the import will fail. +> **Warning** +> +> This command *does not* produce a [closure](@docroot@/glossary.md#gloss-closure) of the specified store paths. +> Trying to import a store object that refers to store paths not available in the target Nix store will fail. +> +> Use [`nix-store --query`](@docroot@/command-ref/nix-store/query.md) to obtain the closure of a store path. + +This command is different from [`nix-store --dump`](./dump.md), which produces a [Nix archive](@docroot@/glossary.md#gloss-nar) that *does not* contain the set of [references](@docroot@/glossary.md#gloss-reference) of a given store path. + +> **Note** +> +> For efficient transfer of closures to remote machines over SSH, use [`nix-copy-closure`](@docroot@/command-ref/nix-copy-closure.md). + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive {{#include ./opt-common.md}} @@ -27,15 +33,21 @@ Nix store, the import will fail. # Examples -To copy a whole closure, do something -like: - -```console -$ nix-store --export $(nix-store --query --requisites paths) > out -``` - -To import the whole closure again, run: - -```console -$ nix-store --import < out -``` +> **Example** +> +> Deploy GNU Hello to an airgapped machine via USB stick. +> +> Write the closure to the block device on a machine with internet connection: +> +> ```shell-session +> [alice@itchy]$ storePath=$(nix-build '' -I nixpkgs=channel:nixpkgs-unstable -A hello --no-out-link) +> [alice@itchy]$ nix-store --export $(nix-store --query --requisites $storePath) | sudo dd of=/dev/usb +> ``` +> +> Read the closure from the block device on the machine without internet connection: +> +> ```shell-session +> [bob@scratchy]$ hello=$(sudo dd if=/dev/usb | nix-store --import | tail -1) +> [bob@scratchy]$ $hello/bin/hello +> Hello, world! +> ``` diff --git a/doc/manual/src/command-ref/nix-store/gc.md b/doc/manual/src/command-ref/nix-store/gc.md index 7be0d559a..f432e00eb 100644 --- a/doc/manual/src/command-ref/nix-store/gc.md +++ b/doc/manual/src/command-ref/nix-store/gc.md @@ -14,30 +14,34 @@ reachable via file system references from a set of “rootsâ€, are deleted. The following suboperations may be specified: - - `--print-roots`\ - This operation prints on standard output the set of roots used by - the garbage collector. +- `--print-roots` - - `--print-live`\ - This operation prints on standard output the set of “live†store - paths, which are all the store paths reachable from the roots. Live - paths should never be deleted, since that would break consistency — - it would become possible that applications are installed that - reference things that are no longer present in the store. + This operation prints on standard output the set of roots used by + the garbage collector. - - `--print-dead`\ - This operation prints out on standard output the set of “dead†store - paths, which is just the opposite of the set of live paths: any path - in the store that is not live (with respect to the roots) is dead. +- `--print-live` + + This operation prints on standard output the set of “live†store + paths, which are all the store paths reachable from the roots. Live + paths should never be deleted, since that would break consistency — + it would become possible that applications are installed that + reference things that are no longer present in the store. + +- `--print-dead` + + This operation prints out on standard output the set of “dead†store + paths, which is just the opposite of the set of live paths: any path + in the store that is not live (with respect to the roots) is dead. By default, all unreachable paths are deleted. The following options control what gets deleted and in what order: - - `--max-freed` *bytes*\ - Keep deleting paths until at least *bytes* bytes have been deleted, - then stop. The argument *bytes* can be followed by the - multiplicative suffix `K`, `M`, `G` or `T`, denoting KiB, MiB, GiB - or TiB units. +- `--max-freed` *bytes* + + Keep deleting paths until at least *bytes* bytes have been deleted, + then stop. The argument *bytes* can be followed by the + multiplicative suffix `K`, `M`, `G` or `T`, denoting KiB, MiB, GiB + or TiB units. The behaviour of the collector is also influenced by the `keep-outputs` and `keep-derivations` settings in the Nix diff --git a/doc/manual/src/command-ref/nix-store/import.md b/doc/manual/src/command-ref/nix-store/import.md index 2711316a7..3f6b3d076 100644 --- a/doc/manual/src/command-ref/nix-store/import.md +++ b/doc/manual/src/command-ref/nix-store/import.md @@ -1,6 +1,8 @@ # Name -`nix-store --import` - import Nix Archive into the store +`nix-store --import` - import [Nix Archive] into the store + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive # Synopsis @@ -8,14 +10,34 @@ # Description -The operation `--import` reads a serialisation of a set of store paths -produced by `nix-store --export` from standard input and adds those -store paths to the Nix store. Paths that already exist in the Nix store -are ignored. If a path refers to another path that doesn’t exist in the -Nix store, the import fails. +The operation `--import` reads a serialisation of a set of [store objects](@docroot@/glossary.md#gloss-store-object) produced by [`nix-store --export`](./export.md) from standard input, and adds those store objects to the specified [Nix store](@docroot@/store/index.md). +Paths that already exist in the target Nix store are ignored. +If a path [refers](@docroot@/glossary.md#gloss-reference) to another path that doesn’t exist in the target Nix store, the import fails. + +> **Note** +> +> For efficient transfer of closures to remote machines over SSH, use [`nix-copy-closure`](@docroot@/command-ref/nix-copy-closure.md). {{#include ./opt-common.md}} {{#include ../opt-common.md}} {{#include ../env-common.md}} + +# Examples + +> **Example** +> +> Given a closure of GNU Hello as a file: +> +> ```shell-session +> $ storePath="$(nix-build '' -I nixpkgs=channel:nixpkgs-unstable -A hello --no-out-link)" +> $ nix-store --export $(nix-store --query --requisites $storePath) > hello.closure +> ``` +> +> Import the closure into a [remote SSH store](@docroot@/store/types/ssh-store.md) using the [`--store`](@docroot@/command-ref/conf-file.md#conf-store) option: +> +> ```console +> $ nix-store --import --store ssh://alice@itchy.example.org < hello.closure +> ``` + diff --git a/doc/manual/src/command-ref/nix-store/optimise.md b/doc/manual/src/command-ref/nix-store/optimise.md index dc392aeb8..b257466b2 100644 --- a/doc/manual/src/command-ref/nix-store/optimise.md +++ b/doc/manual/src/command-ref/nix-store/optimise.md @@ -12,7 +12,7 @@ The operation `--optimise` reduces Nix store disk space usage by finding identical files in the store and hard-linking them to each other. It typically reduces the size of the store by something like 25-35%. Only regular files and symlinks are hard-linked in this manner. Files are -considered identical when they have the same NAR archive serialisation: +considered identical when they have the same [Nix Archive (NAR)][Nix Archive] serialisation: that is, regular files must have the same contents and permission (executable or non-executable), and symlinks must have the same contents. @@ -38,3 +38,4 @@ hashing files in `/nix/store/qhqx7l2f1kmwihc9bnxs7rc159hsxnf3-gcc-4.1.1' there are 114486 files with equal contents out of 215894 files in total ``` +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive diff --git a/doc/manual/src/command-ref/nix-store/query.md b/doc/manual/src/command-ref/nix-store/query.md index a158c76aa..b4efa734e 100644 --- a/doc/manual/src/command-ref/nix-store/query.md +++ b/doc/manual/src/command-ref/nix-store/query.md @@ -24,122 +24,138 @@ symlink. # Common query options - - `--use-output`; `-u`\ - For each argument to the query that is a [store derivation], apply the - query to the output path of the derivation instead. +- `--use-output` / `-u` - - `--force-realise`; `-f`\ - Realise each argument to the query first (see [`nix-store --realise`](./realise.md)). + For each argument to the query that is a [store derivation], apply the + query to the output path of the derivation instead. + +- `--force-realise` / `-f` + + Realise each argument to the query first (see [`nix-store --realise`](./realise.md)). [store derivation]: @docroot@/glossary.md#gloss-store-derivation # Queries - - `--outputs`\ - Prints out the [output paths] of the store - derivations *paths*. These are the paths that will be produced when - the derivation is built. +- `--outputs` - [output paths]: ../../glossary.md#gloss-output-path + Prints out the [output paths] of the store + derivations *paths*. These are the paths that will be produced when + the derivation is built. - - `--requisites`; `-R`\ - Prints out the [closure] of the store path *paths*. + [output paths]: @docroot@/glossary.md#gloss-output-path - [closure]: ../../glossary.md#gloss-closure +- `--requisites` / `-R` - This query has one option: + Prints out the [closure] of the store path *paths*. - - `--include-outputs` - Also include the existing output paths of [store derivation]s, - and their closures. + [closure]: @docroot@/glossary.md#gloss-closure - This query can be used to implement various kinds of deployment. A - *source deployment* is obtained by distributing the closure of a - store derivation. A *binary deployment* is obtained by distributing - the closure of an output path. A *cache deployment* (combined - source/binary deployment, including binaries of build-time-only - dependencies) is obtained by distributing the closure of a store - derivation and specifying the option `--include-outputs`. + This query has one option: - - `--references`\ - Prints the set of [references] of the store paths - *paths*, that is, their immediate dependencies. (For *all* - dependencies, use `--requisites`.) + - `--include-outputs` + Also include the existing output paths of [store derivation]s, + and their closures. - [references]: ../../glossary.md#gloss-reference + This query can be used to implement various kinds of deployment. A + *source deployment* is obtained by distributing the closure of a + store derivation. A *binary deployment* is obtained by distributing + the closure of an output path. A *cache deployment* (combined + source/binary deployment, including binaries of build-time-only + dependencies) is obtained by distributing the closure of a store + derivation and specifying the option `--include-outputs`. - - `--referrers`\ - Prints the set of *referrers* of the store paths *paths*, that is, - the store paths currently existing in the Nix store that refer to - one of *paths*. Note that contrary to the references, the set of - referrers is not constant; it can change as store paths are added or - removed. +- `--references` - - `--referrers-closure`\ - Prints the closure of the set of store paths *paths* under the - referrers relation; that is, all store paths that directly or - indirectly refer to one of *paths*. These are all the path currently - in the Nix store that are dependent on *paths*. + Prints the set of [references] of the store paths + *paths*, that is, their immediate dependencies. (For *all* + dependencies, use `--requisites`.) - - `--deriver`; `-d`\ - Prints the [deriver] that was used to build the store paths *paths*. If - the path has no deriver (e.g., if it is a source file), or if the - deriver is not known (e.g., in the case of a binary-only - deployment), the string `unknown-deriver` is printed. - The returned deriver is not guaranteed to exist in the local store, for - example when *paths* were substituted from a binary cache. - Use `--valid-derivers` instead to obtain valid paths only. + [references]: @docroot@/glossary.md#gloss-reference - [deriver]: ../../glossary.md#gloss-deriver +- `--referrers` - - `--valid-derivers`\ - Prints a set of derivation files (`.drv`) which are supposed produce - said paths when realized. Might print nothing, for example for source paths - or paths subsituted from a binary cache. + Prints the set of *referrers* of the store paths *paths*, that is, + the store paths currently existing in the Nix store that refer to + one of *paths*. Note that contrary to the references, the set of + referrers is not constant; it can change as store paths are added or + removed. - - `--graph`\ - Prints the references graph of the store paths *paths* in the format - of the `dot` tool of AT\&T's [Graphviz - package](http://www.graphviz.org/). This can be used to visualise - dependency graphs. To obtain a build-time dependency graph, apply - this to a store derivation. To obtain a runtime dependency graph, - apply it to an output path. +- `--referrers-closure` - - `--tree`\ - Prints the references graph of the store paths *paths* as a nested - ASCII tree. References are ordered by descending closure size; this - tends to flatten the tree, making it more readable. The query only - recurses into a store path when it is first encountered; this - prevents a blowup of the tree representation of the graph. + Prints the closure of the set of store paths *paths* under the + referrers relation; that is, all store paths that directly or + indirectly refer to one of *paths*. These are all the path currently + in the Nix store that are dependent on *paths*. - - `--graphml`\ - Prints the references graph of the store paths *paths* in the - [GraphML](http://graphml.graphdrawing.org/) file format. This can be - used to visualise dependency graphs. To obtain a build-time - dependency graph, apply this to a [store derivation]. To obtain a - runtime dependency graph, apply it to an output path. +- `--deriver` / `-d` - - `--binding` *name*; `-b` *name*\ - Prints the value of the attribute *name* (i.e., environment - variable) of the [store derivation]s *paths*. It is an error for a - derivation to not have the specified attribute. + Prints the [deriver] that was used to build the store paths *paths*. If + the path has no deriver (e.g., if it is a source file), or if the + deriver is not known (e.g., in the case of a binary-only + deployment), the string `unknown-deriver` is printed. + The returned deriver is not guaranteed to exist in the local store, for + example when *paths* were substituted from a binary cache. + Use `--valid-derivers` instead to obtain valid paths only. - - `--hash`\ - Prints the SHA-256 hash of the contents of the store paths *paths* - (that is, the hash of the output of `nix-store --dump` on the given - paths). Since the hash is stored in the Nix database, this is a fast - operation. + [deriver]: @docroot@/glossary.md#gloss-deriver - - `--size`\ - Prints the size in bytes of the contents of the store paths *paths* - — to be precise, the size of the output of `nix-store --dump` on - the given paths. Note that the actual disk space required by the - store paths may be higher, especially on filesystems with large - cluster sizes. +- `--valid-derivers` - - `--roots`\ - Prints the garbage collector roots that point, directly or - indirectly, at the store paths *paths*. + Prints a set of derivation files (`.drv`) which are supposed produce + said paths when realized. Might print nothing, for example for source paths + or paths subsituted from a binary cache. + +- `--graph` + + Prints the references graph of the store paths *paths* in the format + of the `dot` tool of AT\&T's [Graphviz + package](http://www.graphviz.org/). This can be used to visualise + dependency graphs. To obtain a build-time dependency graph, apply + this to a store derivation. To obtain a runtime dependency graph, + apply it to an output path. + +- `--tree` + + Prints the references graph of the store paths *paths* as a nested + ASCII tree. References are ordered by descending closure size; this + tends to flatten the tree, making it more readable. The query only + recurses into a store path when it is first encountered; this + prevents a blowup of the tree representation of the graph. + +- `--graphml` + + Prints the references graph of the store paths *paths* in the + [GraphML](http://graphml.graphdrawing.org/) file format. This can be + used to visualise dependency graphs. To obtain a build-time + dependency graph, apply this to a [store derivation]. To obtain a + runtime dependency graph, apply it to an output path. + +- `--binding` *name* / `-b` *name* + + Prints the value of the attribute *name* (i.e., environment + variable) of the [store derivation]s *paths*. It is an error for a + derivation to not have the specified attribute. + +- `--hash` + + Prints the SHA-256 hash of the contents of the store paths *paths* + (that is, the hash of the output of `nix-store --dump` on the given + paths). Since the hash is stored in the Nix database, this is a fast + operation. + +- `--size` + + Prints the size in bytes of the contents of the store paths *paths* + — to be precise, the size of the output of `nix-store --dump` on + the given paths. Note that the actual disk space required by the + store paths may be higher, especially on filesystems with large + cluster sizes. + +- `--roots` + + Prints the garbage collector roots that point, directly or + indirectly, at the store paths *paths*. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index 5428d57fa..a899758df 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -25,14 +25,14 @@ Each of *paths* is processed as follows: If no substitutes are available and no store derivation is given, realisation fails. -[store paths]: @docroot@/glossary.md#gloss-store-path +[store paths]: @docroot@/store/store-path.md [valid]: @docroot@/glossary.md#gloss-validity [store derivation]: @docroot@/glossary.md#gloss-store-derivation [output paths]: @docroot@/glossary.md#gloss-output-path -[store objects]: @docroot@/glossary.md#gloss-store-object +[store objects]: @docroot@/store/store-object.md [closure]: @docroot@/glossary.md#gloss-closure [substituters]: @docroot@/command-ref/conf-file.md#conf-substituters -[content-addressed derivations]: @docroot@/contributing/experimental-features.md#xp-feature-ca-derivations +[content-addressed derivations]: @docroot@/development/experimental-features.md#xp-feature-ca-derivations [Nix database]: @docroot@/glossary.md#gloss-nix-database The resulting paths are printed on standard output. @@ -42,23 +42,26 @@ For non-derivation arguments, the argument itself is printed. # Options - - `--dry-run`\ - Print on standard error a description of what packages would be - built or downloaded, without actually performing the operation. +- `--dry-run` - - `--ignore-unknown`\ - If a non-derivation path does not have a substitute, then silently - ignore it. + Print on standard error a description of what packages would be + built or downloaded, without actually performing the operation. - - `--check`\ - This option allows you to check whether a derivation is - deterministic. It rebuilds the specified derivation and checks - whether the result is bitwise-identical with the existing outputs, - printing an error if that’s not the case. The outputs of the - specified derivation must already exist. When used with `-K`, if an - output path is not identical to the corresponding output from the - previous build, the new output path is left in - `/nix/store/name.check.` +- `--ignore-unknown` + + If a non-derivation path does not have a substitute, then silently + ignore it. + +- `--check` + + This option allows you to check whether a derivation is + deterministic. It rebuilds the specified derivation and checks + whether the result is bitwise-identical with the existing outputs, + printing an error if that’s not the case. The outputs of the + specified derivation must already exist. When used with `-K`, if an + output path is not identical to the corresponding output from the + previous build, the new output path is left in + `/nix/store/name.check.` {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/restore.md b/doc/manual/src/command-ref/nix-store/restore.md index fcba43df4..2d0aa3127 100644 --- a/doc/manual/src/command-ref/nix-store/restore.md +++ b/doc/manual/src/command-ref/nix-store/restore.md @@ -8,9 +8,11 @@ ## Description -The operation `--restore` unpacks a NAR archive to *path*, which must +The operation `--restore` unpacks a [Nix Archive (NAR)][Nix Archive] to *path*, which must not already exist. The archive is read from standard input. +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive + {{#include ./opt-common.md}} {{#include ../opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/serve.md b/doc/manual/src/command-ref/nix-store/serve.md index 0f90f65ae..9a4cf5216 100644 --- a/doc/manual/src/command-ref/nix-store/serve.md +++ b/doc/manual/src/command-ref/nix-store/serve.md @@ -14,10 +14,11 @@ access to a restricted ssh user. The following flags are available: - - `--write`\ - Allow the connected client to request the realization of - derivations. In effect, this can be used to make the host act as a - remote builder. +- `--write` + + Allow the connected client to request the realization of + derivations. In effect, this can be used to make the host act as a + remote builder. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/verify.md b/doc/manual/src/command-ref/nix-store/verify.md index 2695b3361..40c9180db 100644 --- a/doc/manual/src/command-ref/nix-store/verify.md +++ b/doc/manual/src/command-ref/nix-store/verify.md @@ -16,18 +16,20 @@ being modified by non-Nix tools, or of bugs in Nix itself. This operation has the following options: - - `--check-contents`\ - Checks that the contents of every valid store path has not been - altered by computing a SHA-256 hash of the contents and comparing it - with the hash stored in the Nix database at build time. Paths that - have been modified are printed out. For large stores, - `--check-contents` is obviously quite slow. +- `--check-contents` - - `--repair`\ - If any valid path is missing from the store, or (if - `--check-contents` is given) the contents of a valid path has been - modified, then try to repair the path by redownloading it. See - `nix-store --repair-path` for details. + Checks that the contents of every valid store path has not been + altered by computing a SHA-256 hash of the contents and comparing it + with the hash stored in the Nix database at build time. Paths that + have been modified are printed out. For large stores, + `--check-contents` is obviously quite slow. + +- `--repair` + + If any valid path is missing from the store, or (if + `--check-contents` is given) the contents of a valid path has been + modified, then try to repair the path by redownloading it. See + `nix-store --repair-path` for details. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index 114b292f9..69a700207 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -37,7 +37,7 @@ Most Nix commands accept the following command-line options: Print even more informational messages. - `4` “Debug†- + Print debug information. - `5` “Vomit†@@ -143,7 +143,7 @@ Most Nix commands accept the following command-line options: This option is accepted by `nix-env`, `nix-instantiate`, `nix-shell` and `nix-build`. When evaluating Nix expressions, the expression evaluator will automatically try to call functions that it encounters. - It can automatically call functions for which every argument has a [default value](@docroot@/language/constructs.md#functions) (e.g., `{ argName ? defaultValue }: ...`). + It can automatically call functions for which every argument has a [default value](@docroot@/language/syntax.md#functions) (e.g., `{ argName ? defaultValue }: ...`). With `--arg`, you can also call functions that have arguments without a default value (or override a default value). That is, if the evaluator encounters a function with an argument named *name*, it will call it with value *value*. @@ -187,11 +187,12 @@ Most Nix commands accept the following command-line options: For `nix-shell`, this option is commonly used to give you a shell in which you can build the packages returned by the expression. If you want to get a shell which contain the *built* packages ready for use, give your expression to the `nix-shell --packages ` convenience flag instead. -- [`-I`](#opt-I) *path* +- [`-I` / `--include`](#opt-I) *path* - Add an entry to the [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path). + Add an entry to the list of search paths used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). This option may be given multiple times. - Paths added through `-I` take precedence over [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH). + + Paths added through `-I` take precedence over the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path) and the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH). - [`--option`](#opt-option) *name* *value* diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/development/building.md similarity index 67% rename from doc/manual/src/contributing/hacking.md rename to doc/manual/src/development/building.md index 916ec3077..5a5fb3368 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/development/building.md @@ -1,24 +1,67 @@ -# Hacking +# Building Nix -This section provides some notes on how to hack on Nix. To get the -latest version of Nix from GitHub: +This section provides some notes on how to start hacking on Nix. +To get the latest version of Nix from GitHub: ```console $ git clone https://github.com/NixOS/nix.git $ cd nix ``` -The following instructions assume you already have some version of Nix installed locally, so that you can use it to set up the development environment. If you don't have it installed, follow the [installation instructions]. +> **Note** +> +> The following instructions assume you already have some version of Nix installed locally, so that you can use it to set up the development environment. +> If you don't have it installed, follow the [installation instructions](../installation/index.md). -[installation instructions]: ../installation/index.md + +To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: + +```console +$ nix-shell +``` + +To get a shell with one of the other [supported compilation environments](#compilation-environments): + +```console +$ nix-shell --attr devShells.x86_64-linux.native-clangStdenvPackages +``` + +> **Note** +> +> You can use `native-ccacheStdenvPackages` to drastically improve rebuild time. +> By default, [ccache](https://ccache.dev) keeps artifacts in `~/.cache/ccache/`. + +To build Nix itself in this shell: + +```console +[nix-shell]$ autoreconfPhase +[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out +[nix-shell]$ make -j $NIX_BUILD_CORES +``` + +To install it in `$(pwd)/outputs` and test it: + +```console +[nix-shell]$ make install +[nix-shell]$ make installcheck -j $NIX_BUILD_CORES +[nix-shell]$ ./outputs/out/bin/nix --version +nix (Nix) 2.12 +``` + +To build a release version of Nix for the current operating system and CPU architecture: + +```console +$ nix-build +``` + +You can also build Nix for one of the [supported platforms](#platforms). ## Building Nix with flakes This section assumes you are using Nix with the [`flakes`] and [`nix-command`] experimental features enabled. -See the [Building Nix](#building-nix) section for equivalent instructions using stable Nix interfaces. -[`flakes`]: @docroot@/contributing/experimental-features.md#xp-feature-flakes -[`nix-command`]: @docroot@/contributing/experimental-features.md#xp-nix-command +[`flakes`]: @docroot@/development/experimental-features.md#xp-feature-flakes +[`nix-command`]: @docroot@/development/experimental-features.md#xp-nix-command To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: @@ -67,50 +110,6 @@ $ nix build You can also build Nix for one of the [supported platforms](#platforms). -## Building Nix - -To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: - -```console -$ nix-shell -``` - -To get a shell with one of the other [supported compilation environments](#compilation-environments): - -```console -$ nix-shell --attr devShells.x86_64-linux.native-clangStdenvPackages -``` - -> **Note** -> -> You can use `native-ccacheStdenvPackages` to drastically improve rebuild time. -> By default, [ccache](https://ccache.dev) keeps artifacts in `~/.cache/ccache/`. - -To build Nix itself in this shell: - -```console -[nix-shell]$ autoreconfPhase -[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out -[nix-shell]$ make -j $NIX_BUILD_CORES -``` - -To install it in `$(pwd)/outputs` and test it: - -```console -[nix-shell]$ make install -[nix-shell]$ make installcheck -j $NIX_BUILD_CORES -[nix-shell]$ ./outputs/out/bin/nix --version -nix (Nix) 2.12 -``` - -To build a release version of Nix for the current operating system and CPU architecture: - -```console -$ nix-build -``` - -You can also build Nix for one of the [supported platforms](#platforms). - ## Makefile variables You may need `profiledir=$out/etc/profile.d` and `sysconfdir=$out/etc` to run `make install`. @@ -122,7 +121,6 @@ Run `make` with [`-e` / `--environment-overrides`](https://www.gnu.org/software/ The docs can take a while to build, so you may want to disable this for local development. - `ENABLE_FUNCTIONAL_TESTS=yes` to enable building the functional tests. -- `ENABLE_UNIT_TESTS=yes` to enable building the unit tests. - `OPTIMIZE=1` to enable optimizations. - `libraries=libutil programs=` to only build a specific library. @@ -144,6 +142,7 @@ Nix can be built for various platforms, as specified in [`flake.nix`]: - `aarch64-darwin` - `armv6l-linux` - `armv7l-linux` +- `riscv64-linux` In order to build Nix for a different platform than the one you're currently on, you need a way for your current Nix installation to build code for that @@ -166,7 +165,10 @@ or for Nix with the [`flakes`] and [`nix-command`] experimental features enabled $ nix build .#packages.aarch64-linux.default ``` -Cross-compiled builds are available for ARMv6 (`armv6l-linux`) and ARMv7 (`armv7l-linux`). +Cross-compiled builds are available for: +- `armv6l-linux` +- `armv7l-linux` +- `riscv64-linux` Add more [system types](#system-type) to `crossSystems` in `flake.nix` to bootstrap Nix on unsupported platforms. ### Building for multiple platforms at once @@ -196,7 +198,7 @@ In order to facilitate this, Nix has some support for being built out of tree ## System type -Nix uses a string with he following format to identify the *system type* or *platform* it runs on: +Nix uses a string with the following format to identify the *system type* or *platform* it runs on: ``` -[-] @@ -258,10 +260,10 @@ See [supported compilation environments](#compilation-environments) and instruct To use the LSP with your editor, you first need to [set up `clangd`](https://clangd.llvm.org/installation#project-setup) by running: ```console -make clean && bear -- make -j$NIX_BUILD_CORES default check install +make compile_commands.json ``` -Configure your editor to use the `clangd` from the shell, either by running it inside the development shell, or by using [nix-direnv](https://github.com/nix-community/nix-direnv) and [the appropriate editor plugin](https://github.com/direnv/direnv/wiki#editor-integration). +Configure your editor to use the `clangd` from the `.#native-clangStdenvPackages` shell. You can do that either by running it inside the development shell, or by using [nix-direnv](https://github.com/nix-community/nix-direnv) and [the appropriate editor plugin](https://github.com/direnv/direnv/wiki#editor-integration). > **Note** > @@ -269,80 +271,25 @@ Configure your editor to use the `clangd` from the shell, either by running it i > Some other editors (e.g. Emacs, Vim) need a plugin to support LSP servers in general (e.g. [lsp-mode](https://github.com/emacs-lsp/lsp-mode) for Emacs and [vim-lsp](https://github.com/prabirshrestha/vim-lsp) for vim). > Editor-specific setup is typically opinionated, so we will not cover it here in more detail. -## Add a release note +## Formatting and pre-commit hooks -`doc/manual/rl-next` contains release notes entries for all unreleased changes. +You may run the formatters as a one-off using: -User-visible changes should come with a release note. - -### Add an entry - -Here's what a complete entry looks like. The file name is not incorporated in the document. - -``` ---- -synopsis: Basically a title -issues: 1234 -prs: 1238 ---- - -Here's one or more paragraphs that describe the change. - -- It's markdown -- Add references to the manual using @docroot@ +```console +make format ``` -Significant changes should add the following header, which moves them to the top. +If you'd like to run the formatters before every commit, install the hooks: ``` -significance: significant +pre-commit-hooks-install ``` - -See also the [format documentation](https://github.com/haskell/cabal/blob/master/CONTRIBUTING.md#changelog). +This installs [pre-commit](https://pre-commit.com) using [cachix/git-hooks.nix](https://github.com/cachix/git-hooks.nix). -### Build process +When making a commit, pay attention to the console output. +If it fails, run `git add --patch` to approve the suggestions _and commit again_. -Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`. - -## Branches - -- [`master`](https://github.com/NixOS/nix/commits/master) - - The main development branch. All changes are approved and merged here. - When developing a change, create a branch based on the latest `master`. - - Maintainers try to [keep it in a release-worthy state](#reverting). - -- [`maintenance-*.*`](https://github.com/NixOS/nix/branches/all?query=maintenance) - - These branches are the subject of backports only, and are - also [kept](#reverting) in a release-worthy state. - - See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) - -- [`latest-release`](https://github.com/NixOS/nix/tree/latest-release) - - The latest patch release of the latest minor version. - - See [`maintainers/release-process.md`](https://github.com/NixOS/nix/blob/master/maintainers/release-process.md) - -- [`backport-*-to-*`](https://github.com/NixOS/nix/branches/all?query=backport) - - Generally branches created by the backport action. - - See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) - -- [_other_](https://github.com/NixOS/nix/branches/all) - - Branches that do not conform to the above patterns should be feature branches. - -## Reverting - -If a change turns out to be merged by mistake, or contain a regression, it may be reverted. -A revert is not a rejection of the contribution, but merely part of an effective development process. -It makes sure that development keeps running smoothly, with minimal uncertainty, and less overhead. -If maintainers have to worry too much about avoiding reverts, they would not be able to merge as much. -By embracing reverts as a good part of the development process, everyone wins. - -However, taking a step back may be frustrating, so maintainers will be extra supportive on the next try. +To refresh pre-commit hook's config file, do the following: +1. Exit the development shell and start it again by running `nix develop`. +2. If you also use the pre-commit hook, also run `pre-commit-hooks-install` again. diff --git a/doc/manual/src/contributing/cli-guideline.md b/doc/manual/src/development/cli-guideline.md similarity index 88% rename from doc/manual/src/contributing/cli-guideline.md rename to doc/manual/src/development/cli-guideline.md index e90d6de8d..23df844ec 100644 --- a/doc/manual/src/contributing/cli-guideline.md +++ b/doc/manual/src/development/cli-guideline.md @@ -389,88 +389,6 @@ colors, no emojis and using ASCII instead of Unicode symbols). The same should happen when TTY is not detected on STDERR. We should not display progress / status section, but only print warnings and errors. -## Returning future proof JSON - -The schema of JSON output should allow for backwards compatible extension. This section explains how to achieve this. - -Two definitions are helpful here, because while JSON only defines one "key-value" -object type, we use it to cover two use cases: - - - **dictionary**: a map from names to value that all have the same type. In - C++ this would be a `std::map` with string keys. - - **record**: a fixed set of attributes each with their own type. In C++, this - would be represented by a `struct`. - -It is best not to mix these use cases, as that may lead to incompatibilities when the schema changes. For example, adding a record field to a dictionary breaks consumers that assume all JSON object fields to have the same meaning and type. - -This leads to the following guidelines: - - - The top-level (root) value must be a record. - - Otherwise, one can not change the structure of a command's output. - - - The value of a dictionary item must be a record. - - Otherwise, the item type can not be extended. - - - List items should be records. - - Otherwise, one can not change the structure of the list items. - - If the order of the items does not matter, and each item has a unique key that is a string, consider representing the list as a dictionary instead. If the order of the items needs to be preserved, return a list of records. - - - Streaming JSON should return records. - - An example of a streaming JSON format is [JSON lines](https://jsonlines.org/), where each line represents a JSON value. These JSON values can be considered top-level values or list items, and they must be records. - -### Examples - - -This is bad, because all keys must be assumed to be store types: - -```json -{ - "local": { ... }, - "remote": { ... }, - "http": { ... } -} -``` - -This is good, because the it is extensible at the root, and is somewhat self-documenting: - -```json -{ - "storeTypes": { "local": { ... }, ... }, - "pluginSupport": true -} -``` - -While the dictionary of store types seems like a very complete response at first, a use case may arise that warrants returning additional information. -For example, the presence of plugin support may be crucial information for a client to proceed when their desired store type is missing. - - - -The following representation is bad because it is not extensible: - -```json -{ "outputs": [ "out" "bin" ] } -``` - -However, simply converting everything to records is not enough, because the order of outputs must be preserved: - -```json -{ "outputs": { "bin": {}, "out": {} } } -``` - -The first item is the default output. Deriving this information from the outputs ordering is not great, but this is how Nix currently happens to work. -While it is possible for a JSON parser to preserve the order of fields, we can not rely on this capability to be present in all JSON libraries. - -This representation is extensible and preserves the ordering: - -```json -{ "outputs": [ { "outputName": "out" }, { "outputName": "bin" } ] } -``` - ## Dialog with the user CLIs don't always make it clear when an action has taken place. For every diff --git a/doc/manual/src/development/contributing.md b/doc/manual/src/development/contributing.md new file mode 100644 index 000000000..7de7489dc --- /dev/null +++ b/doc/manual/src/development/contributing.md @@ -0,0 +1,79 @@ +# Contributing + +## Add a release note + +`doc/manual/rl-next` contains release notes entries for all unreleased changes. + +User-visible changes should come with a release note. + +### Add an entry + +Here's what a complete entry looks like. The file name is not incorporated in the document. + +``` +--- +synopsis: Basically a title +issues: 1234 +prs: 1238 +--- + +Here's one or more paragraphs that describe the change. + +- It's markdown +- Add references to the manual using @docroot@ +``` + +Significant changes should add the following header, which moves them to the top. + +``` +significance: significant +``` + + +See also the [format documentation](https://github.com/haskell/cabal/blob/master/CONTRIBUTING.md#changelog). + +### Build process + +Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`. + +## Branches + +- [`master`](https://github.com/NixOS/nix/commits/master) + + The main development branch. All changes are approved and merged here. + When developing a change, create a branch based on the latest `master`. + + Maintainers try to [keep it in a release-worthy state](#reverting). + +- [`maintenance-*.*`](https://github.com/NixOS/nix/branches/all?query=maintenance) + + These branches are the subject of backports only, and are + also [kept](#reverting) in a release-worthy state. + + See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) + +- [`latest-release`](https://github.com/NixOS/nix/tree/latest-release) + + The latest patch release of the latest minor version. + + See [`maintainers/release-process.md`](https://github.com/NixOS/nix/blob/master/maintainers/release-process.md) + +- [`backport-*-to-*`](https://github.com/NixOS/nix/branches/all?query=backport) + + Generally branches created by the backport action. + + See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) + +- [_other_](https://github.com/NixOS/nix/branches/all) + + Branches that do not conform to the above patterns should be feature branches. + +## Reverting + +If a change turns out to be merged by mistake, or contain a regression, it may be reverted. +A revert is not a rejection of the contribution, but merely part of an effective development process. +It makes sure that development keeps running smoothly, with minimal uncertainty, and less overhead. +If maintainers have to worry too much about avoiding reverts, they would not be able to merge as much. +By embracing reverts as a good part of the development process, everyone wins. + +However, taking a step back may be frustrating, so maintainers will be extra supportive on the next try. diff --git a/doc/manual/src/contributing/cxx.md b/doc/manual/src/development/cxx.md similarity index 100% rename from doc/manual/src/contributing/cxx.md rename to doc/manual/src/development/cxx.md diff --git a/doc/manual/src/contributing/documentation.md b/doc/manual/src/development/documentation.md similarity index 89% rename from doc/manual/src/contributing/documentation.md rename to doc/manual/src/development/documentation.md index 1dddb207c..63f574ab7 100644 --- a/doc/manual/src/contributing/documentation.md +++ b/doc/manual/src/development/documentation.md @@ -24,14 +24,12 @@ nix build .#^doc and open `./result-doc/share/doc/nix/manual/index.html`. -To build the manual incrementally, [enter the development shell](./hacking.md) and run: +To build the manual incrementally, [enter the development shell](./building.md) and run: ```console -make manual-html -j $NIX_BUILD_CORES +make manual-html-open -j $NIX_BUILD_CORES ``` -and open `./outputs/out/share/doc/nix/manual/language/index.html`. - In order to reflect changes to the [Makefile for the manual], clear all generated files before re-building: [Makefile for the manual]: https://github.com/NixOS/nix/blob/master/doc/manual/local.mk @@ -149,7 +147,7 @@ Please observe these guidelines to ease reviews: ``` A [store object] contains a [file system object] and [references] to other store objects. - [store object]: @docroot@/glossary.md#gloss-store-object + [store object]: @docroot@/store/store-object.md [file system object]: @docroot@/architecture/file-system-object.md [references]: @docroot@/glossary.md#gloss-reference ``` @@ -198,13 +196,35 @@ You can also build and view it yourself: [Doxygen API documentation]: https://hydra.nixos.org/job/nix/master/internal-api-docs/latest/download-by-type/doc/internal-api-docs ```console -# nix build .#hydraJobs.internal-api-docs -# xdg-open ./result/share/doc/nix/internal-api/html/index.html +$ nix build .#hydraJobs.internal-api-docs +$ xdg-open ./result/share/doc/nix/internal-api/html/index.html +``` + +or inside `nix-shell` or `nix develop`: + +```console +$ mesonConfigurePhase +$ ninja src/internal-api-docs/html +$ xdg-open src/internal-api-docs/html/index.html +``` + +## C API documentation + +Note that the C API is not yet stable. +[C API documentation] is available online. +You can also build and view it yourself: + +[C API documentation]: https://hydra.nixos.org/job/nix/master/external-api-docs/latest/download-by-type/doc/external-api-docs + +```console +$ nix build .#hydraJobs.external-api-docs +$ xdg-open ./result/share/doc/nix/external-api/html/index.html ``` or inside `nix-shell` or `nix develop`: ``` -# make internal-api-html -# xdg-open ./outputs/doc/share/doc/nix/internal-api/html/index.html +$ mesonConfigurePhase +$ ninja src/external-api-docs/html +$ xdg-open src/external-api-docs/html/index.html ``` diff --git a/doc/manual/src/contributing/experimental-features.md b/doc/manual/src/development/experimental-features.md similarity index 100% rename from doc/manual/src/contributing/experimental-features.md rename to doc/manual/src/development/experimental-features.md diff --git a/doc/manual/src/contributing/index.md b/doc/manual/src/development/index.md similarity index 77% rename from doc/manual/src/contributing/index.md rename to doc/manual/src/development/index.md index 4d55c17a4..6403c3e66 100644 --- a/doc/manual/src/contributing/index.md +++ b/doc/manual/src/development/index.md @@ -5,4 +5,4 @@ Check the [contributing guide](https://github.com/NixOS/nix/blob/master/CONTRIBU This chapter is a collection of guides for making changes to the code and documentation. -If you're not sure where to start, try to [compile Nix from source](./hacking.md) and consider [making improvements to documentation](./documentation.md). +If you're not sure where to start, try to [compile Nix from source](./building.md) and consider [making improvements to documentation](./documentation.md). diff --git a/doc/manual/src/development/json-guideline.md b/doc/manual/src/development/json-guideline.md new file mode 100644 index 000000000..b4bc92af9 --- /dev/null +++ b/doc/manual/src/development/json-guideline.md @@ -0,0 +1,128 @@ +# JSON guideline + +Nix consumes and produces JSON in a variety of contexts. +These guidelines ensure consistent practices for all our JSON interfaces, for ease of use, and so that experience in one part carries over to another. + +## Extensibility + +The schema of JSON input and output should allow for backwards compatible extension. +This section explains how to achieve this. + +Two definitions are helpful here, because while JSON only defines one "key-value" object type, we use it to cover two use cases: + + - **dictionary**: a map from names to value that all have the same type. + In C++ this would be a `std::map` with string keys. + + - **record**: a fixed set of attributes each with their own type. + In C++, this would be represented by a `struct`. + +It is best not to mix these use cases, as that may lead to incompatibilities when the schema changes. +For example, adding a record field to a dictionary breaks consumers that assume all JSON object fields to have the same meaning and type, and dictionary items with a colliding name can not be represented anymore. + +This leads to the following guidelines: + + - The top-level (root) value must be a record. + + Otherwise, one can not change the structure of a command's output. + + - The value of a dictionary item must be a record. + + Otherwise, the item type can not be extended. + + - List items should be records. + + Otherwise, one can not change the structure of the list items. + + If the order of the items does not matter, and each item has a unique key that is a string, consider representing the list as a dictionary instead. + If the order of the items needs to be preserved, return a list of records. + + - Streaming JSON should return records. + + An example of a streaming JSON format is [JSON lines](https://jsonlines.org/), where each line represents a JSON value. + These JSON values can be considered top-level values or list items, and they must be records. + +### Examples + +This is bad, because all keys must be assumed to be store types: + +```json +{ + "local": { ... }, + "remote": { ... }, + "http": { ... } +} +``` + +This is good, because the it is extensible at the root, and is somewhat self-documenting: + +```json +{ + "storeTypes": { "local": { ... }, ... }, + "pluginSupport": true +} +``` + +While the dictionary of store types seems like a very complete response at first, a use case may arise that warrants returning additional information. +For example, the presence of plugin support may be crucial information for a client to proceed when their desired store type is missing. + + + +The following representation is bad because it is not extensible: + +```json +{ "outputs": [ "out" "bin" ] } +``` + +However, simply converting everything to records is not enough, because the order of outputs must be preserved: + +```json +{ "outputs": { "bin": {}, "out": {} } } +``` + +The first item is the default output. Deriving this information from the outputs ordering is not great, but this is how Nix currently happens to work. +While it is possible for a JSON parser to preserve the order of fields, we can not rely on this capability to be present in all JSON libraries. + +This representation is extensible and preserves the ordering: + +```json +{ "outputs": [ { "outputName": "out" }, { "outputName": "bin" } ] } +``` + +## Self-describing values + +As described in the previous section, it's crucial that schemas can be extended with with new fields without breaking compatibility. +However, that should *not* mean we use the presence/absence of fields to indicate optional information *within* a version of the schema. +Instead, always include the field, and use `null` to indicate the "nothing" case. + +### Examples + +Here are two JSON objects: + +```json +{ + "foo": {} +} +``` +```json +{ + "foo": {}, + "bar": {} +} +``` + +Since they differ in which fields they contain, they should *not* both be valid values of the same schema. +At most, they can match two different schemas where the second (with `foo` and `bar`) is considered a newer version of the first (with just `foo`). +Within each version, all fields are mandatory (always `foo`, and always `foo` and `bar`). +Only *between* each version, `bar` gets added as a new mandatory field. + +Here are another two JSON objects: + +```json +{ "foo": null } +``` +```json +{ "foo": { "bar": 1 } } +``` + +Since they both contain a `foo` field, they could be valid values of the same schema. +The schema would have `foo` has an optional field, which is either `null` or an object where `bar` is an integer. diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/development/testing.md similarity index 81% rename from doc/manual/src/contributing/testing.md rename to doc/manual/src/development/testing.md index 31c39c16c..3949164d5 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/development/testing.md @@ -59,15 +59,15 @@ The unit tests are defined using the [googletest] and [rapidcheck] frameworks. > … > ``` -The tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `tests/unit/${library_name_without-nix}`. -Given a interface (header) and implementation pair in the original library, say, `src/libexpr/value/context.{hh,cc}`, we write tests for it in `tests/unit/libexpr/tests/value/context.cc`, and (possibly) declare/define additional interfaces for testing purposes in `tests/unit/libexpr-support/tests/value/context.{hh,cc}`. +The tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `src/${library_name_without-nix}-test`. +Given an interface (header) and implementation pair in the original library, say, `src/libexpr/value/context.{hh,cc}`, we write tests for it in `src/nix-expr-tests/value/context.cc`, and (possibly) declare/define additional interfaces for testing purposes in `src/nix-expr-test-support/tests/value/context.{hh,cc}`. Data for unit tests is stored in a `data` subdir of the directory for each unit test executable. -For example, `libnixstore` code is in `src/libstore`, and its test data is in `tests/unit/libstore/data`. -The path to the `tests/unit/data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`. +For example, `libnixstore` code is in `src/libstore`, and its test data is in `src/nix-store-tests/data`. +The path to the `src/${library_name_without-nix}-test/data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`. Note that each executable only gets the data for its tests. -The unit test libraries are in `tests/unit/${library_name_without-nix}-lib`. +The unit test libraries are in `src/${library_name_without-nix}-test-support`. All headers are in a `tests` subdirectory so they are included with `#include "tests/"`. The use of all these separate directories for the unit tests might seem inconvenient, as for example the tests are not "right next to" the part of the code they are testing. @@ -76,8 +76,25 @@ there is no risk of any build-system wildcards for the library accidentally pick ### Running tests -You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. -Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option, or the `GTEST_FILTER` environment variable, e.g. `GTEST_FILTER='ErrorTraceTest.*' make check`. +You can run the whole testsuite with `meson test` from the Meson build directory, or the tests for a specific component with `meson test nix-store-tests`. +A environment variables that Google Test accepts are also worth knowing: + +1. [`GTEST_FILTER`](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) + + This is used for finer-grained filtering of which tests to run. + + +2. [`GTEST_BRIEF`](https://google.github.io/googletest/advanced.html#suppressing-test-passes) + + This is used to avoid logging passing tests. + +Putting the two together, one might run + +```bash +GTEST_BRIEF=1 GTEST_FILTER='ErrorTraceTest.*' meson test nix-expr-tests -v +``` + +for short but comprensive output. ### Characterisation testing { #characaterisation-testing-unit } @@ -86,7 +103,7 @@ See [functional characterisation testing](#characterisation-testing-functional) Like with the functional characterisation, `_NIX_TEST_ACCEPT=1` is also used. For example: ```shell-session -$ _NIX_TEST_ACCEPT=1 make libstore-tests_RUN +$ _NIX_TEST_ACCEPT=1 meson test nix-store-tests -v ... [ SKIPPED ] WorkerProtoTest.string_read [ SKIPPED ] WorkerProtoTest.string_write @@ -114,6 +131,8 @@ On other platforms they wouldn't be run at all. The functional tests reside under the `tests/functional` directory and are listed in `tests/functional/local.mk`. Each test is a bash script. +Functional tests are run during `installCheck` in the `nix` package build, as well as separately from the build, in VM tests. + ### Running the whole test suite The whole test suite can be run with: @@ -162,14 +181,14 @@ ran test tests/functional/${testName}.sh... [PASS] or without `make`: ```shell-session -$ ./mk/run-test.sh tests/functional/${testName}.sh tests/functional/init.sh +$ ./mk/run-test.sh tests/functional/${testName}.sh ran test tests/functional/${testName}.sh... [PASS] ``` To see the complete output, one can also run: ```shell-session -$ ./mk/debug-test.sh tests/functional/${testName}.sh tests/functional/init.sh +$ ./mk/debug-test.sh tests/functional/${testName}.sh +(${testName}.sh:1) foo output from foo +(${testName}.sh:2) bar @@ -204,7 +223,7 @@ edit it like so: Then, running the test with `./mk/debug-test.sh` will drop you into GDB once the script reaches that point: ```shell-session -$ ./mk/debug-test.sh tests/functional/${testName}.sh tests/functional/init.sh +$ ./mk/debug-test.sh tests/functional/${testName}.sh ... + gdb blash blub GNU gdb (GDB) 12.1 @@ -252,13 +271,30 @@ Regressions are caught, and improvements always show up in code review. To ensure that characterisation testing doesn't make it harder to intentionally change these interfaces, there always must be an easy way to regenerate the expected output, as we do with `_NIX_TEST_ACCEPT=1`. +### Running functional tests on NixOS + +We run the functional tests not just in the build, but also in VM tests. +This helps us ensure that Nix works correctly on NixOS, and environments that have similar characteristics that are hard to reproduce in a build environment. + +The recommended way to run these tests during development is: + +```shell +nix build .#hydraJobs.tests.functional_user.quickBuild +``` + +The `quickBuild` attribute configures the test to use a `nix` package that's built without integration tests, so that you can iterate on the tests without performing recompilations due to the changed sources for `installCheck`. + +Generally, this build is sufficient, but in nightly or CI we also test the attributes `functional_root` and `functional_trusted`, in which the test suite is run with different levels of authorization. + ## Integration tests The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute. These tests include everything that needs to interact with external services or run Nix in a non-trivial distributed setup. Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on ). -You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}` +You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}`. + +If you are testing a build of `nix` that you haven't compiled yet, you may iterate faster by appending the `quickBuild` attribute: `nix build .#hydraJobs.tests.{testName}.quickBuild`. ## Installer tests diff --git a/doc/manual/src/favicon.png b/doc/manual/src/favicon.png new file mode 100644 index 000000000..1ed2b5fe0 Binary files /dev/null and b/doc/manual/src/favicon.png differ diff --git a/doc/manual/src/favicon.svg b/doc/manual/src/favicon.svg new file mode 100644 index 000000000..1d2a6e835 --- /dev/null +++ b/doc/manual/src/favicon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index a71b2e2b3..877c4668b 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -1,5 +1,24 @@ # Glossary +- [content address]{#gloss-content-address} + + A + [*content address*](https://en.wikipedia.org/wiki/Content-addressable_storage) + is a secure way to reference immutable data. + The reference is calculated directly from the content of the data being referenced, which means the reference is + [*tamper proof*](https://en.wikipedia.org/wiki/Tamperproofing) + --- variations of the data should always calculate to distinct content addresses. + + For how Nix uses content addresses, see: + + - [Content-Addressing File System Objects](@docroot@/store/file-system-object/content-address.md) + - [Content-Addressing Store Objects](@docroot@/store/store-object/content-address.md) + - [content-addressed derivation](#gloss-content-addressed-derivation) + + Software Heritage's writing on [*Intrinsic and Extrinsic identifiers*](https://www.softwareheritage.org/2020/07/09/intrinsic-vs-extrinsic-identifiers) is also a good introduction to the value of content-addressing over other referencing schemes. + + Besides content addressing, the Nix store also uses [input addressing](#gloss-input-addressed-store-object). + - [derivation]{#gloss-derivation} A description of a build task. The result of a derivation is a @@ -52,10 +71,9 @@ [`__contentAddressed`](./language/advanced-attributes.md#adv-attr-__contentAddressed) attribute set to `true`. -- [fixed-output derivation]{#gloss-fixed-output-derivation} +- [fixed-output derivation]{#gloss-fixed-output-derivation} (FOD) - A derivation which includes the - [`outputHash`](./language/advanced-attributes.md#adv-attr-outputHash) attribute. + A [derivation] where a cryptographic hash of the [output] is determined in advance using the [`outputHash`](./language/advanced-attributes.md#adv-attr-outputHash) attribute, and where the [`builder`](@docroot@/language/derivations.md#attr-builder) executable has access to the network. - [store]{#gloss-store} @@ -86,7 +104,7 @@ [store path]: #gloss-store-path -- [file system object]{#gloss-store-object} +- [file system object]{#gloss-file-system-object} The Nix data model for representing simplified file system data. @@ -118,9 +136,12 @@ - [content-addressed store object]{#gloss-content-addressed-store-object} - A [store object] whose [store path] is determined by its contents. + A [store object] which is [content-addressed](#gloss-content-address), + i.e. whose [store path] is determined by its contents. This includes derivations, the outputs of [content-addressed derivations](#gloss-content-addressed-derivation), and the outputs of [fixed-output derivations](#gloss-fixed-output-derivation). + See [Content-Addressing Store Objects](@docroot@/store/store-object/content-address.md) for details. + - [substitute]{#gloss-substitute} A substitute is a command invocation stored in the [Nix database] that @@ -147,7 +168,7 @@ - [impure derivation]{#gloss-impure-derivation} - [An experimental feature](#@docroot@/contributing/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, + [An experimental feature](#@docroot@/development/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, so that they are always rebuilt, and their outputs not reused by subsequent calls to realise them. - [Nix database]{#gloss-nix-database} @@ -215,6 +236,20 @@ [output path]: #gloss-output-path +- [output closure]{#gloss-output-closure}\ + The [closure] of an [output path]. It only contains what is [reachable] from the output. + +- [deriving path]{#gloss-deriving-path} + + Deriving paths are a way to refer to [store objects][store object] that ar not yet [realised][realise]. + This is necessary because, in general and particularly for [content-addressed derivations][content-addressed derivation], the [output path] of an [output] is not known in advance. + There are two forms: + + - *constant*: just a [store path] + It can be made [valid][validity] by copying it into the store: from the evaluator, command line interface or another store. + + - *output*: a pair of a [store path] to a [derivation] and an [output] name. + - [deriver]{#gloss-deriver} The [store derivation] that produced an [output path]. @@ -252,13 +287,15 @@ See [installables](./command-ref/new-cli/nix.md#installables) for [`nix` commands](./command-ref/new-cli/nix.md) (experimental) for details. -- [NAR]{#gloss-nar} +- [Nix Archive (NAR)]{#gloss-nar} A *N*ix *AR*chive. This is a serialisation of a path in the Nix store. It can contain regular files, directories and symbolic links. NARs are generated and unpacked using `nix-store --dump` and `nix-store --restore`. + See [Nix Archive](store/file-system-object/content-address.html#serial-nix-archive) for details. + - [`∅`]{#gloss-emtpy-set} The empty set symbol. In the context of profile history, this denotes a package is not present in a particular version of the profile. @@ -275,7 +312,7 @@ - [package attribute set]{#package-attribute-set} - An [attribute set](@docroot@/language/values.md#attribute-set) containing the attribute `type = "derivation";` (derivation for historical reasons), as well as other attributes, such as + An [attribute set](@docroot@/language/types.md#attribute-set) containing the attribute `type = "derivation";` (derivation for historical reasons), as well as other attributes, such as - attributes that refer to the files of a [package], typically in the form of [derivation outputs](#output), - attributes that declare something about how the package is supposed to be installed or used, - other metadata or arbitrary attributes. @@ -288,16 +325,35 @@ See [String interpolation](./language/string-interpolation.md) for details. - [string]: ./language/values.md#type-string - [path]: ./language/values.md#type-path - [attribute name]: ./language/values.md#attribute-set + [string]: ./language/types.md#type-string + [path]: ./language/types.md#type-path + [attribute name]: ./language/types.md#attribute-set + +- [base directory]{#gloss-base-directory} + + The location from which relative paths are resolved. + + - For expressions in a file, the base directory is the directory containing that file. + This is analogous to the directory of a [base URL](https://datatracker.ietf.org/doc/html/rfc1808#section-3.3). + + + + - For expressions written in command line arguments with [`--expr`](@docroot@/command-ref/opt-common.html#opt-expr), the base directory is the current working directory. + + [base directory]: #gloss-base-directory - [experimental feature]{#gloss-experimental-feature} Not yet stabilized functionality guarded by named experimental feature flags. These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting. - See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/contributing/experimental-features.md). + See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/development/experimental-features.md). [Nix language]: ./language/index.md diff --git a/doc/manual/src/installation/env-variables.md b/doc/manual/src/installation/env-variables.md index db98f52ff..035090421 100644 --- a/doc/manual/src/installation/env-variables.md +++ b/doc/manual/src/installation/env-variables.md @@ -53,7 +53,8 @@ ssl-cert-file = /etc/ssl/my-certificate-bundle.crt The Nix installer has special handling for these proxy-related environment variables: `http_proxy`, `https_proxy`, `ftp_proxy`, -`no_proxy`, `HTTP_PROXY`, `HTTPS_PROXY`, `FTP_PROXY`, `NO_PROXY`. +`all_proxy`, `no_proxy`, `HTTP_PROXY`, `HTTPS_PROXY`, `FTP_PROXY`, +`ALL_PROXY`, `NO_PROXY`. If any of these variables are set when running the Nix installer, then the installer will create an override file at diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index 0dc989159..6a168ff3d 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -50,7 +50,7 @@ Supported systems: To explicitly instruct the installer to perform a multi-user installation on your system: ```console -$ curl -L https://nixos.org/nix/install | sh -s -- --daemon +$ bash <(curl -L https://nixos.org/nix/install) --daemon ``` You can run this under your usual user account or `root`. @@ -61,7 +61,7 @@ The script will invoke `sudo` as needed. To explicitly select a single-user installation on your system: ```console -$ curl -L https://nixos.org/nix/install | sh -s -- --no-daemon +$ bash <(curl -L https://nixos.org/nix/install) --no-daemon ``` In a single-user installation, `/nix` is owned by the invoking user. @@ -77,7 +77,7 @@ $ su root # Installing from a binary tarball You can also download a binary tarball that contains Nix and all its dependencies: -- Choose a [version](https://releases.nixos.org/?prefix=nix/) and [system type](../contributing/hacking.md#platforms) +- Choose a [version](https://releases.nixos.org/?prefix=nix/) and [system type](../development/building.md#platforms) - Download and unpack the tarball - Run the installer diff --git a/doc/manual/src/installation/uninstall.md b/doc/manual/src/installation/uninstall.md index 9ead5e53c..590327fea 100644 --- a/doc/manual/src/installation/uninstall.md +++ b/doc/manual/src/installation/uninstall.md @@ -1,16 +1,8 @@ # Uninstalling Nix -## Single User - -If you have a [single-user installation](./installing-binary.md#single-user-installation) of Nix, uninstall it by running: - -```console -$ rm -rf /nix -``` - ## Multi User -Removing a [multi-user installation](./installing-binary.md#multi-user-installation) of Nix is more involved, and depends on the operating system. +Removing a [multi-user installation](./installing-binary.md#multi-user-installation) depends on the operating system. ### Linux @@ -51,7 +43,15 @@ which you may remove. ### macOS -1. Edit `/etc/zshrc`, `/etc/bashrc`, and `/etc/bash.bashrc` to remove the lines sourcing `nix-daemon.sh`, which should look like this: +1. If system-wide shell initialisation files haven't been altered since installing Nix, use the backups made by the installer: + + ```console + sudo mv /etc/zshrc.backup-before-nix /etc/zshrc + sudo mv /etc/bashrc.backup-before-nix /etc/bashrc + sudo mv /etc/bash.bashrc.backup-before-nix /etc/bash.bashrc + ``` + + Otherwise, edit `/etc/zshrc`, `/etc/bashrc`, and `/etc/bash.bashrc` to remove the lines sourcing `nix-daemon.sh`, which should look like this: ```bash # Nix @@ -61,18 +61,6 @@ which you may remove. # End Nix ``` - If these files haven't been altered since installing Nix you can simply put - the backups back in place: - - ```console - sudo mv /etc/zshrc.backup-before-nix /etc/zshrc - sudo mv /etc/bashrc.backup-before-nix /etc/bashrc - sudo mv /etc/bash.bashrc.backup-before-nix /etc/bash.bashrc - ``` - - This will stop shells from sourcing the file and bringing everything you - installed using Nix in scope. - 2. Stop and remove the Nix daemon services: ```console @@ -82,8 +70,7 @@ which you may remove. sudo rm /Library/LaunchDaemons/org.nixos.darwin-store.plist ``` - This stops the Nix daemon and prevents it from being started next time you - boot the system. + This stops the Nix daemon and prevents it from being started next time you boot the system. 3. Remove the `nixbld` group and the `_nixbuildN` users: @@ -94,25 +81,42 @@ which you may remove. This will remove all the build users that no longer serve a purpose. -4. Edit fstab using `sudo vifs` to remove the line mounting the Nix Store - volume on `/nix`, which looks like - `UUID= /nix apfs rw,noauto,nobrowse,suid,owners` or - `LABEL=Nix\040Store /nix apfs rw,nobrowse`. This will prevent automatic - mounting of the Nix Store volume. +4. Edit fstab using `sudo vifs` to remove the line mounting the Nix Store volume on `/nix`, which looks like -5. Edit `/etc/synthetic.conf` to remove the `nix` line. If this is the only - line in the file you can remove it entirely, `sudo rm /etc/synthetic.conf`. - This will prevent the creation of the empty `/nix` directory to provide a - mountpoint for the Nix Store volume. + ``` + UUID= /nix apfs rw,noauto,nobrowse,suid,owners + ``` + or -6. Remove the files Nix added to your system: + ``` + LABEL=Nix\040Store /nix apfs rw,nobrowse + ``` + + by setting the cursor on the respective line using the error keys, and pressing `dd`, and then `:wq` to save the file. + + This will prevent automatic mounting of the Nix Store volume. + +5. Edit `/etc/synthetic.conf` to remove the `nix` line. + If this is the only line in the file you can remove it entirely: + + ```bash + if [ -f /etc/synthetic.conf ]; then + if [ "$(cat /etc/synthetic.conf)" = "nix" ]; then + sudo rm /etc/synthetic.conf + else + sudo vi /etc/synthetic.conf + fi + fi + ``` + + This will prevent the creation of the empty `/nix` directory. + +6. Remove the files Nix added to your system, except for the store: ```console sudo rm -rf /etc/nix /var/root/.nix-profile /var/root/.nix-defexpr /var/root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels ``` - This gets rid of any data Nix may have created except for the store which is - removed next. 7. Remove the Nix Store volume: @@ -120,29 +124,32 @@ which you may remove. sudo diskutil apfs deleteVolume /nix ``` - This will remove the Nix Store volume and everything that was added to the - store. + This will remove the Nix Store volume and everything that was added to the store. - If the output indicates that the command couldn't remove the volume, you should - make sure you don't have an _unmounted_ Nix Store volume. Look for a - "Nix Store" volume in the output of the following command: + If the output indicates that the command couldn't remove the volume, you should make sure you don't have an _unmounted_ Nix Store volume. + Look for a "Nix Store" volume in the output of the following command: ```console diskutil list ``` - If you _do_ see a "Nix Store" volume, delete it by re-running the diskutil - deleteVolume command, but replace `/nix` with the store volume's `diskXsY` - identifier. + If you _do_ find a "Nix Store" volume, delete it by running `diskutil deleteVolume` with the store volume's `diskXsY` identifier. > **Note** > -> After you complete the steps here, you will still have an empty `/nix` -> directory. This is an expected sign of a successful uninstall. The empty -> `/nix` directory will disappear the next time you reboot. +> After you complete the steps here, you will still have an empty `/nix` directory. +> This is an expected sign of a successful uninstall. +> The empty `/nix` directory will disappear the next time you reboot. > -> You do not have to reboot to finish uninstalling Nix. The uninstall is -> complete. macOS (Catalina+) directly controls root directories and its -> read-only root will prevent you from manually deleting the empty `/nix` -> mountpoint. +> You do not have to reboot to finish uninstalling Nix. +> The uninstall is complete. +> macOS (Catalina+) directly controls root directories, and its read-only root will prevent you from manually deleting the empty `/nix` mountpoint. +## Single User + +To remove a [single-user installation](./installing-binary.md#single-user-installation) of Nix, run: + +```console +$ rm -rf /nix ~/.nix-channels ~/.nix-defexpr ~/.nix-profile +``` +You might also want to manually remove references to Nix from your `~/.profile`. diff --git a/doc/manual/src/installation/upgrading.md b/doc/manual/src/installation/upgrading.md index 38edcdbc5..a433f1d30 100644 --- a/doc/manual/src/installation/upgrading.md +++ b/doc/manual/src/installation/upgrading.md @@ -28,7 +28,7 @@ $ sudo su ## macOS multi-user ```console -$ sudo nix-env --install --file '' --attr nix -I nixpkgs=channel:nixpkgs-unstable +$ sudo nix-env --install --file '' --attr nix cacert -I nixpkgs=channel:nixpkgs-unstable $ sudo launchctl remove org.nixos.nix-daemon $ sudo launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist ``` diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 7306fc182..51b83fc8a 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -113,19 +113,18 @@ Derivations can declare some infrequently used optional attributes. > `nix-build`. If the [`configurable-impure-env` experimental - feature](@docroot@/contributing/experimental-features.md#xp-feature-configurable-impure-env) + feature](@docroot@/development/experimental-features.md#xp-feature-configurable-impure-env) is enabled, these environment variables can also be controlled through the [`impure-env`](@docroot@/command-ref/conf-file.md#conf-impure-env) configuration setting. - [`outputHash`]{#adv-attr-outputHash}; [`outputHashAlgo`]{#adv-attr-outputHashAlgo}; [`outputHashMode`]{#adv-attr-outputHashMode}\ - These attributes declare that the derivation is a so-called - *fixed-output derivation*, which means that a cryptographic hash of - the output is already known in advance. When the build of a - fixed-output derivation finishes, Nix computes the cryptographic - hash of the output and compares it to the hash declared with these - attributes. If there is a mismatch, the build fails. + These attributes declare that the derivation is a so-called *fixed-output derivation* (FOD), which means that a cryptographic hash of the output is already known in advance. + + As opposed to regular derivations, the [`builder`] executable of a fixed-output derivation has access to the network. + Nix computes a cryptographic hash of its output and compares that to the hash declared with these attributes. + If there is a mismatch, the derivation fails. The rationale for fixed-output derivations is derivations such as those produced by the `fetchurl` function. This function downloads a @@ -188,38 +187,49 @@ Derivations can declare some infrequently used optional attributes. } ``` - The `outputHashAlgo` attribute specifies the hash algorithm used to - compute the hash. It can currently be `"sha1"`, `"sha256"` or - `"sha512"`. + The `outputHash` attribute must be a string containing the hash in either hexadecimal or "nix32" encoding, or following the format for integrity metadata as defined by [SRI](https://www.w3.org/TR/SRI/). + The "nix32" encoding is an adaptation of base-32 encoding. + The [`convertHash`](@docroot@/language/builtins.md#builtins-convertHash) function shows how to convert between different encodings, and the [`nix-hash` command](../command-ref/nix-hash.md) has information about obtaining the hash for some contents, as well as converting to and from encodings. + + The `outputHashAlgo` attribute specifies the hash algorithm used to compute the hash. + It can currently be `"sha1"`, `"sha256"`, `"sha512"`, or `null`. + `outputHashAlgo` can only be `null` when `outputHash` follows the SRI format. The `outputHashMode` attribute determines how the hash is computed. - It must be one of the following two values: + It must be one of the following values: - - `"flat"`\ - The output must be a non-executable regular file. If it isn’t, - the build fails. The hash is simply computed over the contents - of that file (so it’s equal to what Unix commands like - `sha256sum` or `sha1sum` produce). + - [`"flat"`](@docroot@/store/store-object/content-address.md#method-flat) This is the default. - - `"recursive"`\ - The hash is computed over the NAR archive dump of the output - (i.e., the result of [`nix-store --dump`](@docroot@/command-ref/nix-store/dump.md)). In - this case, the output can be anything, including a directory - tree. + - [`"recursive"` or `"nar"`](@docroot@/store/store-object/content-address.md#method-nix-archive) - The `outputHash` attribute, finally, must be a string containing - the hash in either hexadecimal or base-32 notation. (See the - [`nix-hash` command](../command-ref/nix-hash.md) for information - about converting to and from base-32 notation.) + > **Compatibility** + > + > `"recursive"` is the traditional way of indicating this, + > and is supported since 2005 (virtually the entire history of Nix). + > `"nar"` is more clear, and consistent with other parts of Nix (such as the CLI), + > however support for it is only added in Nix version 2.21. + + - [`"text"`](@docroot@/store/store-object/content-address.md#method-text) + + > **Warning** + > + > The use of this method for derivation outputs is part of the [`dynamic-derivations`][xp-feature-dynamic-derivations] experimental feature. + + - [`"git"`](@docroot@/store/store-object/content-address.md#method-git) + + > **Warning** + > + > This method is part of the [`git-hashing`][xp-feature-git-hashing] experimental feature. - [`__contentAddressed`]{#adv-attr-__contentAddressed} + > **Warning** - > This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md). + > This attribute is part of an [experimental feature](@docroot@/development/experimental-features.md). > > To use this attribute, you must enable the - > [`ca-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-ca-derivations) experimental feature. + > [`ca-derivations`][xp-feature-ca-derivations] experimental feature. > For example, in [nix.conf](../command-ref/conf-file.md) you could add: > > ``` @@ -268,7 +278,9 @@ Derivations can declare some infrequently used optional attributes. > **Note** > - > If set to `false`, the [`builder`](./derivations.md#attr-builder) should be able to run on the system type specified in the [`system` attribute](./derivations.md#attr-system), since the derivation cannot be substituted. + > If set to `false`, the [`builder`] should be able to run on the system type specified in the [`system` attribute](./derivations.md#attr-system), since the derivation cannot be substituted. + + [`builder`]: ./derivations.md#attr-builder - [`__structuredAttrs`]{#adv-attr-structuredAttrs}\ If the special attribute `__structuredAttrs` is set to `true`, the other derivation @@ -290,6 +302,12 @@ Derivations can declare some infrequently used optional attributes. (associative) arrays. For example, the attribute `hardening.format = true` ends up as the Bash associative array element `${hardening[format]}`. + > **Warning** + > + > If set to `true`, other advanced attributes such as [`allowedReferences`](#adv-attr-allowedReferences), [`allowedReferences`](#adv-attr-allowedReferences), [`allowedRequisites`](#adv-attr-allowedRequisites), + [`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites), maxSize, and maxClosureSize. + will have no effect. + - [`outputChecks`]{#adv-attr-outputChecks}\ When using [structured attributes](#adv-attr-structuredAttrs), the `outputChecks` attribute allows defining checks per-output. @@ -299,7 +317,7 @@ Derivations can declare some infrequently used optional attributes. [`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites), the following attributes are available: - - `maxSize` defines the maximum size of the resulting [store object](../glossary.md#gloss-store-object). + - `maxSize` defines the maximum size of the resulting [store object](@docroot@/store/store-object.md). - `maxClosureSize` defines the maximum size of the output's closure. - `ignoreSelfRefs` controls whether self-references should be considered when checking for allowed references/requisites. @@ -351,3 +369,7 @@ Derivations can declare some infrequently used optional attributes. ``` ensures that the derivation can only be built on a machine with the `kvm` feature. + +[xp-feature-ca-derivations]: @docroot@/development/experimental-features.md#xp-feature-ca-derivations +[xp-feature-dynamic-derivations]: @docroot@/development/experimental-features.md#xp-feature-dynamic-derivations +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/doc/manual/src/language/builtin-constants-prefix.md b/doc/manual/src/language/builtin-constants-prefix.md deleted file mode 100644 index 50f43006d..000000000 --- a/doc/manual/src/language/builtin-constants-prefix.md +++ /dev/null @@ -1,5 +0,0 @@ -# Built-in Constants - -These constants are built into the Nix language evaluator: - - diff --git a/doc/manual/src/language/builtin-constants-suffix.md b/doc/manual/src/language/builtin-constants-suffix.md deleted file mode 100644 index a74db2857..000000000 --- a/doc/manual/src/language/builtin-constants-suffix.md +++ /dev/null @@ -1 +0,0 @@ - diff --git a/doc/manual/src/language/builtins-prefix.md b/doc/manual/src/language/builtins-prefix.md index 7b2321466..fb983bb7f 100644 --- a/doc/manual/src/language/builtins-prefix.md +++ b/doc/manual/src/language/builtins-prefix.md @@ -1,9 +1,11 @@ -# Built-in Functions +# Built-ins -This section lists the functions built into the Nix language evaluator. -All built-in functions are available through the global [`builtins`](./builtin-constants.md#builtins-builtins) constant. +This section lists the values and functions built into the Nix language evaluator. +All built-ins are available through the global [`builtins`](#builtins-builtins) constant. -For convenience, some built-ins can be accessed directly: +Some built-ins are also exposed directly in the global scope: + + - [`derivation`](#builtins-derivation) - [`import`](#builtins-import) diff --git a/doc/manual/src/language/constructs.md b/doc/manual/src/language/constructs.md deleted file mode 100644 index a82ec5960..000000000 --- a/doc/manual/src/language/constructs.md +++ /dev/null @@ -1,408 +0,0 @@ -# Language Constructs - -## Recursive sets - -Recursive sets are like normal [attribute sets](./values.md#attribute-set), but the attributes can refer to each other. - -> *rec-attrset* = `rec {` [ *name* `=` *expr* `;` `]`... `}` - -Example: - -```nix -rec { - x = y; - y = 123; -}.x -``` - -This evaluates to `123`. - -Note that without `rec` the binding `x = y;` would -refer to the variable `y` in the surrounding scope, if one exists, and -would be invalid if no such variable exists. That is, in a normal -(non-recursive) set, attributes are not added to the lexical scope; in a -recursive set, they are. - -Recursive sets of course introduce the danger of infinite recursion. For -example, the expression - -```nix -rec { - x = y; - y = x; -}.x -``` - -will crash with an `infinite recursion encountered` error message. - -## Let-expressions - -A let-expression allows you to define local variables for an expression. - -> *let-in* = `let` [ *identifier* = *expr* ]... `in` *expr* - -Example: - -```nix -let - x = "foo"; - y = "bar"; -in x + y -``` - -This evaluates to `"foobar"`. - -## Inheriting attributes - -When defining an [attribute set](./values.md#attribute-set) or in a [let-expression](#let-expressions) it is often convenient to copy variables from the surrounding lexical scope (e.g., when you want to propagate attributes). -This can be shortened using the `inherit` keyword. - -Example: - -```nix -let x = 123; in -{ - inherit x; - y = 456; -} -``` - -is equivalent to - -```nix -let x = 123; in -{ - x = x; - y = 456; -} -``` - -and both evaluate to `{ x = 123; y = 456; }`. - -> **Note** -> -> This works because `x` is added to the lexical scope by the `let` construct. - -It is also possible to inherit attributes from another attribute set. - -Example: - -In this fragment from `all-packages.nix`, - -```nix -graphviz = (import ../tools/graphics/graphviz) { - inherit fetchurl stdenv libpng libjpeg expat x11 yacc; - inherit (xorg) libXaw; -}; - -xorg = { - libX11 = ...; - libXaw = ...; - ... -} - -libpng = ...; -libjpg = ...; -... -``` - -the set used in the function call to the function defined in -`../tools/graphics/graphviz` inherits a number of variables from the -surrounding scope (`fetchurl` ... `yacc`), but also inherits `libXaw` -(the X Athena Widgets) from the `xorg` set. - -Summarizing the fragment - -```nix -... -inherit x y z; -inherit (src-set) a b c; -... -``` - -is equivalent to - -```nix -... -x = x; y = y; z = z; -a = src-set.a; b = src-set.b; c = src-set.c; -... -``` - -when used while defining local variables in a let-expression or while -defining a set. - -In a `let` expression, `inherit` can be used to selectively bring specific attributes of a set into scope. For example - - -```nix -let - x = { a = 1; b = 2; }; - inherit (builtins) attrNames; -in -{ - names = attrNames x; -} -``` - -is equivalent to - -```nix -let - x = { a = 1; b = 2; }; -in -{ - names = builtins.attrNames x; -} -``` - -both evaluate to `{ names = [ "a" "b" ]; }`. - -## Functions - -Functions have the following form: - -```nix -pattern: body -``` - -The pattern specifies what the argument of the function must look like, -and binds variables in the body to (parts of) the argument. There are -three kinds of patterns: - - - If a pattern is a single identifier, then the function matches any - argument. Example: - - ```nix - let negate = x: !x; - concat = x: y: x + y; - in if negate true then concat "foo" "bar" else "" - ``` - - Note that `concat` is a function that takes one argument and returns - a function that takes another argument. This allows partial - parameterisation (i.e., only filling some of the arguments of a - function); e.g., - - ```nix - map (concat "foo") [ "bar" "bla" "abc" ] - ``` - - evaluates to `[ "foobar" "foobla" "fooabc" ]`. - - - A *set pattern* of the form `{ name1, name2, …, nameN }` matches a - set containing the listed attributes, and binds the values of those - attributes to variables in the function body. For example, the - function - - ```nix - { x, y, z }: z + y + x - ``` - - can only be called with a set containing exactly the attributes `x`, - `y` and `z`. No other attributes are allowed. If you want to allow - additional arguments, you can use an ellipsis (`...`): - - ```nix - { x, y, z, ... }: z + y + x - ``` - - This works on any set that contains at least the three named - attributes. - - It is possible to provide *default values* for attributes, in - which case they are allowed to be missing. A default value is - specified by writing `name ? e`, where *e* is an arbitrary - expression. For example, - - ```nix - { x, y ? "foo", z ? "bar" }: z + y + x - ``` - - specifies a function that only requires an attribute named `x`, but - optionally accepts `y` and `z`. - - - An `@`-pattern provides a means of referring to the whole value - being matched: - - ```nix - args@{ x, y, z, ... }: z + y + x + args.a - ``` - - but can also be written as: - - ```nix - { x, y, z, ... } @ args: z + y + x + args.a - ``` - - Here `args` is bound to the argument *as passed*, which is further - matched against the pattern `{ x, y, z, ... }`. - The `@`-pattern makes mainly sense with an ellipsis(`...`) as - you can access attribute names as `a`, using `args.a`, which was - given as an additional attribute to the function. - - > **Warning** - > - > `args@` binds the name `args` to the attribute set that is passed to the function. - > In particular, `args` does *not* include any default values specified with `?` in the function's set pattern. - > - > For instance - > - > ```nix - > let - > f = args@{ a ? 23, ... }: [ a args ]; - > in - > f {} - > ``` - > - > is equivalent to - > - > ```nix - > let - > f = args @ { ... }: [ (args.a or 23) args ]; - > in - > f {} - > ``` - > - > and both expressions will evaluate to: - > - > ```nix - > [ 23 {} ] - > ``` - -Note that functions do not have names. If you want to give them a name, -you can bind them to an attribute, e.g., - -```nix -let concat = { x, y }: x + y; -in concat { x = "foo"; y = "bar"; } -``` - -## Conditionals - -Conditionals look like this: - -```nix -if e1 then e2 else e3 -``` - -where *e1* is an expression that should evaluate to a Boolean value -(`true` or `false`). - -## Assertions - -Assertions are generally used to check that certain requirements on or -between features and dependencies hold. They look like this: - -```nix -assert e1; e2 -``` - -where *e1* is an expression that should evaluate to a Boolean value. If -it evaluates to `true`, *e2* is returned; otherwise expression -evaluation is aborted and a backtrace is printed. - -Here is a Nix expression for the Subversion package that shows how -assertions can be used:. - -```nix -{ localServer ? false -, httpServer ? false -, sslSupport ? false -, pythonBindings ? false -, javaSwigBindings ? false -, javahlBindings ? false -, stdenv, fetchurl -, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null -}: - -assert localServer -> db4 != null; â‘ -assert httpServer -> httpd != null && httpd.expat == expat; â‘¡ -assert sslSupport -> openssl != null && (httpServer -> httpd.openssl == openssl); â‘¢ -assert pythonBindings -> swig != null && swig.pythonSupport; -assert javaSwigBindings -> swig != null && swig.javaSupport; -assert javahlBindings -> j2sdk != null; - -stdenv.mkDerivation { - name = "subversion-1.1.1"; - ... - openssl = if sslSupport then openssl else null; â‘£ - ... -} -``` - -The points of interest are: - -1. This assertion states that if Subversion is to have support for - local repositories, then Berkeley DB is needed. So if the Subversion - function is called with the `localServer` argument set to `true` but - the `db4` argument set to `null`, then the evaluation fails. - - Note that `->` is the [logical - implication](https://en.wikipedia.org/wiki/Truth_table#Logical_implication) - Boolean operation. - -2. This is a more subtle condition: if Subversion is built with Apache - (`httpServer`) support, then the Expat library (an XML library) used - by Subversion should be same as the one used by Apache. This is - because in this configuration Subversion code ends up being linked - with Apache code, and if the Expat libraries do not match, a build- - or runtime link error or incompatibility might occur. - -3. This assertion says that in order for Subversion to have SSL support - (so that it can access `https` URLs), an OpenSSL library must be - passed. Additionally, it says that *if* Apache support is enabled, - then Apache's OpenSSL should match Subversion's. (Note that if - Apache support is not enabled, we don't care about Apache's - OpenSSL.) - -4. The conditional here is not really related to assertions, but is - worth pointing out: it ensures that if SSL support is disabled, then - the Subversion derivation is not dependent on OpenSSL, even if a - non-`null` value was passed. This prevents an unnecessary rebuild of - Subversion if OpenSSL changes. - -## With-expressions - -A *with-expression*, - -```nix -with e1; e2 -``` - -introduces the set *e1* into the lexical scope of the expression *e2*. -For instance, - -```nix -let as = { x = "foo"; y = "bar"; }; -in with as; x + y -``` - -evaluates to `"foobar"` since the `with` adds the `x` and `y` attributes -of `as` to the lexical scope in the expression `x + y`. The most common -use of `with` is in conjunction with the `import` function. E.g., - -```nix -with (import ./definitions.nix); ... -``` - -makes all attributes defined in the file `definitions.nix` available as -if they were defined locally in a `let`-expression. - -The bindings introduced by `with` do not shadow bindings introduced by -other means, e.g. - -```nix -let a = 3; in with { a = 1; }; let a = 4; in with { a = 2; }; ... -``` - -establishes the same scope as - -```nix -let a = 1; in let a = 2; in let a = 3; in let a = 4; in ... -``` - -## Comments - -Comments can be single-line, started with a `#` character, or -inline/multi-line, enclosed within `/* ... */`. diff --git a/doc/manual/src/language/constructs/lookup-path.md b/doc/manual/src/language/constructs/lookup-path.md index e87d2922b..11b9fe88c 100644 --- a/doc/manual/src/language/constructs/lookup-path.md +++ b/doc/manual/src/language/constructs/lookup-path.md @@ -4,9 +4,9 @@ > > *lookup-path* = `<` *identifier* [ `/` *identifier* ]... `>` -A lookup path is an identifier with an optional path suffix that resolves to a [path value](@docroot@/language/values.md#type-path) if the identifier matches a search path entry. +A lookup path is an identifier with an optional path suffix that resolves to a [path value](@docroot@/language/types.md#type-path) if the identifier matches a search path entry. -The value of a lookup path is determined by [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath). +The value of a lookup path is determined by [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath). See [`builtins.findFile`](@docroot@/language/builtins.md#builtins-findFile) for details on lookup path resolution. diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index 75f824a34..8e3f0f791 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -12,12 +12,12 @@ It outputs an attribute set, and produces a [store derivation] as a side effect ### Required -- [`name`]{#attr-name} ([String](@docroot@/language/values.md#type-string)) +- [`name`]{#attr-name} ([String](@docroot@/language/types.md#type-string)) A symbolic name for the derivation. It is added to the [store path] of the corresponding [store derivation] as well as to its [output paths](@docroot@/glossary.md#gloss-output-path). - [store path]: @docroot@/glossary.md#gloss-store-path + [store path]: @docroot@/store/store-path.md > **Example** > @@ -31,7 +31,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect > The store derivation's path will be `/nix/store/-hello.drv`. > The [output](#attr-outputs) paths will be of the form `/nix/store/-hello[-]` -- [`system`]{#attr-system} ([String](@docroot@/language/values.md#type-string)) +- [`system`]{#attr-system} ([String](@docroot@/language/types.md#type-string)) The system type on which the [`builder`](#attr-builder) executable is meant to be run. @@ -64,9 +64,9 @@ It outputs an attribute set, and produces a [store derivation] as a side effect > } > ``` > - > [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) has the value of the [`system` configuration option], and defaults to the system type of the current Nix installation. + > [`builtins.currentSystem`](@docroot@/language/builtins.md#builtins-currentSystem) has the value of the [`system` configuration option], and defaults to the system type of the current Nix installation. -- [`builder`]{#attr-builder} ([Path](@docroot@/language/values.md#type-path) | [String](@docroot@/language/values.md#type-string)) +- [`builder`]{#attr-builder} ([Path](@docroot@/language/types.md#type-path) | [String](@docroot@/language/types.md#type-string)) Path to an executable that will perform the build. @@ -113,7 +113,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect ### Optional -- [`args`]{#attr-args} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) +- [`args`]{#attr-args} ([List](@docroot@/language/types.md#list) of [String](@docroot@/language/types.md#type-string)) Default: `[ ]` @@ -132,7 +132,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect > }; > ``` -- [`outputs`]{#attr-outputs} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) +- [`outputs`]{#attr-outputs} ([List](@docroot@/language/types.md#list) of [String](@docroot@/language/types.md#type-string)) Default: `[ "out" ]` @@ -141,7 +141,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect By default, a derivation produces a single output called `out`. However, derivations can produce multiple outputs. - This allows the associated [store objects](@docroot@/glossary.md#gloss-store-object) and their [closures](@docroot@/glossary.md#gloss-closure) to be copied or garbage-collected separately. + This allows the associated [store objects](@docroot@/store/store-object.md) and their [closures](@docroot@/glossary.md#gloss-closure) to be copied or garbage-collected separately. > **Example** > diff --git a/doc/manual/src/language/identifiers.md b/doc/manual/src/language/identifiers.md new file mode 100644 index 000000000..861ee3e20 --- /dev/null +++ b/doc/manual/src/language/identifiers.md @@ -0,0 +1,51 @@ +# Identifiers + +An *identifier* is an [ASCII](https://en.wikipedia.org/wiki/ASCII) character sequence that: +- Starts with a letter (`a-z`, `A-Z`) or underscore (`_`) +- Can contain any number of: + - Letters (`a-z`, `A-Z`) + - Digits (`0-9`) + - Underscores (`_`) + - Apostrophes (`'`) + - Hyphens (`-`) +- Is not one of the [keywords](#keywords) + +> **Syntax** +> +> *identifier* ~ `[A-Za-z_][A-Za-z0-9_'-]*` + +# Names + +A *name* can be written as an [identifier](#identifier) or a [string literal](./syntax.md#string-literal). + +> **Syntax** +> +> *name* → *identifier* | *string* + +Names are used in [attribute sets](./syntax.md#attrs-literal), [`let` bindings](./syntax.md#let-expressions), and [`inherit`](./syntax.md#inheriting-attributes). +Two names are the same if they represent the same sequence of characters, regardless of whether they are written as identifiers or strings. + +# Keywords + +These keywords are reserved and cannot be used as [identifiers](#identifiers): + +- [`assert`](./syntax.md#assertions) +- [`else`][if] +- [`if`][if] +- [`in`][let] +- [`inherit`](./syntax.md#inheriting-attributes) +- [`let`][let] +- [`or`](./operators.md#attribute-selection) (see note) +- [`rec`](./syntax.md#recursive-sets) +- [`then`][if] +- [`with`](./syntax.md#with-expressions) + +[if]: ./syntax.md#conditionals +[let]: ./syntax.md#let-expressions + +> **Note** +> +> The Nix language evaluator currently allows `or` to be used as a name in some contexts, for backwards compatibility reasons. +> Users are advised not to rely on this. +> +> There are long-standing issues with how `or` is parsed as a name, which can't be resolved without making a breaking change to the language. diff --git a/doc/manual/src/language/import-from-derivation.md b/doc/manual/src/language/import-from-derivation.md index fb12ba51a..e901f5bcf 100644 --- a/doc/manual/src/language/import-from-derivation.md +++ b/doc/manual/src/language/import-from-derivation.md @@ -2,9 +2,9 @@ The value of a Nix expression can depend on the contents of a [store object]. -[store object]: @docroot@/glossary.md#gloss-store-object +[store object]: @docroot@/store/store-object.md -Passing an expression `expr` that evaluates to a [store path](@docroot@/glossary.md#gloss-store-path) to any built-in function which reads from the filesystem constitutes Import From Derivation (IFD): +Passing an expression `expr` that evaluates to a [store path](@docroot@/store/store-path.md) to any built-in function which reads from the filesystem constitutes Import From Derivation (IFD): - [`import`](./builtins.md#builtins-import)` expr` - [`builtins.readFile`](./builtins.md#builtins-readFile)` expr` diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md index a26e43a05..2bfdbb8a0 100644 --- a/doc/manual/src/language/index.md +++ b/doc/manual/src/language/index.md @@ -1,7 +1,13 @@ # Nix Language The Nix language is designed for conveniently creating and composing *derivations* – precise descriptions of how contents of existing files are used to derive new files. -It is: + +> **Tip** +> +> These pages are written as a reference. +> If you are learning Nix, nix.dev has a good [introduction to the Nix language](https://nix.dev/tutorials/nix-language). + +The language is: - *domain-specific* @@ -47,7 +53,7 @@ This is an incomplete overview of language features, by example. - *Basic values* + *Basic values ([primitives](@docroot@/language/types.md#primitives))* @@ -65,7 +71,7 @@ This is an incomplete overview of language features, by example. - A string + A [string](@docroot@/language/types.md#type-string) @@ -88,6 +94,18 @@ This is an incomplete overview of language features, by example. + + + + `# Explanation` + + + + + A [comment](@docroot@/language/syntax.md#comments). + + + @@ -100,7 +118,7 @@ This is an incomplete overview of language features, by example. - String interpolation (expands to `"hello world"`, `"1 2 3"`, `"/nix/store/-bash-/bin/sh"`) + [String interpolation](@docroot@/language/string-interpolation.md) (expands to `"hello world"`, `"1 2 3"`, `"/nix/store/-bash-/bin/sh"`) @@ -112,7 +130,7 @@ This is an incomplete overview of language features, by example. - Booleans + [Booleans](@docroot@/language/types.md#type-boolean) @@ -124,7 +142,7 @@ This is an incomplete overview of language features, by example. - Null value + [Null](@docroot@/language/types.md#type-null) value @@ -136,7 +154,7 @@ This is an incomplete overview of language features, by example. - An integer + An [integer](@docroot@/language/types.md#type-int) @@ -148,7 +166,7 @@ This is an incomplete overview of language features, by example. - A floating point number + A [floating point number](@docroot@/language/types.md#type-float) @@ -160,7 +178,7 @@ This is an incomplete overview of language features, by example. - An absolute path + An absolute [path](@docroot@/language/types.md#type-path) @@ -172,7 +190,7 @@ This is an incomplete overview of language features, by example. - A path relative to the file containing this Nix expression + A [path](@docroot@/language/types.md#type-path) relative to the file containing this Nix expression @@ -184,7 +202,7 @@ This is an incomplete overview of language features, by example. - A home path. Evaluates to the `"/.config"`. + A home [path](@docroot@/language/types.md#type-path). Evaluates to the `"/.config"`. @@ -196,7 +214,7 @@ This is an incomplete overview of language features, by example. - Search path for Nix files. Value determined by [`$NIX_PATH` environment variable](../command-ref/env-common.md#env-NIX_PATH). + A [lookup path](@docroot@/language/constructs/lookup-path.md) for Nix files. Value determined by [`$NIX_PATH` environment variable](../command-ref/env-common.md#env-NIX_PATH). @@ -220,7 +238,7 @@ This is an incomplete overview of language features, by example. - A set with attributes named `x` and `y` + An [attribute set](@docroot@/language/types.md#attribute-set) with attributes named `x` and `y` @@ -244,7 +262,7 @@ This is an incomplete overview of language features, by example. - A recursive set, equivalent to `{ x = "foo"; y = "foobar"; }` + A [recursive set](@docroot@/language/syntax.md#recursive-sets), equivalent to `{ x = "foo"; y = "foobar"; }`. @@ -260,7 +278,7 @@ This is an incomplete overview of language features, by example. - Lists with three elements. + [Lists](@docroot@/language/types.md#list) with three elements. @@ -344,7 +362,7 @@ This is an incomplete overview of language features, by example. - Attribute selection (evaluates to `1`) + [Attribute selection](@docroot@/language/types.md#attribute-set) (evaluates to `1`) @@ -356,7 +374,7 @@ This is an incomplete overview of language features, by example. - Attribute selection with default (evaluates to `3`) + [Attribute selection](@docroot@/language/types.md#attribute-set) with default (evaluates to `3`) @@ -392,7 +410,7 @@ This is an incomplete overview of language features, by example. - Conditional expression + [Conditional expression](@docroot@/language/syntax.md#conditionals). @@ -404,7 +422,7 @@ This is an incomplete overview of language features, by example. - Assertion check (evaluates to `"yes!"`). + [Assertion](@docroot@/language/syntax.md#assertions) check (evaluates to `"yes!"`). @@ -416,7 +434,7 @@ This is an incomplete overview of language features, by example. - Variable definition + Variable definition. See [`let`-expressions](@docroot@/language/syntax.md#let-expressions). @@ -428,14 +446,44 @@ This is an incomplete overview of language features, by example. - Add all attributes from the given set to the scope (evaluates to `1`) + Add all attributes from the given set to the scope (evaluates to `1`). + + See [`with`-expressions](@docroot@/language/syntax.md#with-expressions) for details and shadowing caveats. - *Functions (lambdas)* + `inherit pkgs src;` + + + + + Adds the variables to the current scope (attribute set or `let` binding). + Desugars to `pkgs = pkgs; src = src;`. + See [Inheriting attributes](@docroot@/language/syntax.md#inheriting-attributes). + + + + + + + `inherit (pkgs) lib stdenv;` + + + + + Adds the attributes, from the attribute set in parentheses, to the current scope (attribute set or `let` binding). + Desugars to `lib = pkgs.lib; stdenv = pkgs.stdenv;`. + See [Inheriting attributes](@docroot@/language/syntax.md#inheriting-attributes). + + + + + + + *[Functions](@docroot@/language/syntax.md#functions) (lambdas)* @@ -452,7 +500,7 @@ This is an incomplete overview of language features, by example. - A function that expects an integer and returns it increased by 1 + A [function](@docroot@/language/syntax.md#functions) that expects an integer and returns it increased by 1. @@ -464,7 +512,7 @@ This is an incomplete overview of language features, by example. - Curried function, equivalent to `x: (y: x + y)`. Can be used like a function that takes two arguments and returns their sum. + Curried [function](@docroot@/language/syntax.md#functions), equivalent to `x: (y: x + y)`. Can be used like a function that takes two arguments and returns their sum. @@ -476,7 +524,7 @@ This is an incomplete overview of language features, by example. - A function call (evaluates to 101) + A [function](@docroot@/language/syntax.md#functions) call (evaluates to 101) @@ -488,7 +536,7 @@ This is an incomplete overview of language features, by example. - A function bound to a variable and subsequently called by name (evaluates to 103) + A [function](@docroot@/language/syntax.md#functions) bound to a variable and subsequently called by name (evaluates to 103) @@ -500,7 +548,7 @@ This is an incomplete overview of language features, by example. - A function that expects a set with required attributes `x` and `y` and concatenates them + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attributes `x` and `y` and concatenates them @@ -512,7 +560,7 @@ This is an incomplete overview of language features, by example. - A function that expects a set with required attribute `x` and optional `y`, using `"bar"` as default value for `y` + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attribute `x` and optional `y`, using `"bar"` as default value for `y` @@ -524,7 +572,7 @@ This is an incomplete overview of language features, by example. - A function that expects a set with required attributes `x` and `y` and ignores any other attributes + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attributes `x` and `y` and ignores any other attributes @@ -538,7 +586,7 @@ This is an incomplete overview of language features, by example. - A function that expects a set with required attributes `x` and `y`, and binds the whole set to `args` + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attributes `x` and `y`, and binds the whole set to `args` @@ -562,7 +610,8 @@ This is an incomplete overview of language features, by example. - Load and return Nix expression in given file + Load and return Nix expression in given file. + See [import](@docroot@/language/builtins.md#builtins-import). @@ -574,7 +623,8 @@ This is an incomplete overview of language features, by example. - Apply a function to every element of a list (evaluates to `[ 2 4 6 ]`) + Apply a function to every element of a list (evaluates to `[ 2 4 6 ]`). + See [`map`](@docroot@/language/builtins.md#builtins-map). diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index 6fd66864b..e1c020781 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -26,12 +26,16 @@ | Logical conjunction (`AND`) | *bool* `&&` *bool* | left | 12 | | Logical disjunction (`OR`) | *bool* \|\| *bool* | left | 13 | | [Logical implication] | *bool* `->` *bool* | right | 14 | +| [Pipe operator] (experimental) | *expr* `\|>` *func* | left | 15 | +| [Pipe operator] (experimental) | *func* `<\|` *expr* | right | 15 | -[string]: ./values.md#type-string -[path]: ./values.md#type-path -[number]: ./values.md#type-number -[list]: ./values.md#list -[attribute set]: ./values.md#attribute-set +[string]: ./types.md#type-string +[path]: ./types.md#type-path +[number]: ./types.md#type-float +[list]: ./types.md#list +[attribute set]: ./types.md#attribute-set + + ## Attribute selection @@ -42,12 +46,6 @@ Select the attribute denoted by attribute path *attrpath* from [attribute set] *attrset*. If the attribute doesn’t exist, return the *expr* after `or` if provided, otherwise abort evaluation. -An attribute path is a dot-separated list of [attribute names](./values.md#attribute-set). - -> **Syntax** -> -> *attrpath* = *name* [ `.` *name* ]... - [Attribute selection]: #attribute-selection ## Has attribute @@ -61,7 +59,7 @@ The result is a [Boolean] value. See also: [`builtins.hasAttr`](@docroot@/language/builtins.md#builtins-hasAttr) -[Boolean]: ./values.md#type-boolean +[Boolean]: ./types.md#type-boolean [Has attribute]: #has-attribute @@ -128,8 +126,8 @@ The result is a string. > The file or directory at *path* must exist and is copied to the [store]. > The path appears in the result as the corresponding [store path]. -[store path]: ../glossary.md#gloss-store-path -[store]: ../glossary.md#gloss-store +[store path]: @docroot@/store/store-path.md +[store]: @docroot@/glossary.md#gloss-store [String and path concatenation]: #string-and-path-concatenation @@ -141,7 +139,7 @@ The result is a string. Update [attribute set] *attrset1* with names and values from *attrset2*. -The returned attribute set will have of all the attributes in *attrset1* and *attrset2*. +The returned attribute set will have all of the attributes in *attrset1* and *attrset2*. If an attribute name is present in both, the attribute value from the latter is taken. [Update]: #update @@ -172,7 +170,7 @@ All comparison operators are implemented in terms of `<`, and the following equi - Numbers are type-compatible, see [arithmetic] operators. - Floating point numbers only differ up to a limited precision. -[function]: ./constructs.md#functions +[function]: ./syntax.md#functions [Equality]: #equality @@ -182,3 +180,34 @@ Equivalent to `!`*b1* `||` *b2*. [Logical implication]: #logical-implication +## Pipe operators + +- *a* `|>` *b* is equivalent to *b* *a* +- *a* `<|` *b* is equivalent to *a* *b* + +> **Example** +> +> ``` +> nix-repl> 1 |> builtins.add 2 |> builtins.mul 3 +> 9 +> +> nix-repl> builtins.add 1 <| builtins.mul 2 <| 3 +> 7 +> ``` + +> **Warning** +> +> This syntax is part of an +> [experimental feature](@docroot@/development/experimental-features.md) +> and may change in future releases. +> +> To use this syntax, make sure the +> [`pipe-operators` experimental feature](@docroot@/development/experimental-features.md#xp-feature-pipe-operators) +> is enabled. +> For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): +> +> ``` +> extra-experimental-features = pipe-operators +> ``` + +[Pipe operator]: #pipe-operators diff --git a/doc/manual/src/language/scope.md b/doc/manual/src/language/scope.md new file mode 100644 index 000000000..9373324e2 --- /dev/null +++ b/doc/manual/src/language/scope.md @@ -0,0 +1,28 @@ +# Scoping rules + +A *scope* in the Nix language is a dictionary keyed by [name](./identifiers.md#names), mapping each name to an expression and a *definition type*. +The definition type is either *explicit* or *implicit*. +Each entry in this dictionary is a *definition*. + +Explicit definitions are created by the following expressions: +- [let-expressions](syntax.md#let-expressions) +- [recursive attribute set literals](syntax.md#recursive-sets) (`rec`) +- [function literals](syntax.md#functions) + +Implicit definitions are only created by [with-expressions](./syntax.md#with-expressions). + +Every expression is *enclosed* by a scope. +The outermost expression is enclosed by the [built-in, global scope](./builtins.md), which contains only explicit definitions. +The expressions listed above *extend* their enclosing scope by adding new definitions, or replacing existing ones with the same name. +An explicit definition can replace a definition of any type; an implicit definition can only replace another implicit definition. + +Each of the above expressions defines which of its subexpressions are enclosed by the extended scope. +In all other cases, the same scope that encloses an expression is the enclosing scope for its subexpressions. + +The Nix language is [statically scoped](https://en.wikipedia.org/wiki/Scope_(computer_science)#Lexical_scope); +the value of a variable is determined only by the variable's enclosing scope, and not by the dynamic context in which the variable is evaluated. + +> **Note** +> +> Expressions entered into the [Nix REPL](@docroot@/command-ref/new-cli/nix3-repl.md) are enclosed by a scope that can be extended by command line arguments or previous REPL commands. +> These ways of extending scope are not, strictly speaking, part of the Nix language. diff --git a/doc/manual/src/language/string-context.md b/doc/manual/src/language/string-context.md new file mode 100644 index 000000000..6a3482cfd --- /dev/null +++ b/doc/manual/src/language/string-context.md @@ -0,0 +1,134 @@ +# String context + +> **Note** +> +> This is an advanced topic. +> The Nix language is designed to be used without the programmer consciously dealing with string contexts or even knowing what they are. + +A string in the Nix language is not just a sequence of characters like strings in other languages. +It is actually a pair of a sequence of characters and a *string context*. +The string context is an (unordered) set of *string context elements*. + +The purpose of string contexts is to collect non-string values attached to strings via +[string concatenation](./operators.md#string-concatenation), +[string interpolation](./string-interpolation.md), +and similar operations. +The idea is that a user can combine together values to create a build instructions for derivations without manually keeping track of where they come from. +Then the Nix language implicitly does that bookkeeping to efficiently obtain the closure of derivation inputs. + +> **Note** +> +> String contexts are *not* explicitly manipulated in idiomatic Nix language code. + +String context elements come in different forms: + +- [deriving path]{#string-context-element-derived-path} + + A string context element of this type is a [deriving path](@docroot@/glossary.md#gloss-deriving-path). + They can be either of type [constant](#string-context-constant) or [output](#string-context-output), which correspond to the types of deriving paths. + + - [Constant string context elements]{#string-context-constant} + + > **Example** + > + > [`builtins.storePath`] creates a string with a single constant string context element: + > + > ```nix + > builtins.getContext (builtins.storePath "/nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10") + > ``` + > evaluates to + > ```nix + > { + > "/nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10" = { + > path = true; + > }; + > } + > ``` + + [deriving path]: @docroot@/glossary.md#gloss-deriving-path + [store path]: @docroot@/glossary.md#gloss-store-path + [`builtins.storePath`]: ./builtins.md#builtins-storePath + + - [Output string context elements]{#string-context-output} + + > **Example** + > + > The behavior of string contexts are best demonstrated with a built-in function that is still experimental: [`builtins.outputOf`]. + > This example will *not* work with stable Nix! + > + > ```nix + > builtins.getContext + > (builtins.outputOf + > (builtins.storePath "/nix/store/fvchh9cvcr7kdla6n860hshchsba305w-hello-2.12.drv") + > "out") + > ``` + > evaluates to + > ```nix + > { + > "/nix/store/fvchh9cvcr7kdla6n860hshchsba305w-hello-2.12.drv" = { + > outputs = [ "out" ]; + > }; + > } + > ``` + + [`builtins.outputOf`]: ./builtins.md#builtins-outputOf + +- [*derivation deep*]{#string-context-element-derivation-deep} + + *derivation deep* is an advanced feature intended to be used with the + [`exportReferencesGraph` derivation attribute](./advanced-attributes.html#adv-attr-exportReferencesGraph). + A *derivation deep* string context element is a derivation path, and refers to both its outputs and the entire build closure of that derivation: + all its outputs, all the other derivations the given derivation depends on, and all the outputs of those. + + > **Example** + > + > The best way to illustrate *derivation deep* string contexts is with [`builtins.addDrvOutputDependencies`]. + > Take a regular constant string context element pointing to a derivation, and transform it into a "Derivation deep" string context element. + > + > ```nix + > builtins.getContext + > (builtins.addDrvOutputDependencies + > (builtins.storePath "/nix/store/fvchh9cvcr7kdla6n860hshchsba305w-hello-2.12.drv")) + > ``` + > evaluates to + > ```nix + > { + > "/nix/store/fvchh9cvcr7kdla6n860hshchsba305w-hello-2.12.drv" = { + > allOutputs = true; + > }; + > } + > ``` + + [`builtins.addDrvOutputDependencies`]: ./builtins.md#builtins-addDrvOutputDependencies + [`builtins.unsafeDiscardOutputDependency`]: ./builtins.md#builtins-unsafeDiscardOutputDependency + +## Inspecting string contexts + +Most basically, [`builtins.hasContext`] will tell whether a string has a non-empty context. + +When more granular information is needed, [`builtins.getContext`] can be used. +It creates an [attribute set] representing the string context, which can be inspected as usual. + +[`builtins.hasContext`]: ./builtins.md#builtins-hasContext +[`builtins.getContext`]: ./builtins.md#builtins-getContext +[attribute set]: ./types.md#attribute-set + +## Clearing string contexts + +[`buitins.unsafeDiscardStringContext`](./builtins.md#builtins-unsafeDiscardStringContext) will make a copy of a string, but with an empty string context. +The returned string can be used in more ways, e.g. by operators that require the string context to be empty. +The requirement to explicitly discard the string context in such use cases helps ensure that string context elements are not lost by mistake. +The "unsafe" marker is only there to remind that Nix normally guarantees that dependencies are tracked, whereas the returned string has lost them. + +## Constructing string contexts + +[`builtins.appendContext`] will create a copy of a string, but with additional string context elements. +The context is specified explicitly by an [attribute set] in the format that [`builtins.hasContext`] produces. +A string with arbitrary contexts can be made like this: + +1. Create a string with the desired string context elements. + (The contents of the string do not matter.) +2. Dump its context with [`builtins.getContext`]. +3. Combine it with a base string and repeated [`builtins.appendContext`] calls. + +[`builtins.appendContext`]: ./builtins.md#builtins-appendContext diff --git a/doc/manual/src/language/string-interpolation.md b/doc/manual/src/language/string-interpolation.md index 7d81c2020..1778bdfa0 100644 --- a/doc/manual/src/language/string-interpolation.md +++ b/doc/manual/src/language/string-interpolation.md @@ -4,9 +4,9 @@ String interpolation is a language feature where a [string], [path], or [attribu Such a construct is called *interpolated string*, and the expression inside is an [interpolated expression](#interpolated-expression). -[string]: ./values.md#type-string -[path]: ./values.md#type-path -[attribute set]: ./values.md#attribute-set +[string]: ./types.md#type-string +[path]: ./types.md#type-path +[attribute set]: ./types.md#attribute-set ## Examples @@ -20,7 +20,7 @@ Rather than writing (where `freetype` is a [derivation]), you can instead write -[derivation]: ../glossary.md#gloss-derivation +[derivation]: @docroot@/glossary.md#gloss-derivation ```nix "--with-freetype2-library=${freetype}/lib" @@ -43,6 +43,47 @@ configureFlags = " Note that Nix expressions and strings can be arbitrarily nested; in this case the outer string contains various interpolated expressions that themselves contain strings (e.g., `"-thread"`), some of which in turn contain interpolated expressions (e.g., `${mesa}`). +To write a literal `${` in an regular string, escape it with a backslash (`\`). + +> **Example** +> +> ```nix +> "echo \${PATH}" +> ``` +> +> "echo ${PATH}" + +To write a literal `${` in an indented string, escape it with two single quotes (`''`). + +> **Example** +> +> ```nix +> '' +> echo ''${PATH} +> '' +> ``` +> +> "echo ${PATH}\n" + +`$${` can be written literally in any string. + +> **Example** +> +> In Make, `$` in file names or recipes is represented as `$$`, see [GNU `make`: Basics of Variable Reference](https://www.gnu.org/software/make/manual/html_node/Reference.html#Basics-of-Variable-References). +> This can be expressed directly in the Nix language strings: +> +> ```nix +> '' +> MAKEVAR = Hello +> all: +> @export BASHVAR=world; echo $(MAKEVAR) $${BASHVAR} +> '' +> ``` +> +> "MAKEVAR = Hello\nall:\n\t@export BASHVAR=world; echo $(MAKEVAR) $\${BASHVAR}\n" + +See the [documentation on strings][string] for details. + ### Path Rather than writing @@ -107,9 +148,9 @@ An expression that is interpolated must evaluate to one of the following: A string interpolates to itself. -A path in an interpolated expression is first copied into the Nix store, and the resulting string is the [store path] of the newly created [store object](../glossary.md#gloss-store-object). +A path in an interpolated expression is first copied into the Nix store, and the resulting string is the [store path] of the newly created [store object](@docroot@/store/store-object.md). -[store path]: ../glossary.md#gloss-store-path +[store path]: @docroot@/store/store-path.md > **Example** > diff --git a/doc/manual/src/language/syntax.md b/doc/manual/src/language/syntax.md new file mode 100644 index 000000000..6108bacd6 --- /dev/null +++ b/doc/manual/src/language/syntax.md @@ -0,0 +1,866 @@ +# Language Constructs + +This section covers syntax and semantics of the Nix language. + +## Basic Literals + +### String {#string-literal} + + *Strings* can be written in three ways. + + The most common way is to enclose the string between double quotes, e.g., `"foo bar"`. + Strings can span multiple lines. + The results of other expressions can be included into a string by enclosing them in `${ }`, a feature known as [string interpolation]. + + [string interpolation]: ./string-interpolation.md + + The following must be escaped to represent them within a string, by prefixing with a backslash (`\`): + + - Double quote (`"`) + + > **Example** + > + > ```nix + > "\"" + > ``` + > + > "\"" + + - Backslash (`\`) + + > **Example** + > + > ```nix + > "\\" + > ``` + > + > "\\" + + - Dollar sign followed by an opening curly bracket (`${`) – "dollar-curly" + + > **Example** + > + > ```nix + > "\${" + > ``` + > + > "\${" + + The newline, carriage return, and tab characters can be written as `\n`, `\r` and `\t`, respectively. + + A "double-dollar-curly" (`$${`) can be written literally. + + > **Example** + > + > ```nix + > "$${" + > ``` + > + > "$\${" + + String values are output on the terminal with Nix-specific escaping. + Strings written to files will contain the characters encoded by the escaping. + + The second way to write string literals is as an *indented string*, which is enclosed between pairs of *double single-quotes* (`''`), like so: + + ```nix + '' + This is the first line. + This is the second line. + This is the third line. + '' + ``` + + This kind of string literal intelligently strips indentation from + the start of each line. To be precise, it strips from each line a + number of spaces equal to the minimal indentation of the string as a + whole (disregarding the indentation of empty lines). For instance, + the first and second line are indented two spaces, while the third + line is indented four spaces. Thus, two spaces are stripped from + each line, so the resulting string is + + ```nix + "This is the first line.\nThis is the second line.\n This is the third line.\n" + ``` + + > **Note** + > + > Whitespace and newline following the opening `''` is ignored if there is no non-whitespace text on the initial line. + + > **Warning** + > + > Prefixed tab characters are not stripped. + > + > > **Example** + > > + > > The following indented string is prefixed with tabs: + > > + > > '' + > > all: + > > @echo hello + > > '' + > > + > > "\tall:\n\t\t@echo hello\n" + + Indented strings support [string interpolation]. + + The following must be escaped to represent them in an indented string: + + - `$` is escaped by prefixing it with two single quotes (`''`) + + > **Example** + > + > ```nix + > '' + > ''$ + > '' + > ``` + > + > "$\n" + + - `''` is escaped by prefixing it with one single quote (`'`) + + > **Example** + > + > ```nix + > '' + > ''' + > '' + > ``` + > + > "''\n" + + These special characters are escaped as follows: + - Linefeed (`\n`): `''\n` + - Carriage return (`\r`): `''\r` + - Tab (`\t`): `''\t` + + `''\` escapes any other character. + + A "double-dollar-curly" (`$${`) can be written literally. + + > **Example** + > + > ```nix + > '' + > $${ + > '' + > ``` + > + > "$\${\n" + + Indented strings are primarily useful in that they allow multi-line + string literals to follow the indentation of the enclosing Nix + expression, and that less escaping is typically necessary for + strings representing languages such as shell scripts and + configuration files because `''` is much less common than `"`. + Example: + + ```nix + stdenv.mkDerivation { + ... + postInstall = + '' + mkdir $out/bin $out/etc + cp foo $out/bin + echo "Hello World" > $out/etc/foo.conf + ${if enableBar then "cp bar $out/bin" else ""} + ''; + ... + } + ``` + + Finally, as a convenience, *URIs* as defined in appendix B of + [RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as + is*, without quotes. For instance, the string + `"http://example.org/foo.tar.bz2"` can also be written as + `http://example.org/foo.tar.bz2`. + +### Number {#number-literal} + + + + Numbers, which can be *integers* (like `123`) or *floating point* + (like `123.43` or `.27e13`). + + See [arithmetic] and [comparison] operators for semantics. + + [arithmetic]: ./operators.md#arithmetic + [comparison]: ./operators.md#comparison + +### Path {#path-literal} + + *Paths* can be expressed by path literals such as `./builder.sh`. + + A path literal must contain at least one slash to be recognised as such. + For instance, `builder.sh` is not a path: + it's parsed as an expression that selects the attribute `sh` from the variable `builder`. + + Path literals are resolved relative to their [base directory](@docroot@/glossary.md#gloss-base-directory). + Path literals may also refer to absolute paths by starting with a slash. + + > **Note** + > + > Absolute paths make expressions less portable. + > In the case where a function translates a path literal into an absolute path string for a configuration file, it is recommended to write a string literal instead. + > This avoids some confusion about whether files at that location will be used during evaluation. + > It also avoids unintentional situations where some function might try to copy everything at the location into the store. + + If the first component of a path is a `~`, it is interpreted such that the rest of the path were relative to the user's home directory. + For example, `~/foo` would be equivalent to `/home/edolstra/foo` for a user whose home directory is `/home/edolstra`. + Path literals that start with `~` are not allowed in [pure](@docroot@/command-ref/conf-file.md#conf-pure-eval) evaluation. + + Path literals can also include [string interpolation], besides being [interpolated into other expressions]. + + [interpolated into other expressions]: ./string-interpolation.md#interpolated-expressions + + At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path. + + `a.${foo}/b.${bar}` is a syntactically valid number division operation. + `./a.${foo}/b.${bar}` is a path. + + [Lookup path](./constructs/lookup-path.md) literals such as `` also resolve to path values. + +## List {#list-literal} + +Lists are formed by enclosing a whitespace-separated list of values +between square brackets. For example, + +```nix +[ 123 ./foo.nix "abc" (f { x = y; }) ] +``` + +defines a list of four elements, the last being the result of a call to +the function `f`. Note that function calls have to be enclosed in +parentheses. If they had been omitted, e.g., + +```nix +[ 123 ./foo.nix "abc" f { x = y; } ] +``` + +the result would be a list of five elements, the fourth one being a +function and the fifth being a set. + +Note that lists are only lazy in values, and they are strict in length. + +Elements in a list can be accessed using [`builtins.elemAt`](./builtins.md#builtins-elemAt). + +## Attribute Set {#attrs-literal} + +An attribute set is a collection of name-value-pairs called *attributes*. + +Attribute sets are written enclosed in curly brackets (`{ }`). +Attribute names and attribute values are separated by an equal sign (`=`). +Each value can be an arbitrary expression, terminated by a semicolon (`;`) + +An attribute name is a string without context, and is denoted by a [name] (an [identifier](./identifiers.md#identifiers) or [string literal](#string-literal)). + +[name]: ./identifiers.md#names + +> **Syntax** +> +> *attrset* → `{` { *name* `=` *expr* `;` } `}` + +Attributes can appear in any order. +An attribute name may only occur once in each attribute set. + +> **Example** +> +> This defines an attribute set with attributes named: +> - `x` with the value `123`, an integer +> - `text` with the value `"Hello"`, a string +> - `y` where the value is the result of applying the function `f` to the attribute set `{ bla = 456; }` +> +> ```nix +> { +> x = 123; +> text = "Hello"; +> y = f { bla = 456; }; +> } +> ``` + +Attributes in nested attribute sets can be written using *attribute paths*. + +> **Syntax** +> +> *attrset* → `{` { *attrpath* `=` *expr* `;` } `}` + +An attribute path is a dot-separated list of [names][name]. + +> **Syntax** +> +> *attrpath* = *name* { `.` *name* } + + + +> **Example** +> +> ```nix +> { a.b.c = 1; a.b.d = 2; } +> ``` +> +> { +> a = { +> b = { +> c = 1; +> d = 2; +> }; +> }; +> } + +Attribute names can also be set implicitly by using the [`inherit` keyword](#inheriting-attributes). + +> **Example** +> +> ```nix +> { inherit (builtins) true; } +> ``` +> +> { true = true; } + +Attributes can be accessed with the [`.` operator](./operators.md#attribute-selection). + +Example: + +```nix +{ a = "Foo"; b = "Bar"; }.a +``` + +This evaluates to `"Foo"`. + +It is possible to provide a default value in an attribute selection using the `or` keyword. + +Example: + +```nix +{ a = "Foo"; b = "Bar"; }.c or "Xyzzy" +``` + +```nix +{ a = "Foo"; b = "Bar"; }.c.d.e.f.g or "Xyzzy" +``` + +will both evaluate to `"Xyzzy"` because there is no `c` attribute in the set. + +You can use arbitrary double-quoted strings as attribute names: + +```nix +{ "$!@#?" = 123; }."$!@#?" +``` + +```nix +let bar = "bar"; in +{ "foo ${bar}" = 123; }."foo ${bar}" +``` + +Both will evaluate to `123`. + +Attribute names support [string interpolation]: + +```nix +let bar = "foo"; in +{ foo = 123; }.${bar} +``` + +```nix +let bar = "foo"; in +{ ${bar} = 123; }.foo +``` + +Both will evaluate to `123`. + +In the special case where an attribute name inside of a set declaration +evaluates to `null` (which is normally an error, as `null` cannot be coerced to +a string), that attribute is simply not added to the set: + +```nix +{ ${if foo then "bar" else null} = true; } +``` + +This will evaluate to `{}` if `foo` evaluates to `false`. + +A set that has a `__functor` attribute whose value is callable (i.e. is +itself a function or a set with a `__functor` attribute whose value is +callable) can be applied as if it were a function, with the set itself +passed in first , e.g., + +```nix +let add = { __functor = self: x: x + self.x; }; + inc = add // { x = 1; }; +in inc 1 +``` + +evaluates to `2`. This can be used to attach metadata to a function +without the caller needing to treat it specially, or to implement a form +of object-oriented programming, for example. + +## Recursive sets + +Recursive sets are like normal [attribute sets](./types.md#attribute-set), but the attributes can refer to each other. + +> *rec-attrset* = `rec {` [ *name* `=` *expr* `;` `]`... `}` + +Example: + +```nix +rec { + x = y; + y = 123; +}.x +``` + +This evaluates to `123`. + +Note that without `rec` the binding `x = y;` would +refer to the variable `y` in the surrounding scope, if one exists, and +would be invalid if no such variable exists. That is, in a normal +(non-recursive) set, attributes are not added to the lexical scope; in a +recursive set, they are. + +Recursive sets of course introduce the danger of infinite recursion. For +example, the expression + +```nix +rec { + x = y; + y = x; +}.x +``` + +will crash with an `infinite recursion encountered` error message. + +## Let-expressions + +A let-expression allows you to define local variables for an expression. + +> *let-in* = `let` [ *identifier* = *expr* ]... `in` *expr* + +Example: + +```nix +let + x = "foo"; + y = "bar"; +in x + y +``` + +This evaluates to `"foobar"`. + +## Inheriting attributes + +When defining an [attribute set](./types.md#attribute-set) or in a [let-expression](#let-expressions) it is often convenient to copy variables from the surrounding lexical scope (e.g., when you want to propagate attributes). +This can be shortened using the `inherit` keyword. + +Example: + +```nix +let x = 123; in +{ + inherit x; + y = 456; +} +``` + +is equivalent to + +```nix +let x = 123; in +{ + x = x; + y = 456; +} +``` + +and both evaluate to `{ x = 123; y = 456; }`. + +> **Note** +> +> This works because `x` is added to the lexical scope by the `let` construct. + +It is also possible to inherit attributes from another attribute set. + +Example: + +In this fragment from `all-packages.nix`, + +```nix +graphviz = (import ../tools/graphics/graphviz) { + inherit fetchurl stdenv libpng libjpeg expat x11 yacc; + inherit (xorg) libXaw; +}; + +xorg = { + libX11 = ...; + libXaw = ...; + ... +} + +libpng = ...; +libjpg = ...; +... +``` + +the set used in the function call to the function defined in +`../tools/graphics/graphviz` inherits a number of variables from the +surrounding scope (`fetchurl` ... `yacc`), but also inherits `libXaw` +(the X Athena Widgets) from the `xorg` set. + +Summarizing the fragment + +```nix +... +inherit x y z; +inherit (src-set) a b c; +... +``` + +is equivalent to + +```nix +... +x = x; y = y; z = z; +a = src-set.a; b = src-set.b; c = src-set.c; +... +``` + +when used while defining local variables in a let-expression or while +defining a set. + +In a `let` expression, `inherit` can be used to selectively bring specific attributes of a set into scope. For example + + +```nix +let + x = { a = 1; b = 2; }; + inherit (builtins) attrNames; +in +{ + names = attrNames x; +} +``` + +is equivalent to + +```nix +let + x = { a = 1; b = 2; }; +in +{ + names = builtins.attrNames x; +} +``` + +both evaluate to `{ names = [ "a" "b" ]; }`. + +## Functions + +Functions have the following form: + +```nix +pattern: body +``` + +The pattern specifies what the argument of the function must look like, +and binds variables in the body to (parts of) the argument. There are +three kinds of patterns: + + - If a pattern is a single identifier, then the function matches any + argument. Example: + + ```nix + let negate = x: !x; + concat = x: y: x + y; + in if negate true then concat "foo" "bar" else "" + ``` + + Note that `concat` is a function that takes one argument and returns + a function that takes another argument. This allows partial + parameterisation (i.e., only filling some of the arguments of a + function); e.g., + + ```nix + map (concat "foo") [ "bar" "bla" "abc" ] + ``` + + evaluates to `[ "foobar" "foobla" "fooabc" ]`. + + - A *set pattern* of the form `{ name1, name2, …, nameN }` matches a + set containing the listed attributes, and binds the values of those + attributes to variables in the function body. For example, the + function + + ```nix + { x, y, z }: z + y + x + ``` + + can only be called with a set containing exactly the attributes `x`, + `y` and `z`. No other attributes are allowed. If you want to allow + additional arguments, you can use an ellipsis (`...`): + + ```nix + { x, y, z, ... }: z + y + x + ``` + + This works on any set that contains at least the three named + attributes. + + It is possible to provide *default values* for attributes, in + which case they are allowed to be missing. A default value is + specified by writing `name ? e`, where *e* is an arbitrary + expression. For example, + + ```nix + { x, y ? "foo", z ? "bar" }: z + y + x + ``` + + specifies a function that only requires an attribute named `x`, but + optionally accepts `y` and `z`. + + - An `@`-pattern provides a means of referring to the whole value + being matched: + + ```nix + args@{ x, y, z, ... }: z + y + x + args.a + ``` + + but can also be written as: + + ```nix + { x, y, z, ... } @ args: z + y + x + args.a + ``` + + Here `args` is bound to the argument *as passed*, which is further + matched against the pattern `{ x, y, z, ... }`. + The `@`-pattern makes mainly sense with an ellipsis(`...`) as + you can access attribute names as `a`, using `args.a`, which was + given as an additional attribute to the function. + + > **Warning** + > + > `args@` binds the name `args` to the attribute set that is passed to the function. + > In particular, `args` does *not* include any default values specified with `?` in the function's set pattern. + > + > For instance + > + > ```nix + > let + > f = args@{ a ? 23, ... }: [ a args ]; + > in + > f {} + > ``` + > + > is equivalent to + > + > ```nix + > let + > f = args @ { ... }: [ (args.a or 23) args ]; + > in + > f {} + > ``` + > + > and both expressions will evaluate to: + > + > ```nix + > [ 23 {} ] + > ``` + +Note that functions do not have names. If you want to give them a name, +you can bind them to an attribute, e.g., + +```nix +let concat = { x, y }: x + y; +in concat { x = "foo"; y = "bar"; } +``` + +## Conditionals + +Conditionals look like this: + +```nix +if e1 then e2 else e3 +``` + +where *e1* is an expression that should evaluate to a Boolean value +(`true` or `false`). + +## Assertions + +Assertions are generally used to check that certain requirements on or +between features and dependencies hold. They look like this: + +```nix +assert e1; e2 +``` + +where *e1* is an expression that should evaluate to a Boolean value. If +it evaluates to `true`, *e2* is returned; otherwise expression +evaluation is aborted and a backtrace is printed. + +Here is a Nix expression for the Subversion package that shows how +assertions can be used:. + +```nix +{ localServer ? false +, httpServer ? false +, sslSupport ? false +, pythonBindings ? false +, javaSwigBindings ? false +, javahlBindings ? false +, stdenv, fetchurl +, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null +}: + +assert localServer -> db4 != null; â‘ +assert httpServer -> httpd != null && httpd.expat == expat; â‘¡ +assert sslSupport -> openssl != null && (httpServer -> httpd.openssl == openssl); â‘¢ +assert pythonBindings -> swig != null && swig.pythonSupport; +assert javaSwigBindings -> swig != null && swig.javaSupport; +assert javahlBindings -> j2sdk != null; + +stdenv.mkDerivation { + name = "subversion-1.1.1"; + ... + openssl = if sslSupport then openssl else null; â‘£ + ... +} +``` + +The points of interest are: + +1. This assertion states that if Subversion is to have support for + local repositories, then Berkeley DB is needed. So if the Subversion + function is called with the `localServer` argument set to `true` but + the `db4` argument set to `null`, then the evaluation fails. + + Note that `->` is the [logical + implication](https://en.wikipedia.org/wiki/Truth_table#Logical_implication) + Boolean operation. + +2. This is a more subtle condition: if Subversion is built with Apache + (`httpServer`) support, then the Expat library (an XML library) used + by Subversion should be same as the one used by Apache. This is + because in this configuration Subversion code ends up being linked + with Apache code, and if the Expat libraries do not match, a build- + or runtime link error or incompatibility might occur. + +3. This assertion says that in order for Subversion to have SSL support + (so that it can access `https` URLs), an OpenSSL library must be + passed. Additionally, it says that *if* Apache support is enabled, + then Apache's OpenSSL should match Subversion's. (Note that if + Apache support is not enabled, we don't care about Apache's + OpenSSL.) + +4. The conditional here is not really related to assertions, but is + worth pointing out: it ensures that if SSL support is disabled, then + the Subversion derivation is not dependent on OpenSSL, even if a + non-`null` value was passed. This prevents an unnecessary rebuild of + Subversion if OpenSSL changes. + +## With-expressions + +A *with-expression*, + +```nix +with e1; e2 +``` + +introduces the set *e1* into the lexical scope of the expression *e2*. +For instance, + +```nix +let as = { x = "foo"; y = "bar"; }; +in with as; x + y +``` + +evaluates to `"foobar"` since the `with` adds the `x` and `y` attributes +of `as` to the lexical scope in the expression `x + y`. The most common +use of `with` is in conjunction with the `import` function. E.g., + +```nix +with (import ./definitions.nix); ... +``` + +makes all attributes defined in the file `definitions.nix` available as +if they were defined locally in a `let`-expression. + +The bindings introduced by `with` do not shadow bindings introduced by +other means, e.g. + +```nix +let a = 3; in with { a = 1; }; let a = 4; in with { a = 2; }; ... +``` + +establishes the same scope as + +```nix +let a = 1; in let a = 2; in let a = 3; in let a = 4; in ... +``` + +Variables coming from outer `with` expressions *are* shadowed: + +```nix +with { a = "outer"; }; +with { a = "inner"; }; +a +``` + +Does evaluate to `"inner"`. + +## Comments + +- Inline comments start with `#` and run until the end of the line. + + > **Example** + > + > ```nix + > # A number + > 2 # Equals 1 + 1 + > ``` + > + > ```console + > 2 + > ``` + +- Block comments start with `/*` and run until the next occurrence of `*/`. + + > **Example** + > + > ```nix + > /* + > Block comments + > can span multiple lines. + > */ "hello" + > ``` + > + > ```console + > "hello" + > ``` + + This means that block comments cannot be nested. + + > **Example** + > + > ```nix + > /* /* nope */ */ 1 + > ``` + > + > ```console + > error: syntax error, unexpected '*' + > + > at «string»:1:15: + > + > 1| /* /* nope */ * + > | ^ + > ``` + + Consider escaping nested comments and unescaping them in post-processing. + + > **Example** + > + > ```nix + > /* /* nested *\/ */ 1 + > ``` + > + > ```console + > 1 + > ``` diff --git a/doc/manual/src/language/types.md b/doc/manual/src/language/types.md new file mode 100644 index 000000000..229756e6b --- /dev/null +++ b/doc/manual/src/language/types.md @@ -0,0 +1,120 @@ +# Data Types + +Every value in the Nix language has one of the following types: + +* [Integer](#type-int) +* [Float](#type-float) +* [Boolean](#type-bool) +* [String](#type-string) +* [Path](#type-path) +* [Null](#type-null) +* [Attribute set](#type-attrs) +* [List](#type-list) +* [Function](#type-function) +* [External](#type-external) + +## Primitives + +### Integer {#type-int} + +An _integer_ in the Nix language is a signed 64-bit integer. + +Non-negative integers can be expressed as [integer literals](syntax.md#number-literal). +Negative integers are created with the [arithmetic negation operator](./operators.md#arithmetic). +The function [`builtins.isInt`](builtins.md#builtins-isInt) can be used to determine if a value is an integer. + +### Float {#type-float} + +A _float_ in the Nix language is a 64-bit [IEEE 754](https://en.wikipedia.org/wiki/IEEE_754) floating-point number. + +Most non-negative floats can be expressed as [float literals](syntax.md#number-literal). +Negative floats are created with the [arithmetic negation operator](./operators.md#arithmetic). +The function [`builtins.isFloat`](builtins.md#builtins-isFloat) can be used to determine if a value is a float. + +### Boolean {#type-bool} + +A _boolean_ in the Nix language is one of _true_ or _false_. + + + +These values are available as attributes of [`builtins`](builtins.md#builtins-builtins) as [`builtins.true`](builtins.md#builtins-true) and [`builtins.false`](builtins.md#builtins-false). +The function [`builtins.isBool`](builtins.md#builtins-isBool) can be used to determine if a value is a boolean. + +### String {#type-string} + +A _string_ in the Nix language is an immutable, finite-length sequence of bytes, along with a [string context](string-context.md). +Nix does not assume or support working natively with character encodings. + +String values without string context can be expressed as [string literals](syntax.md#string-literal). +The function [`builtins.isString`](builtins.md#builtins-isString) can be used to determine if a value is a string. + +### Path {#type-path} + +A _path_ in the Nix language is an immutable, finite-length sequence of bytes starting with `/`, representing a POSIX-style, canonical file system path. +Path values are distinct from string values, even if they contain the same sequence of bytes. +Operations that produce paths will simplify the result as the standard C function [`realpath`] would, except that there is no symbolic link resolution. + +[`realpath`]: https://pubs.opengroup.org/onlinepubs/9699919799/functions/realpath.html + +Paths are suitable for referring to local files, and are often preferable over strings. +- Path values do not contain trailing or duplicate slashes, `.`, or `..`. +- Relative path literals are automatically resolved relative to their [base directory]. +- Tooling can recognize path literals and provide additional features, such as autocompletion, refactoring automation and jump-to-file. + +[base directory]: @docroot@/glossary.md#gloss-base-directory + +A file is not required to exist at a given path in order for that path value to be valid, but a path that is converted to a string with [string interpolation] or [string-and-path concatenation] must resolve to a readable file or directory which will be copied into the Nix store. +For instance, evaluating `"${./foo.txt}"` will cause `foo.txt` from the same directory to be copied into the Nix store and result in the string `"/nix/store/-foo.txt"`. +Operations such as [`import`] can also expect a path to resolve to a readable file or directory. + +[string interpolation]: string-interpolation.md#interpolated-expression +[string-and-path concatenation]: operators.md#string-and-path-concatenation +[`import`]: builtins.md#builtins-import + +> **Note** +> +> The Nix language assumes that all input files will remain _unchanged_ while evaluating a Nix expression. +> For example, assume you used a file path in an interpolated string during a `nix repl` session. +> Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new [store path], since Nix might not re-read the file contents. +> Use `:r` to reset the repl as needed. + +[store path]: @docroot@/store/store-path.md + +Path values can be expressed as [path literals](syntax.md#path-literal). +The function [`builtins.isPath`](builtins.md#builtins-isPath) can be used to determine if a value is a path. + +### Null {#type-null} + +There is a single value of type _null_ in the Nix language. + + + +This value is available as an attribute on the [`builtins`](builtins.md#builtins-builtins) attribute set as [`builtins.null`](builtins.md#builtins-null). + +## Compound values + +### Attribute set {#type-attrs} + + + +An attribute set can be constructed with an [attribute set literal](syntax.md#attrs-literal). +The function [`builtins.isAttrs`](builtins.md#builtins-isAttrs) can be used to determine if a value is an attribute set. + +### List {#type-list} + + + +A list can be constructed with a [list literal](syntax.md#list-literal). +The function [`builtins.isList`](builtins.md#builtins-isList) can be used to determine if a value is a list. + +## Function {#type-function} + + + +A function can be constructed with a [function expression](syntax.md#functions). +The function [`builtins.isFunction`](builtins.md#builtins-isFunction) can be used to determine if a value is a function. + +## External {#type-external} + +An _external_ value is an opaque value created by a Nix [plugin](../command-ref/conf-file.md#conf-plugin-files). +Such a value can be substituted in Nix expressions but only created and used by plugin code. diff --git a/doc/manual/src/language/values.md b/doc/manual/src/language/values.md deleted file mode 100644 index 74ffc7070..000000000 --- a/doc/manual/src/language/values.md +++ /dev/null @@ -1,269 +0,0 @@ -# Data Types - -## Primitives - -- String - - *Strings* can be written in three ways. - - The most common way is to enclose the string between double quotes, - e.g., `"foo bar"`. Strings can span multiple lines. The special - characters `"` and `\` and the character sequence `${` must be - escaped by prefixing them with a backslash (`\`). Newlines, carriage - returns and tabs can be written as `\n`, `\r` and `\t`, - respectively. - - You can include the results of other expressions into a string by enclosing them in `${ }`, a feature known as [string interpolation]. - - [string interpolation]: ./string-interpolation.md - - The second way to write string literals is as an *indented string*, - which is enclosed between pairs of *double single-quotes*, like so: - - ```nix - '' - This is the first line. - This is the second line. - This is the third line. - '' - ``` - - This kind of string literal intelligently strips indentation from - the start of each line. To be precise, it strips from each line a - number of spaces equal to the minimal indentation of the string as a - whole (disregarding the indentation of empty lines). For instance, - the first and second line are indented two spaces, while the third - line is indented four spaces. Thus, two spaces are stripped from - each line, so the resulting string is - - ```nix - "This is the first line.\nThis is the second line.\n This is the third line.\n" - ``` - - Note that the whitespace and newline following the opening `''` is - ignored if there is no non-whitespace text on the initial line. - - Indented strings support [string interpolation]. - - Since `${` and `''` have special meaning in indented strings, you - need a way to quote them. `$` can be escaped by prefixing it with - `''` (that is, two single quotes), i.e., `''$`. `''` can be escaped - by prefixing it with `'`, i.e., `'''`. `$` removes any special - meaning from the following `$`. Linefeed, carriage-return and tab - characters can be written as `''\n`, `''\r`, `''\t`, and `''\` - escapes any other character. - - Indented strings are primarily useful in that they allow multi-line - string literals to follow the indentation of the enclosing Nix - expression, and that less escaping is typically necessary for - strings representing languages such as shell scripts and - configuration files because `''` is much less common than `"`. - Example: - - ```nix - stdenv.mkDerivation { - ... - postInstall = - '' - mkdir $out/bin $out/etc - cp foo $out/bin - echo "Hello World" > $out/etc/foo.conf - ${if enableBar then "cp bar $out/bin" else ""} - ''; - ... - } - ``` - - Finally, as a convenience, *URIs* as defined in appendix B of - [RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as - is*, without quotes. For instance, the string - `"http://example.org/foo.tar.bz2"` can also be written as - `http://example.org/foo.tar.bz2`. - -- Number - - Numbers, which can be *integers* (like `123`) or *floating point* - (like `123.43` or `.27e13`). - - See [arithmetic] and [comparison] operators for semantics. - - [arithmetic]: ./operators.md#arithmetic - [comparison]: ./operators.md#comparison - -- Path - - *Paths*, e.g., `/bin/sh` or `./builder.sh`. A path must contain at - least one slash to be recognised as such. For instance, `builder.sh` - is not a path: it's parsed as an expression that selects the - attribute `sh` from the variable `builder`. If the file name is - relative, i.e., if it does not begin with a slash, it is made - absolute at parse time relative to the directory of the Nix - expression that contained it. For instance, if a Nix expression in - `/foo/bar/bla.nix` refers to `../xyzzy/fnord.nix`, the absolute path - is `/foo/xyzzy/fnord.nix`. - - If the first component of a path is a `~`, it is interpreted as if - the rest of the path were relative to the user's home directory. - e.g. `~/foo` would be equivalent to `/home/edolstra/foo` for a user - whose home directory is `/home/edolstra`. - - For instance, evaluating `"${./foo.txt}"` will cause `foo.txt` in the current directory to be copied into the Nix store and result in the string `"/nix/store/-foo.txt"`. - - Note that the Nix language assumes that all input files will remain _unchanged_ while evaluating a Nix expression. - For example, assume you used a file path in an interpolated string during a `nix repl` session. - Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new [store path], since Nix might not re-read the file contents. - - [store path]: ../glossary.md#gloss-store-path - - Paths can include [string interpolation] and can themselves be [interpolated in other expressions]. - - [interpolated in other expressions]: ./string-interpolation.md#interpolated-expressions - - At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path. - - `a.${foo}/b.${bar}` is a syntactically valid division operation. - `./a.${foo}/b.${bar}` is a path. - - [Lookup paths](./constructs/lookup-path.md) such as `` resolve to path values. - -- Boolean - - *Booleans* with values `true` and `false`. - -- Null - - The null value, denoted as `null`. - -## List - -Lists are formed by enclosing a whitespace-separated list of values -between square brackets. For example, - -```nix -[ 123 ./foo.nix "abc" (f { x = y; }) ] -``` - -defines a list of four elements, the last being the result of a call to -the function `f`. Note that function calls have to be enclosed in -parentheses. If they had been omitted, e.g., - -```nix -[ 123 ./foo.nix "abc" f { x = y; } ] -``` - -the result would be a list of five elements, the fourth one being a -function and the fifth being a set. - -Note that lists are only lazy in values, and they are strict in length. - -Elements in a list can be accessed using [`builtins.elemAt`](./builtins.md#builtins-elemAt). - -## Attribute Set - -An attribute set is a collection of name-value-pairs (called *attributes*) enclosed in curly brackets (`{ }`). - -An attribute name can be an identifier or a [string](#string). -An identifier must start with a letter (`a-z`, `A-Z`) or underscore (`_`), and can otherwise contain letters (`a-z`, `A-Z`), numbers (`0-9`), underscores (`_`), apostrophes (`'`), or dashes (`-`). - -> **Syntax** -> -> *name* = *identifier* | *string* \ -> *identifier* ~ `[a-zA-Z_][a-zA-Z0-9_'-]*` - -Names and values are separated by an equal sign (`=`). -Each value is an arbitrary expression terminated by a semicolon (`;`). - -> **Syntax** -> -> *attrset* = `{` [ *name* `=` *expr* `;` ]... `}` - -Attributes can appear in any order. -An attribute name may only occur once. - -Example: - -```nix -{ - x = 123; - text = "Hello"; - y = f { bla = 456; }; -} -``` - -This defines a set with attributes named `x`, `text`, `y`. - -Attributes can be accessed with the [`.` operator](./operators.md#attribute-selection). - -Example: - -```nix -{ a = "Foo"; b = "Bar"; }.a -``` - -This evaluates to `"Foo"`. - -It is possible to provide a default value in an attribute selection using the `or` keyword. - -Example: - -```nix -{ a = "Foo"; b = "Bar"; }.c or "Xyzzy" -``` - -```nix -{ a = "Foo"; b = "Bar"; }.c.d.e.f.g or "Xyzzy" -``` - -will both evaluate to `"Xyzzy"` because there is no `c` attribute in the set. - -You can use arbitrary double-quoted strings as attribute names: - -```nix -{ "$!@#?" = 123; }."$!@#?" -``` - -```nix -let bar = "bar"; in -{ "foo ${bar}" = 123; }."foo ${bar}" -``` - -Both will evaluate to `123`. - -Attribute names support [string interpolation]: - -```nix -let bar = "foo"; in -{ foo = 123; }.${bar} -``` - -```nix -let bar = "foo"; in -{ ${bar} = 123; }.foo -``` - -Both will evaluate to `123`. - -In the special case where an attribute name inside of a set declaration -evaluates to `null` (which is normally an error, as `null` cannot be coerced to -a string), that attribute is simply not added to the set: - -```nix -{ ${if foo then "bar" else null} = true; } -``` - -This will evaluate to `{}` if `foo` evaluates to `false`. - -A set that has a `__functor` attribute whose value is callable (i.e. is -itself a function or a set with a `__functor` attribute whose value is -callable) can be applied as if it were a function, with the set itself -passed in first , e.g., - -```nix -let add = { __functor = self: x: x + self.x; }; - inc = add // { x = 1; }; -in inc 1 -``` - -evaluates to `2`. This can be used to attach metadata to a function -without the caller needing to treat it specially, or to implement a form -of object-oriented programming, for example. diff --git a/doc/manual/src/language/variables.md b/doc/manual/src/language/variables.md new file mode 100644 index 000000000..af6aff8a2 --- /dev/null +++ b/doc/manual/src/language/variables.md @@ -0,0 +1,10 @@ +# Variables + +A *variable* is an [identifier](identifiers.md) used as an expression. + +> **Syntax** +> +> *expression* → *identifier* + +A variable must have the same name as a definition in the [scope](./scope.md) that encloses it. +The value of a variable is the value of the corresponding expression in the enclosing scope. diff --git a/doc/manual/src/package-management/copy-closure.md b/doc/manual/src/package-management/copy-closure.md deleted file mode 100644 index 14326298b..000000000 --- a/doc/manual/src/package-management/copy-closure.md +++ /dev/null @@ -1,34 +0,0 @@ -# Copying Closures via SSH - -The command `nix-copy-closure` copies a Nix store path along with all -its dependencies to or from another machine via the SSH protocol. It -doesn’t copy store paths that are already present on the target machine. -For example, the following command copies Firefox with all its -dependencies: - - $ nix-copy-closure --to alice@itchy.example.org $(type -p firefox) - -See the [manpage for `nix-copy-closure`](../command-ref/nix-copy-closure.md) for details. - -With `nix-store ---export` and `nix-store --import` you can write the closure of a store -path (that is, the path and all its dependencies) to a file, and then -unpack that file into another Nix store. For example, - - $ nix-store --export $(nix-store --query --requisites $(type -p firefox)) > firefox.closure - -writes the closure of Firefox to a file. You can then copy this file to -another machine and install the closure: - - $ nix-store --import < firefox.closure - -Any store paths in the closure that are already present in the target -store are ignored. It is also possible to pipe the export into another -command, e.g. to copy and install a closure directly to/on another -machine: - - $ nix-store --export $(nix-store --query --requisites $(type -p firefox)) | bzip2 | \ - ssh alice@itchy.example.org "bunzip2 | nix-store --import" - -However, `nix-copy-closure` is generally more efficient because it only -copies paths that are not already present in the target Nix store. diff --git a/doc/manual/src/protocols/derivation-aterm.md b/doc/manual/src/protocols/derivation-aterm.md index e58b602a3..1ba757ae0 100644 --- a/doc/manual/src/protocols/derivation-aterm.md +++ b/doc/manual/src/protocols/derivation-aterm.md @@ -14,6 +14,6 @@ Derivations are serialised in one of the following formats: DrvWithVersion(, ...) ``` - The only `version-string`s that are in use today are for [experimental features](@docroot@/contributing/experimental-features.md): + The only `version-string`s that are in use today are for [experimental features](@docroot@/development/experimental-features.md): - - `"xp-dyn-drv"` for the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. + - `"xp-dyn-drv"` for the [`dynamic-derivations`](@docroot@/development/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. diff --git a/doc/manual/src/protocols/json/derivation.md b/doc/manual/src/protocols/json/derivation.md index 649d543cc..2f85340d6 100644 --- a/doc/manual/src/protocols/json/derivation.md +++ b/doc/manual/src/protocols/json/derivation.md @@ -3,7 +3,7 @@ > **Warning** > > This JSON format is currently -> [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) +> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) > and subject to change. The JSON serialization of a @@ -18,10 +18,30 @@ is a JSON object with the following fields: Information about the output paths of the derivation. This is a JSON object with one member per output, where the key is the output name and the value is a JSON object with these fields: - * `path`: The output path. + * `path`: + The output path, if it is known in advanced. + Otherwise, `null`. + + + * `method`: + For an output which will be [content addresed], a string representing the [method](@docroot@/store/store-object/content-address.md) of content addressing that is chosen. + Valid method strings are: + + - [`flat`](@docroot@/store/store-object/content-address.md#method-flat) + - [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive) + - [`text`](@docroot@/store/store-object/content-address.md#method-text) + - [`git`](@docroot@/store/store-object/content-address.md#method-git) + + Otherwise, `null`. * `hashAlgo`: - For fixed-output derivations, the hashing algorithm (e.g. `sha256`), optionally prefixed by `r:` if `hash` denotes a NAR hash rather than a flat file hash. + For an output which will be [content addresed], the name of the hash algorithm used. + Valid algorithm strings are: + + - `md5` + - `sha1` + - `sha256` + - `sha512` * `hash`: For fixed-output derivations, the expected content hash in base-16. @@ -32,7 +52,8 @@ is a JSON object with the following fields: > "outputs": { > "out": { > "path": "/nix/store/2543j7c6jn75blc3drf4g5vhb1rhdq29-source", - > "hashAlgo": "r:sha256", + > "method": "nar", + > "hashAlgo": "sha256", > "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62" > } > } diff --git a/doc/manual/src/protocols/json/store-object-info.md b/doc/manual/src/protocols/json/store-object-info.md index ba4ab098f..6b4f48437 100644 --- a/doc/manual/src/protocols/json/store-object-info.md +++ b/doc/manual/src/protocols/json/store-object-info.md @@ -3,7 +3,7 @@ > **Warning** > > This JSON format is currently -> [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) +> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) > and subject to change. Info about a [store object]. @@ -24,41 +24,45 @@ Info about a [store object]. An array of [store paths][store path], possibly including this one. -* `ca` (optional): +* `ca`: - Content address of this store object's file system object, used to compute its store path. + If the store object is [content-addressed], + this is the content address of this store object's file system object, used to compute its store path. + Otherwise (i.e. if it is [input-addressed]), this is `null`. -[store path]: @docroot@/glossary.md#gloss-store-path +[store path]: @docroot@/store/store-path.md [file system object]: @docroot@/store/file-system-object.md -[Nix Archive]: @docroot@/glossary.md#gloss-nar +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive ## Impure fields These are not intrinsic properties of the store object. In other words, the same store object residing in different store could have different values for these properties. -* `deriver` (optional): +* `deriver`: - The path to the [derivation] from which this store object is produced. + If known, the path to the [derivation] from which this store object was produced. + Otherwise `null`. [derivation]: @docroot@/glossary.md#gloss-store-derivation * `registrationTime` (optional): - When this derivation was added to the store. + If known, when this derivation was added to the store. + Otherwise `null`. -* `ultimate` (optional): +* `ultimate`: Whether this store object is trusted because we built it ourselves, rather than substituted a build product from elsewhere. -* `signatures` (optional): +* `signatures`: Signatures claiming that this store object is what it claims to be. Not relevant for [content-addressed] store objects, but useful for [input-addressed] store objects. - [content-addressed]: @docroot@/glossary.md#gloss-content-addressed-store-object - [input-addressed]: @docroot@/glossary.md#gloss-input-addressed-store-object +[content-addressed]: @docroot@/store/store-object/content-address.md +[input-addressed]: @docroot@/glossary.md#gloss-input-addressed-store-object ### `.narinfo` extra fields @@ -83,7 +87,7 @@ This information is not intrinsic to the store object, but about how it is store ## Computed closure fields -These fields are not stored at all, but computed by traverising the other other fields across all the store objects in a [closure]. +These fields are not stored at all, but computed by traversing the other fields across all the store objects in a [closure]. * `closureSize`: diff --git a/doc/manual/src/protocols/nix-archive.md b/doc/manual/src/protocols/nix-archive.md new file mode 100644 index 000000000..640b527f1 --- /dev/null +++ b/doc/manual/src/protocols/nix-archive.md @@ -0,0 +1,43 @@ +# Nix Archive (NAR) format + +This is the complete specification of the [Nix Archive] format. +The Nix Archive format closely follows the abstract specification of a [file system object] tree, +because it is designed to serialize exactly that data structure. + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#nix-archive +[file system object]: @docroot@/store/file-system-object.md + +The format of this specification is close to [Extended Backus–Naur form](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form), with the exception of the `str(..)` function / parameterized rule, which length-prefixes and pads strings. +This makes the resulting binary format easier to parse. + +Regular users do *not* need to know this information. +But for those interested in exactly how Nix works, e.g. if they are reimplementing it, this information can be useful. + +```ebnf +nar = str("nix-archive-1"), nar-obj; + +nar-obj = str("("), nar-obj-inner, str(")"); + +nar-obj-inner + = str("type"), str("regular") regular + | str("type"), str("symlink") symlink + | str("type"), str("directory") directory + ; + +regular = [ str("executable"), str("") ], str("contents"), str(contents); + +symlink = str("target"), str(target); + +(* side condition: directory entries must be ordered by their names *) +directory = { directory-entry }; + +directory-entry = str("entry"), str("("), str("name"), str(name), str("node"), nar-obj, str(")"); +``` + +The `str` function / parameterized rule is defined as follows: + +- `str(s)` = `int(|s|), pad(s);` + +- `int(n)` = the 64-bit little endian representation of the number `n` + +- `pad(s)` = the byte sequence `s`, padded with 0s to a multiple of 8 byte diff --git a/doc/manual/src/protocols/store-path.md b/doc/manual/src/protocols/store-path.md index 565c4fa75..52352d358 100644 --- a/doc/manual/src/protocols/store-path.md +++ b/doc/manual/src/protocols/store-path.md @@ -1,12 +1,14 @@ # Complete Store Path Calculation -This is the complete specification for how store paths are calculated. +This is the complete specification for how [store path]s are calculated. The format of this specification is close to [Extended Backus–Naur form](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form), but must deviate for a few things such as hash functions which we treat as bidirectional for specification purposes. Regular users do *not* need to know this information --- store paths can be treated as black boxes computed from the properties of the store objects they refer to. But for those interested in exactly how Nix works, e.g. if they are reimplementing it, this information can be useful. +[store path](@docroot@/store/store-path.md) + ## Store path proper ```ebnf @@ -34,18 +36,23 @@ where - `type` = one of: - ```ebnf - | "text" ( ":" store-path )* + | "text" { ":" store-path } ``` - for encoded derivations written to the store. + This is for the + ["Text"](@docroot@/store/store-object/content-address.md#method-text) + method of content addressing store objects. The optional trailing store paths are the references of the store object. - ```ebnf - | "source" ( ":" store-path )* + | "source" { ":" store-path } [ ":self" ] ``` - For paths copied to the store and hashed via a [Nix Archive (NAR)] and [SHA-256][sha-256]. - Just like in the text case, we can have the store objects referenced by their paths. + This is for the + ["Nix Archive"](@docroot@/store/store-object/content-address.md#method-nix-archive) + method of content addressing store objects, + if the hash algorithm is [SHA-256]. + Just like in the "Text" case, we can have the store objects referenced by their paths. Additionally, we can have an optional `:self` label to denote self reference. - ```ebnf @@ -53,8 +60,12 @@ where ``` For either the outputs built from derivations, - paths copied to the store hashed that area single file hashed directly, or the via a hash algorithm other than [SHA-256][sha-256]. - (in that case "source" is used; this is only necessary for compatibility). + or content-addressed store objects that are not using one of the two above cases. + To be explicit about the latter, that is currently these methods: + + - ["Flat"](@docroot@/store/store-object/content-address.md#method-flat) + - ["Git"](@docroot@/store/store-object/content-address.md#method-git) + - ["Nix Archive"](@docroot@/store/store-object/content-address.md#method-nix-archive) if the hash algorithm is not [SHA-256]. `id` is the name of the output (usually, "out"). For content-addressed store objects, `id`, is always "out". @@ -113,8 +124,8 @@ where Note that `id` = `"out"`, regardless of the name part of the store path. Also note that NAR + SHA-256 must not use this case, and instead must use the `type` = `"source:" ...` case. -[Nix Archive (NAR)]: @docroot@/glossary.md#gloss-NAR -[sha-256]: https://en.m.wikipedia.org/wiki/SHA-256 +[Nix Archive (NAR)]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive +[SHA-256]: https://en.m.wikipedia.org/wiki/SHA-256 ### Historical Note diff --git a/doc/manual/src/protocols/tarball-fetcher.md b/doc/manual/src/protocols/tarball-fetcher.md index 274fa6d63..5cff05d66 100644 --- a/doc/manual/src/protocols/tarball-fetcher.md +++ b/doc/manual/src/protocols/tarball-fetcher.md @@ -22,7 +22,7 @@ Link: ; rel="immutable" *flakeref* must be a tarball flakeref. It can contain the tarball flake attributes `narHash`, `rev`, `revCount` and `lastModified`. If `narHash` is included, its -value must be the NAR hash of the unpacked tarball (as computed via +value must be the [NAR hash][Nix Archive] of the unpacked tarball (as computed via `nix hash path`). Nix checks the contents of the returned tarball against the `narHash` attribute. The `rev` and `revCount` attributes are useful when the tarball flake is a mirror of a fetcher type that @@ -40,3 +40,31 @@ Link: ///archive/.tar.gz +``` + +> **Example** +> +> +> ```nix +> # flake.nix +> { +> inputs = { +> foo.url = "https://gitea.example.org/some-person/some-flake/archive/main.tar.gz"; +> bar.url = "https://gitea.example.org/some-other-person/other-flake/archive/442793d9ec0584f6a6e82fa253850c8085bb150a.tar.gz"; +> qux = { +> url = "https://forgejo.example.org/another-person/some-non-flake-repo/archive/development.tar.gz"; +> flake = false; +> }; +> }; +> outputs = { foo, bar, qux }: { /* ... */ }; +> } +``` + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive diff --git a/doc/manual/src/quick-start.md b/doc/manual/src/quick-start.md index 75853ced7..9eb7a3265 100644 --- a/doc/manual/src/quick-start.md +++ b/doc/manual/src/quick-start.md @@ -34,7 +34,7 @@ For more in-depth information you are kindly referred to subsequent chapters. lolcat: command not found ``` -1. Search for more packages on to try them out. +1. Search for more packages on [search.nixos.org](https://search.nixos.org/) to try them out. 1. Free up storage space: diff --git a/doc/manual/src/release-notes/index.md b/doc/manual/src/release-notes/index.md index cc805e631..d4e6292a6 100644 --- a/doc/manual/src/release-notes/index.md +++ b/doc/manual/src/release-notes/index.md @@ -1,12 +1,13 @@ # Nix Release Notes +The Nix release cycle is calendar-based as follows: + Nix has a release cycle of roughly 6 weeks. Notable changes and additions are announced in the release notes for each version. -Bugfixes can be backported on request to previous Nix releases. -We typically backport only as far back as the Nix version used in the latest NixOS release, which is announced in the [NixOS release notes](https://nixos.org/manual/nixos/stable/release-notes.html#ch-release-notes). - -Backports never skip releases. -If a feature is backported to version `x.y`, it must also be available in version `x.(y+1)`. -This ensures that upgrading from an older version with backports is still safe and no backported functionality will go missing. +The supported Nix versions are: +- The latest release +- The version used in the stable NixOS release, which is announced in the [NixOS release notes](https://nixos.org/manual/nixos/stable/release-notes.html#ch-release-notes). +Bugfixes and security issues are backported to every supported version. +Patch releases are published as needed. diff --git a/doc/manual/src/release-notes/rl-2.15.md b/doc/manual/src/release-notes/rl-2.15.md index 133121999..e7e52631b 100644 --- a/doc/manual/src/release-notes/rl-2.15.md +++ b/doc/manual/src/release-notes/rl-2.15.md @@ -11,7 +11,7 @@ As the choice of hash formats is no longer binary, the `--base16` flag is also added to explicitly specify the Base16 format, which is still the default. -* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](../glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents. +* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](@docroot@/glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents. The new `^` syntax for store paths introduced in Nix 2.13 allows explicitly referencing output paths of a derivation. Using this is better and more clear than relying on the now-removed `.drv` special handling. diff --git a/doc/manual/src/release-notes/rl-2.18.md b/doc/manual/src/release-notes/rl-2.18.md index 4bbc52b50..eb26fc9e7 100644 --- a/doc/manual/src/release-notes/rl-2.18.md +++ b/doc/manual/src/release-notes/rl-2.18.md @@ -13,7 +13,7 @@ - The `discard-references` feature has been stabilized. This means that the - [unsafeDiscardReferences](@docroot@/contributing/experimental-features.md#xp-feature-discard-references) + [unsafeDiscardReferences](@docroot@/development/experimental-features.md#xp-feature-discard-references) attribute is no longer guarded by an experimental flag and can be used freely. @@ -21,7 +21,7 @@ This only affects `nix-build --json` when "building" non-derivation things like fetched sources, which is a no-op. - A new builtin [`outputOf`](@docroot@/language/builtins.md#builtins-outputOf) has been added. - It is part of the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. + It is part of the [`dynamic-derivations`](@docroot@/development/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. - Flake follow paths at depths greater than 2 are now handled correctly, preventing "follows a non-existent input" errors. diff --git a/doc/manual/src/release-notes/rl-2.19.md b/doc/manual/src/release-notes/rl-2.19.md index ba6eb9c64..e2e2f85cc 100644 --- a/doc/manual/src/release-notes/rl-2.19.md +++ b/doc/manual/src/release-notes/rl-2.19.md @@ -17,8 +17,8 @@ - `nix-shell` shebang lines now support single-quoted arguments. -- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). - This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). +- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/development/experimental-features.md#xp-fetch-tree). + This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/development/experimental-features.md#xp-fetch-tree). - The interface for creating and updating lock files has been overhauled: @@ -33,7 +33,7 @@ - The flake-specific flags `--recreate-lock-file` and `--update-input` have been removed from all commands operating on installables. They are superceded by `nix flake update`. -- Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). +- Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches). - [`nix path-info --json`](@docroot@/command-ref/new-cli/nix3-path-info.md) (experimental) now returns a JSON map rather than JSON list. diff --git a/doc/manual/src/release-notes/rl-2.20.md b/doc/manual/src/release-notes/rl-2.20.md index 8ede168a4..eb724f600 100644 --- a/doc/manual/src/release-notes/rl-2.20.md +++ b/doc/manual/src/release-notes/rl-2.20.md @@ -200,3 +200,9 @@ while performing various operations (including `nix develop`, `nix flake update`, and so on). With several fixes to Nix's signal handlers, Nix commands will now exit quickly after Ctrl-C is pressed. + +- `nix copy` to a `ssh-ng` store now needs `--substitute-on-destination` (a.k.a. `-s`) + in order to substitute paths on the remote store instead of copying them. + The behavior is consistent with `nix copy` to a different kind of remote store. + Previously this behavior was controlled by the + `builders-use-substitutes` setting and `--substitute-on-destination` was ignored. diff --git a/doc/manual/src/release-notes/rl-2.21.md b/doc/manual/src/release-notes/rl-2.21.md new file mode 100644 index 000000000..75114f117 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.21.md @@ -0,0 +1,302 @@ +# Release 2.21.0 (2024-03-11) + +- Fix a fixed-output derivation sandbox escape (CVE-2024-27297) + + Cooperating Nix derivations could send file descriptors to files in the Nix + store to each other via Unix domain sockets in the abstract namespace. This + allowed one derivation to modify the output of the other derivation, after Nix + has registered the path as "valid" and immutable in the Nix database. + In particular, this allowed the output of fixed-output derivations to be + modified from their expected content. + + This isn't the case any more. + +- CLI options `--arg-from-file` and `--arg-from-stdin` [#10122](https://github.com/NixOS/nix/pull/10122) + + The new CLI option `--arg-from-file` *name* *path* passes the contents + of file *path* as a string value via the function argument *name* to a + Nix expression. Similarly, the new option `--arg-from-stdin` *name* + reads the contents of the string from standard input. + +- Concise error printing in `nix repl` [#9928](https://github.com/NixOS/nix/pull/9928) + + Previously, if an element of a list or attribute set threw an error while + evaluating, `nix repl` would print the entire error (including source location + information) inline. This output was clumsy and difficult to parse: + + ``` + nix-repl> { err = builtins.throw "uh oh!"; } + { err = «error: + … while calling the 'throw' builtin + at «string»:1:9: + 1| { err = builtins.throw "uh oh!"; } + | ^ + + error: uh oh!»; } + ``` + + Now, only the error message is displayed, making the output much more readable. + ``` + nix-repl> { err = builtins.throw "uh oh!"; } + { err = «error: uh oh!»; } + ``` + + However, if the whole expression being evaluated throws an error, source + locations and (if applicable) a stack trace are printed, just like you'd expect: + + ``` + nix-repl> builtins.throw "uh oh!" + error: + … while calling the 'throw' builtin + at «string»:1:1: + 1| builtins.throw "uh oh!" + | ^ + + error: uh oh! + ``` + +- `--debugger` can now access bindings from `let` expressions [#8827](https://github.com/NixOS/nix/issues/8827) [#9918](https://github.com/NixOS/nix/pull/9918) + + Breakpoints and errors in the bindings of a `let` expression can now access + those bindings in the debugger. Previously, only the body of `let` expressions + could access those bindings. + +- Enter the `--debugger` when `builtins.trace` is called if `debugger-on-trace` is set [#9914](https://github.com/NixOS/nix/pull/9914) + + If the `debugger-on-trace` option is set and `--debugger` is given, + `builtins.trace` calls will behave similarly to `builtins.break` and will enter + the debug REPL. This is useful for determining where warnings are being emitted + from. + +- Debugger prints source position information [#9913](https://github.com/NixOS/nix/pull/9913) + + The `--debugger` now prints source location information, instead of the + pointers of source location information. Before: + + ``` + nix-repl> :bt + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + 0x600001522598 + ``` + + After: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + /nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27 + + 131| + 132| bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs; + | ^ + 133| in + ``` + +- The `--debugger` will start more reliably in `let` expressions and function calls [#6649](https://github.com/NixOS/nix/issues/6649) [#9917](https://github.com/NixOS/nix/pull/9917) + + Previously, if you attempted to evaluate this file with the debugger: + + ```nix + let + a = builtins.trace "before inner break" ( + builtins.break "hello" + ); + b = builtins.trace "before outer break" ( + builtins.break a + ); + in + b + ``` + + Nix would correctly enter the debugger at `builtins.break a`, but if you asked + it to `:continue`, it would skip over the `builtins.break "hello"` expression + entirely. + + Now, Nix will correctly enter the debugger at both breakpoints. + +- Nested debuggers are no longer supported [#9920](https://github.com/NixOS/nix/pull/9920) + + Previously, evaluating an expression that throws an error in the debugger would + enter a second, nested debugger: + + ``` + nix-repl> builtins.throw "what" + error: what + + + Starting REPL to allow you to inspect the current state of the evaluator. + + Welcome to Nix 2.18.1. Type :? for help. + + nix-repl> + ``` + + Now, it just prints the error message like `nix repl`: + + ``` + nix-repl> builtins.throw "what" + error: + … while calling the 'throw' builtin + at «string»:1:1: + 1| builtins.throw "what" + | ^ + + error: what + ``` + +- Consistent order of function arguments in printed expressions [#9874](https://github.com/NixOS/nix/pull/9874) + + Function arguments are now printed in lexicographic order rather than the internal, creation-time based symbol order. + +- Fix duplicate attribute error positions for `inherit` [#9874](https://github.com/NixOS/nix/pull/9874) + + When an `inherit` caused a duplicate attribute error the position of the error was not reported correctly, placing the error with the inherit itself or at the start of the bindings block instead of the offending attribute name. + +- `inherit (x) ...` evaluates `x` only once [#9847](https://github.com/NixOS/nix/pull/9847) + + `inherit (x) a b ...` now evaluates the expression `x` only once for all inherited attributes rather than once for each inherited attribute. + This does not usually have a measurable impact, but side-effects (such as `builtins.trace`) would be duplicated and expensive expressions (such as derivations) could cause a measurable slowdown. + +- Store paths are allowed to start with `.` [#912](https://github.com/NixOS/nix/issues/912) [#9091](https://github.com/NixOS/nix/pull/9091) [#9095](https://github.com/NixOS/nix/pull/9095) [#9120](https://github.com/NixOS/nix/pull/9120) [#9121](https://github.com/NixOS/nix/pull/9121) [#9122](https://github.com/NixOS/nix/pull/9122) [#9130](https://github.com/NixOS/nix/pull/9130) [#9219](https://github.com/NixOS/nix/pull/9219) [#9224](https://github.com/NixOS/nix/pull/9224) [#9867](https://github.com/NixOS/nix/pull/9867) + + Leading periods were allowed by accident in Nix 2.4. The Nix team has considered this to be a bug, but this behavior has since been relied on by users, leading to unnecessary difficulties. + From now on, leading periods are supported. The names `.` and `..` are disallowed, as well as those starting with `.-` or `..-`. + + Nix versions that denied leading periods are documented [in the issue](https://github.com/NixOS/nix/issues/912#issuecomment-1919583286). + +- `nix repl` pretty-prints values [#9931](https://github.com/NixOS/nix/pull/9931) + + `nix repl` will now pretty-print values: + + ``` + { + attrs = { + a = { + b = { + c = { }; + }; + }; + }; + list = [ 1 ]; + list' = [ + 1 + 2 + 3 + ]; + } + ``` + +- Introduction of `--regex` and `--all` in `nix profile remove` and `nix profile upgrade` [#10166](https://github.com/NixOS/nix/pull/10166) + + Previously the command-line arguments for `nix profile remove` and `nix profile upgrade` matched the package entries using regular expression. + For instance: + + ``` + nix profile remove '.*vim.*' + ``` + + This would remove all packages that contain `vim` in their name. + + In most cases, only singular package names were used to remove and upgrade packages. Mixing this with regular expressions sometimes lead to unintended behavior. For instance, `python3.1` could match `python311`. + + To avoid unintended behavior, the arguments are now only matching exact names. + + Matching using regular expressions is still possible by using the new `--regex` flag: + + ``` + nix profile remove --regex '.*vim.*' + ``` + + One of the most useful cases for using regular expressions was to upgrade all packages. This was previously accomplished by: + + ``` + nix profile upgrade '.*' + ``` + + With the introduction of the `--all` flag, this now becomes more straightforward: + + ``` + nix profile upgrade --all + ``` + +- Visual clutter in `--debugger` is reduced [#9919](https://github.com/NixOS/nix/pull/9919) + + Before: + ``` + info: breakpoint reached + + + Starting REPL to allow you to inspect the current state of the evaluator. + + Welcome to Nix 2.20.0pre20231222_dirty. Type :? for help. + + nix-repl> :continue + error: uh oh + + + Starting REPL to allow you to inspect the current state of the evaluator. + + Welcome to Nix 2.20.0pre20231222_dirty. Type :? for help. + + nix-repl> + ``` + + After: + + ``` + info: breakpoint reached + + Nix 2.20.0pre20231222_dirty debugger + Type :? for help. + nix-repl> :continue + error: uh oh + + nix-repl> + ``` + +- Cycle detection in `nix repl` is simpler and more reliable [#8672](https://github.com/NixOS/nix/issues/8672) [#9926](https://github.com/NixOS/nix/pull/9926) + + The cycle detection in `nix repl`, `nix eval`, `builtins.trace`, and everywhere + else values are printed is now simpler and matches the cycle detection in + `nix-instantiate --eval` output. + + Before: + + ``` + nix eval --expr 'let self = { inherit self; }; in self' + { self = { self = «repeated»; }; } + ``` + + After: + + ``` + { self = «repeated»; } + ``` + +- In the debugger, `while evaluating the attribute` errors now include position information [#9915](https://github.com/NixOS/nix/pull/9915) + + Before: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + 0x600001522598 + ``` + + After: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + /nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27 + + 131| + 132| bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs; + | ^ + 133| in + ``` + +- Stack size is increased on macOS [#9860](https://github.com/NixOS/nix/pull/9860) + + Previously, Nix would set the stack size to 64MiB on Linux, but would leave the + stack size set to the default (approximately 8KiB) on macOS. Now, the stack + size is correctly set to 64MiB on macOS as well, which should reduce stack + overflow segfaults in deeply-recursive Nix expressions. + diff --git a/doc/manual/src/release-notes/rl-2.22.md b/doc/manual/src/release-notes/rl-2.22.md new file mode 100644 index 000000000..c78d3d692 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.22.md @@ -0,0 +1,21 @@ +# Release 2.22.0 (2024-04-23) + +### Significant changes + +- Remove experimental repl-flake [#10103](https://github.com/NixOS/nix/issues/10103) [#10299](https://github.com/NixOS/nix/pull/10299) + + The `repl-flake` experimental feature has been removed. The `nix repl` command now works like the rest of the new CLI in that `nix repl {path}` now tries to load a flake at `{path}` (or fails if the `flakes` experimental feature isn't enabled). + +### Other changes + +- `nix eval` prints derivations as `.drv` paths [#10200](https://github.com/NixOS/nix/pull/10200) + + `nix eval` will now print derivations as their `.drv` paths, rather than as + attribute sets. This makes commands like `nix eval nixpkgs#bash` terminate + instead of infinitely looping into recursive self-referential attributes: + + ```ShellSession + $ nix eval nixpkgs#bash + «derivation /nix/store/m32cbgbd598f4w299g0hwyv7gbw6rqcg-bash-5.2p26.drv» + ``` + diff --git a/doc/manual/src/release-notes/rl-2.23.md b/doc/manual/src/release-notes/rl-2.23.md new file mode 100644 index 000000000..ac842fdc0 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.23.md @@ -0,0 +1,102 @@ +# Release 2.23.0 (2024-06-03) + +- New builtin: `builtins.warn` [#306026](https://github.com/NixOS/nix/issues/306026) [#10592](https://github.com/NixOS/nix/pull/10592) + + `builtins.warn` behaves like `builtins.trace "warning: ${msg}"`, has an accurate log level, and is controlled by the options + [`debugger-on-trace`](@docroot@/command-ref/conf-file.md#conf-debugger-on-trace), + [`debugger-on-warn`](@docroot@/command-ref/conf-file.md#conf-debugger-on-warn) and + [`abort-on-warn`](@docroot@/command-ref/conf-file.md#conf-abort-on-warn). + +- Make `nix build --keep-going` consistent with `nix-build --keep-going` + + This means that if e.g. multiple fixed-output derivations fail to + build, all hash mismatches are displayed. + +- Modify `nix derivation {add,show}` JSON format [#9866](https://github.com/NixOS/nix/issues/9866) [#10722](https://github.com/NixOS/nix/pull/10722) + + The JSON format for derivations has been slightly revised to better conform to our [JSON guidelines](@docroot@/development/cli-guideline.md#returning-future-proof-json). + In particular, the hash algorithm and content addressing method of content-addresed derivation outputs are now separated into two fields `hashAlgo` and `method`, + rather than one field with an arcane `:`-separated format. + + This JSON format is only used by the experimental `nix derivation` family of commands, at this time. + Future revisions are expected as the JSON format is still not entirely in compliance even after these changes. + +- Warn on unknown settings anywhere in the command line [#10701](https://github.com/NixOS/nix/pull/10701) + + All `nix` commands will now properly warn when an unknown option is specified anywhere in the command line. + + Before: + + ```console + $ nix-instantiate --option foobar baz --expr '{}' + warning: unknown setting 'foobar' + $ nix-instantiate '{}' --option foobar baz --expr + $ nix eval --expr '{}' --option foobar baz + { } + ``` + + After: + + ```console + $ nix-instantiate --option foobar baz --expr '{}' + warning: unknown setting 'foobar' + $ nix-instantiate '{}' --option foobar baz --expr + warning: unknown setting 'foobar' + $ nix eval --expr '{}' --option foobar baz + warning: unknown setting 'foobar' + { } + ``` + +- `nix env shell` is the new `nix shell`, and `nix shell` remains an accepted alias [#10504](https://github.com/NixOS/nix/issues/10504) [#10807](https://github.com/NixOS/nix/pull/10807) + + This is part of an effort to bring more structure to the CLI subcommands. + + `nix env` will be about the process environment. + Future commands may include `nix env run` and `nix env print-env`. + + It is also somewhat analogous to the [planned](https://github.com/NixOS/nix/issues/10504) `nix dev shell` (currently `nix develop`), which is less about environment variables, and more about running a development shell, which is a more powerful command, but also requires more setup. + +- Flake operations that expect derivations now print the failing value and its type [#10778](https://github.com/NixOS/nix/pull/10778) + + In errors like `flake output attribute 'nixosConfigurations.yuki.config' is not a derivation or path`, the message now includes the failing value and type. + + Before: + + ``` + error: flake output attribute 'nixosConfigurations.yuki.config' is not a derivation or path + ```` + + After: + + ``` + error: expected flake output attribute 'nixosConfigurations.yuki.config' to be a derivation or path but found a set: { appstream = «thunk»; assertions = «thunk»; boot = { bcache = «thunk»; binfmt = «thunk»; binfmtMiscRegistrations = «thunk»; blacklistedKernelModules = «thunk»; bootMount = «thunk»; bootspec = «thunk»; cleanTmpDir = «thunk»; consoleLogLevel = «thunk»; «43 attributes elided» }; «48 attributes elided» } + ``` + +- `fetchTree` now fetches Git repositories shallowly by default [#10028](https://github.com/NixOS/nix/pull/10028) + + `builtins.fetchTree` now clones Git repositories shallowly by default, which reduces network traffic and disk usage significantly in many cases. + + Previously, the default behavior was to clone the full history of a specific tag or branch (e.g. `ref`) and only afterwards extract the files of one specific revision. + + From now on, the `ref` and `allRefs` arguments will be ignored, except if shallow cloning is disabled by setting `shallow = false`. + + The defaults for `builtins.fetchGit` remain unchanged. Here, shallow cloning has to be enabled manually by passing `shallow = true`. + +- Store object info JSON format now uses `null` rather than omitting fields [#9995](https://github.com/NixOS/nix/pull/9995) + + The [store object info JSON format](@docroot@/protocols/json/store-object-info.md), used for e.g. `nix path-info`, no longer omits fields to indicate absent information, but instead includes the fields with a `null` value. + For example, `"ca": null` is used to to indicate a store object that isn't content-addressed rather than omitting the `ca` field entirely. + This makes records of this sort more self-describing, and easier to consume programmatically. + + We will follow this design principle going forward; + the [JSON guidelines](@docroot@/development/json-guideline.md) in the contributing section have been updated accordingly. + +- Large path warnings [#10661](https://github.com/NixOS/nix/pull/10661) + + Nix can now warn when evaluation of a Nix expression causes a large + path to be copied to the Nix store. The threshold for this warning can + be configured using [the `warn-large-path-threshold` + setting](@docroot@/command-ref/conf-file.md#warn-large-path-threshold), + e.g. `--warn-large-path-threshold 100M` will warn about paths larger + than 100 MiB. + diff --git a/doc/manual/src/release-notes/rl-2.24.md b/doc/manual/src/release-notes/rl-2.24.md new file mode 100644 index 000000000..5bcc1d79c --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.24.md @@ -0,0 +1,324 @@ +# Release 2.24.0 (2024-07-31) + +### Significant changes + +- Harden user sandboxing + + The build directory has been hardened against interference with the outside world by nesting it inside another directory owned by (and only readable by) the daemon user. + + This is a low severity security fix, [CVE-2024-38531](https://www.cve.org/CVERecord?id=CVE-2024-38531). + + Credit: [**@alois31**](https://github.com/alois31), [**Linus Heckemann (@lheckemann)**](https://github.com/lheckemann) + Co-authors: [**@edolstra**](https://github.com/edolstra) + +- `nix-shell ` looks for `shell.nix` [#496](https://github.com/NixOS/nix/issues/496) [#2279](https://github.com/NixOS/nix/issues/2279) [#4529](https://github.com/NixOS/nix/issues/4529) [#5431](https://github.com/NixOS/nix/issues/5431) [#11053](https://github.com/NixOS/nix/issues/11053) [#11057](https://github.com/NixOS/nix/pull/11057) + + `nix-shell $x` now looks for `$x/shell.nix` when `$x` resolves to a directory. + + Although this might be seen as a breaking change, its primarily interactive usage makes it a minor issue. + This adjustment addresses a commonly reported problem. + + This also applies to `nix-shell` shebang scripts. Consider the following example: + + ```shell + #!/usr/bin/env nix-shell + #!nix-shell -i bash + ``` + + This will now load `shell.nix` from the script's directory, if it exists; `default.nix` otherwise. + + The old behavior can be opted into by setting the option [`nix-shell-always-looks-for-shell-nix`](@docroot@/command-ref/conf-file.md#conf-nix-shell-always-looks-for-shell-nix) to `false`. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + +- `nix-repl`'s `:doc` shows documentation comments [#3904](https://github.com/NixOS/nix/issues/3904) [#10771](https://github.com/NixOS/nix/issues/10771) [#1652](https://github.com/NixOS/nix/pull/1652) [#9054](https://github.com/NixOS/nix/pull/9054) [#11072](https://github.com/NixOS/nix/pull/11072) + + `nix repl` has a `:doc` command that previously only rendered documentation for internally defined functions. + This feature has been extended to also render function documentation comments, in accordance with [RFC 145]. + + Example: + + ``` + nix-repl> :doc lib.toFunction + Function toFunction + … defined at /home/user/h/nixpkgs/lib/trivial.nix:1072:5 + + Turns any non-callable values into constant functions. Returns + callable values as is. + + Inputs + + v + + : Any value + + Examples + + :::{.example} + + ## lib.trivial.toFunction usage example + + | nix-repl> lib.toFunction 1 2 + | 1 + | + | nix-repl> lib.toFunction (x: x + 1) 2 + | 3 + + ::: + ``` + + Known limitations: + - It does not render documentation for "formals", such as `{ /** the value to return */ x, ... }: x`. + - Some extensions to markdown are not yet supported, as you can see in the example above. + + We'd like to acknowledge [Yingchi Long (@inclyc)](https://github.com/inclyc) for proposing a proof of concept for this functionality in [#9054](https://github.com/NixOS/nix/pull/9054), as well as [@sternenseemann](https://github.com/sternenseemann) and [Johannes Kirschbauer (@hsjobeki)](https://github.com/hsjobeki) for their contributions, proposals, and their work on [RFC 145]. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + + [RFC 145]: https://github.com/NixOS/rfcs/pull/145 + +### Other changes + +- Solve `cached failure of attribute X` [#9165](https://github.com/NixOS/nix/issues/9165) [#10513](https://github.com/NixOS/nix/issues/10513) [#10564](https://github.com/NixOS/nix/pull/10564) + + This eliminates all "cached failure of attribute X" messages by forcing evaluation of the original value when needed to show the exception to the user. This enhancement improves error reporting by providing the underlying message and stack trace. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Run the flake regressions test suite [#10603](https://github.com/NixOS/nix/pull/10603) + + This update introduces a GitHub action to run a subset of the [flake regressions test suite](https://github.com/NixOS/flake-regressions), which includes 259 flakes with their expected evaluation results. Currently, the action runs the first 25 flakes due to the full test suite's extensive runtime. A manually triggered action may be implemented later to run the entire test suite. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Support unit prefixes in configuration settings [#10668](https://github.com/NixOS/nix/pull/10668) + + Configuration settings in Nix now support unit prefixes, allowing for more intuitive and readable configurations. For example, you can now specify [`--min-free 1G`](@docroot@/command-ref/opt-common.md#opt-min-free) to set the minimum free space to 1 gigabyte. + + This enhancement was extracted from [#7851](https://github.com/NixOS/nix/pull/7851) and is also useful for PR [#10661](https://github.com/NixOS/nix/pull/10661). + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- `nix build`: show all FOD errors with `--keep-going` [#10734](https://github.com/NixOS/nix/pull/10734) + + The [`nix build`](@docroot@/command-ref/new-cli/nix3-build.md) command has been updated to improve the behavior of the [`--keep-going`] flag. Now, when `--keep-going` is used, all hash-mismatch errors of failing fixed-output derivations (FODs) are displayed, similar to the behavior for other build failures. This enhancement ensures that all relevant build errors are shown, making it easier for users to update multiple derivations at once or to diagnose and fix issues. + + Author: [**Jörg Thalheim (@Mic92)**](https://github.com/Mic92), [**Maximilian Bosch (@Ma27)**](https://github.com/Ma27) + + [`--keep-going`](@docroot@/command-ref/opt-common.md#opt-keep-going) + +- Build with Meson [#2503](https://github.com/NixOS/nix/issues/2503) [#10378](https://github.com/NixOS/nix/pull/10378) [#10855](https://github.com/NixOS/nix/pull/10855) [#10904](https://github.com/NixOS/nix/pull/10904) [#10908](https://github.com/NixOS/nix/pull/10908) [#10914](https://github.com/NixOS/nix/pull/10914) [#10933](https://github.com/NixOS/nix/pull/10933) [#10936](https://github.com/NixOS/nix/pull/10936) [#10954](https://github.com/NixOS/nix/pull/10954) [#10955](https://github.com/NixOS/nix/pull/10955) [#10963](https://github.com/NixOS/nix/pull/10963) [#10967](https://github.com/NixOS/nix/pull/10967) [#10973](https://github.com/NixOS/nix/pull/10973) [#11034](https://github.com/NixOS/nix/pull/11034) [#11054](https://github.com/NixOS/nix/pull/11054) [#11055](https://github.com/NixOS/nix/pull/11055) [#11060](https://github.com/NixOS/nix/pull/11060) [#11064](https://github.com/NixOS/nix/pull/11064) [#11155](https://github.com/NixOS/nix/pull/11155) + + These changes aim to replace the use of autotools and `make` with Meson for building various components of Nix. Additionally, each library is built in its own derivation, leveraging Meson's "subprojects" feature to allow a single development shell for building all libraries while also supporting separate builds. This approach aims to improve productivity and build modularity, compared to both make and a monolithic Meson-based derivation. + + Special thanks to everyone who has contributed to the Meson port, particularly [**@p01arst0rm**](https://github.com/p01arst0rm) and [**@Qyriad**](https://github.com/Qyriad). + + Authors: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314), [**Tom Bereknyei**](https://github.com/tomberek), [**Théophane Hufschmitt (@thufschmitt)**](https://github.com/thufschmitt), [**Valentin Gagarin (@fricklerhandwerk)**](https://github.com/fricklerhandwerk), [**Robert Hensing (@roberth)**](https://github.com/roberth) + Co-authors: [**@p01arst0rm**](https://github.com/p01arst0rm), [**@Qyriad**](https://github.com/Qyriad) + +- Evaluation cache: fix cache regressions [#10570](https://github.com/NixOS/nix/issues/10570) [#11086](https://github.com/NixOS/nix/pull/11086) + + This update addresses two bugs in the evaluation cache system: + + 1. Regression in #10570: The evaluation cache was not being persisted in `nix develop`. + 2. Nix could sometimes try to commit the evaluation cache SQLite transaction without there being an active transaction, resulting in non-error errors being printed. + + Author: [**Lexi Mattick (@kognise)**](https://github.com/kognise) + +- Introduce `libnixflake` [#9063](https://github.com/NixOS/nix/pull/9063) + + A new library, `libnixflake`, has been introduced to better separate the Flakes layer within Nix. This change refactors the codebase to encapsulate Flakes-specific functionality within its own library. + + See the commits in the pull request for detailed changes, with the only significant code modifications happening in the initial commit. + + This change was alluded to in [RFC 134](https://github.com/nixos/rfcs/blob/master/rfcs/0134-nix-store-layer.md) and is a step towards a more modular and maintainable codebase. + + Author: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314) + +- CLI options `--arg-from-file` and `--arg-from-stdin` [#9913](https://github.com/NixOS/nix/pull/9913) + +- The `--debugger` now prints source location information, instead of the + pointers of source location information. Before: + + ``` + nix-repl> :bt + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + 0x600001522598 + ``` + + After: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + /nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27 + + 131| + 132| bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs; + | ^ + 133| in + ``` + +- Stop vendoring `toml11` + + We don't apply any patches to it, and vendoring it locks users into + bugs (it hasn't been updated since its introduction in late 2021). + + Author: [**Winter (@winterqt)**](https://github.com/winterqt) + +- Rename hash format `base32` to `nix32` [#8678](https://github.com/NixOS/nix/pull/8678) + + Hash format `base32` was renamed to `nix32` since it used a special nix-specific character set for + [Base32](https://en.wikipedia.org/wiki/Base32). + + **Deprecation**: Use `nix32` instead of `base32` as `toHashFormat` + + For the builtin `convertHash`, the `toHashFormat` parameter now accepts the same hash formats as the `--to`/`--from` + parameters of the `nix hash conert` command: `"base16"`, `"nix32"`, `"base64"`, and `"sri"`. The former `"base32"` value + remains as a deprecated alias for `"nix32"`. Please convert your code from: + + ```nix + builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base32";} + ``` + + to + + ```nix + builtins.convertHash { inherit hash hashAlgo; toHashFormat = "nix32";} + ``` + +- Add `pipe-operators` experimental feature [#11131](https://github.com/NixOS/nix/pull/11131) + + This is a draft implementation of [RFC 0148](https://github.com/NixOS/rfcs/pull/148). + + The `pipe-operators` experimental feature adds [`<|` and `|>` operators][pipe operators] to the Nix language. + *a* `|>` *b* is equivalent to the function application *b* *a*, and + *a* `<|` *b* is equivalent to the function application *a* *b*. + + For example: + + ``` + nix-repl> 1 |> builtins.add 2 |> builtins.mul 3 + 9 + + nix-repl> builtins.add 1 <| builtins.mul 2 <| 3 + 7 + ``` + + `<|` and `|>` are right and left associative, respectively, and have lower precedence than any other operator. + These properties may change in future releases. + + See [the RFC](https://github.com/NixOS/rfcs/pull/148) for more examples and rationale. + + [pipe operators]: @docroot@/language/operators.md#pipe-operators + +- `nix-shell` shebang uses relative path [#4232](https://github.com/NixOS/nix/issues/4232) [#5088](https://github.com/NixOS/nix/pull/5088) [#11058](https://github.com/NixOS/nix/pull/11058) + + + Relative [path](@docroot@/language/types.md#type-path) literals in `nix-shell` shebang scripts' options are now resolved relative to the [script's location](@docroot@/glossary.md?highlight=base%20directory#gloss-base-directory). + Previously they were resolved relative to the current working directory. + + For example, consider the following script in `~/myproject/say-hi`: + + ```shell + #!/usr/bin/env nix-shell + #!nix-shell --expr 'import ./shell.nix' + #!nix-shell --arg toolset './greeting-tools.nix' + #!nix-shell -i bash + hello + ``` + + Older versions of `nix-shell` would resolve `shell.nix` relative to the current working directory, such as the user's home directory in this example: + + ```console + [hostname:~]$ ./myproject/say-hi + error: + … while calling the 'import' builtin + at «string»:1:2: + 1| (import ./shell.nix) + | ^ + + error: path '/home/user/shell.nix' does not exist + ``` + + Since this release, `nix-shell` resolves `shell.nix` relative to the script's location, and `~/myproject/shell.nix` is used. + + ```console + $ ./myproject/say-hi + Hello, world! + ``` + + **Opt-out** + + This is technically a breaking change, so we have added an option so you can adapt independently of your Nix update. + The old behavior can be opted into by setting the option [`nix-shell-shebang-arguments-relative-to-script`](@docroot@/command-ref/conf-file.md#conf-nix-shell-shebang-arguments-relative-to-script) to `false`. + This option will be removed in a future release. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + +- Improve handling of tarballs that don't consist of a single top-level directory [#11195](https://github.com/NixOS/nix/pull/11195) + + In previous Nix releases, the tarball fetcher (used by `builtins.fetchTarball`) erroneously merged top-level directories into a single directory, and silently discarded top-level files that are not directories. This is no longer the case. The new behaviour is that *only* if the tarball consists of a single directory, the top-level path component of the files in the tarball is removed (similar to `tar`'s `--strip-components=1`). + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Improve handling of tarballs that don't consist of a single top-level directory [#11195](https://github.com/NixOS/nix/pull/11195) + + In previous Nix releases, the tarball fetcher (used by `builtins.fetchTarball`) erroneously merged top-level directories into a single directory, and silently discarded top-level files that are not directories. This is no longer the case. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Setting to warn about large paths [#10778](https://github.com/NixOS/nix/pull/10778) + + Nix can now warn when evaluation of a Nix expression causes a large + path to be copied to the Nix store. The threshold for this warning can + be configured using the `warn-large-path-threshold` setting, + e.g. `--warn-large-path-threshold 100M`. + + +# Contributors + +This release was made possible by the following 43 contributors: + +- Andreas Rammhold [**(@andir)**](https://github.com/andir) +- Andrew Marshall [**(@amarshall)**](https://github.com/amarshall) +- Brian McKenna [**(@puffnfresh)**](https://github.com/puffnfresh) +- Cameron [**(@SkamDart)**](https://github.com/SkamDart) +- Cole Helbling [**(@cole-h)**](https://github.com/cole-h) +- Corbin Simpson [**(@MostAwesomeDude)**](https://github.com/MostAwesomeDude) +- Eelco Dolstra [**(@edolstra)**](https://github.com/edolstra) +- Emily [**(@emilazy)**](https://github.com/emilazy) +- Enno Richter [**(@elohmeier)**](https://github.com/elohmeier) +- Farid Zakaria [**(@fzakaria)**](https://github.com/fzakaria) +- HaeNoe [**(@haenoe)**](https://github.com/haenoe) +- Hamir Mahal [**(@hamirmahal)**](https://github.com/hamirmahal) +- Harmen [**(@alicebob)**](https://github.com/alicebob) +- Ivan Trubach [**(@tie)**](https://github.com/tie) +- Jared Baur [**(@jmbaur)**](https://github.com/jmbaur) +- John Ericson [**(@Ericson2314)**](https://github.com/Ericson2314) +- Jonathan De Troye [**(@detroyejr)**](https://github.com/detroyejr) +- Jörg Thalheim [**(@Mic92)**](https://github.com/Mic92) +- Klemens Nanni [**(@klemensn)**](https://github.com/klemensn) +- Las Safin [**(@L-as)**](https://github.com/L-as) +- Lexi Mattick [**(@kognise)**](https://github.com/kognise) +- Matthew Bauer [**(@matthewbauer)**](https://github.com/matthewbauer) +- Max “Goldstein†Siling [**(@GoldsteinE)**](https://github.com/GoldsteinE) +- Mingye Wang [**(@Artoria2e5)**](https://github.com/Artoria2e5) +- Philip Taron [**(@philiptaron)**](https://github.com/philiptaron) +- Pierre Bourdon [**(@delroth)**](https://github.com/delroth) +- Pino Toscano [**(@pinotree)**](https://github.com/pinotree) +- RTUnreal [**(@RTUnreal)**](https://github.com/RTUnreal) +- Robert Hensing [**(@roberth)**](https://github.com/roberth) +- Romain Neil [**(@romain-neil)**](https://github.com/romain-neil) +- Ryan Hendrickson [**(@rhendric)**](https://github.com/rhendric) +- Sergei Trofimovich [**(@trofi)**](https://github.com/trofi) +- Shogo Takata [**(@pineapplehunter)**](https://github.com/pineapplehunter) +- Siddhant Kumar [**(@siddhantk232)**](https://github.com/siddhantk232) +- Silvan Mosberger [**(@infinisil)**](https://github.com/infinisil) +- Théophane Hufschmitt [**(@thufschmitt)**](https://github.com/thufschmitt) +- Valentin Gagarin [**(@fricklerhandwerk)**](https://github.com/fricklerhandwerk) +- Winter [**(@winterqt)**](https://github.com/winterqt) +- jade [**(@lf-)**](https://github.com/lf-) +- kirillrdy [**(@kirillrdy)**](https://github.com/kirillrdy) +- pennae [**(@pennae)**](https://github.com/pennae) +- poweredbypie [**(@poweredbypie)**](https://github.com/poweredbypie) +- tomberek [**(@tomberek)**](https://github.com/tomberek) diff --git a/doc/manual/src/release-notes/rl-2.4.md b/doc/manual/src/release-notes/rl-2.4.md index 8b566fc7b..1201e53b6 100644 --- a/doc/manual/src/release-notes/rl-2.4.md +++ b/doc/manual/src/release-notes/rl-2.4.md @@ -23,7 +23,7 @@ more than 2800 commits from 195 contributors since release 2.3. * The **`nix` command** has seen a lot of work and is now almost at feature parity with the old command-line interface (the `nix-*` commands). It aims to be [more modern, consistent and pleasant to - use](../contributing/cli-guideline.md) than the old CLI. It is still + use](../development/cli-guideline.md) than the old CLI. It is still marked as experimental but its interface should not change much anymore in future releases. diff --git a/doc/manual/src/store/file-system-object/content-address.md b/doc/manual/src/store/file-system-object/content-address.md new file mode 100644 index 000000000..410d7fb7c --- /dev/null +++ b/doc/manual/src/store/file-system-object/content-address.md @@ -0,0 +1,85 @@ +# Content-Addressing File System Objects + +For many operations, Nix needs to calculate [a content addresses](@docroot@/glossary.md#gloss-content-address) of [a file system object][file system object]. +Usually this is needed as part of +[content addressing store objects](../store-object/content-address.md), +since store objects always have a root file system object. +But some command-line utilities also just work on "raw" file system objects, not part of any store object. + +Every content addressing scheme Nix uses ultimately involves feeding data into a [hash function](https://en.wikipedia.org/wiki/Hash_function), and getting back an opaque fixed-size digest which is deemed a content address. +The various *methods* of content addressing thus differ in how abstract data (in this case, a file system object and its descendents) are fed into the hash function. + +## Serialising File System Objects { #serial } + +The simplest method is to serialise the entire file system object tree into a single binary string, and then hash that binary string, yielding the content address. +In this section we describe the currently-supported methods of serialising file system objects. + +### Flat { #serial-flat } + +A single file object can just be hashed by its contents. +This is not enough information to encode the fact that the file system object is a file, +but if we *already* know that the FSO is a single non-executable file by other means, it is sufficient. + +Because the hashed data is just the raw file, as is, this choice is good for compatibility with other systems. +For example, Unix commands like `sha256sum` or `sha1sum` will produce hashes for single files that match this. + +### Nix Archive (NAR) { #serial-nix-archive } + +For the other cases of [file system objects][file system object], especially directories with arbitrary descendents, we need a more complex serialisation format. +Examples of such serialisations are the ZIP and TAR file formats. +However, for our purposes these formats have two problems: + +- They do not have a canonical serialisation, meaning that given an FSO, there can +be many different serialisations. + For instance, TAR files can have variable amounts of padding between archive members; + and some archive formats leave the order of directory entries undefined. + This would be bad because we use serialisation to compute cryptographic hashes over file system objects, and for those hashes to be useful as a content address or for integrity checking, uniqueness is crucial. + Otherwise, correct hashes would report false mismatches, and the store would fail to find the content. + +- They store more information than we have in our notion of FSOs, such as time stamps. + This can cause FSOs that Nix should consider equal to hash to different values on different machines, just because the dates differ. + +- As a practical consideration, the TAR format is the only truly universal format in the Unix environment. + It has many problems, such as an inability to deal with long file names and files larger than 2^33 bytes. + Current implementations such as GNU Tar work around these limitations in various ways. + +For these reasons, Nix has its very own archive format—the Nix Archive (NAR) format, +which is carefully designed to avoid the problems described above. + +The exact specification of the Nix Archive format is in `protocols/nix-archive.md` + +## Content addressing File System Objects beyond a single serialisation pass + +Serialising the entire tree and then hashing that binary string is not the only option for content addressing, however. +Another technique is that of a [Merkle graph](https://en.wikipedia.org/wiki/Merkle_tree), where previously computed hashes are included in subsequent byte strings to be hashed. + +In particular, the Merkle graphs can match the original graph structure of file system objects: +we can first hash (serialised) child file system objects, and then hash parent objects using the hashes of their children in the serialisation (to be hashed) of the parent file system objects. + +Currently, there is one such Merkle DAG content addressing method supported. + +### Git ([experimental][xp-feature-git-hashing]) { #git } + +> **Warning** +> +> This method is part of the [`git-hashing`][xp-feature-git-hashing] experimental feature. + +Git's file system model is very close to Nix's, and so Git's content addressing method is a pretty good fit. +Just as with regular Git, files and symlinks are hashed as git "blobs", and directories are hashed as git "trees". + +However, one difference between Nix's and Git's file system model needs special treatment. +Plain files, executable files, and symlinks are not differentiated as distinctly addressable objects, but by their context: by the directory entry that refers to them. +That means so long as the root object is a directory, there is no problem: +every non-directory object is owned by a parent directory, and the entry that refers to it provides the missing information. +However, if the root object is not a directory, then we have no way of knowing which one of an executable file, non-executable file, or symlink it is supposed to be. + +In response to this, we have decided to treat a bare file as non-executable file. +This is similar to do what we do with [flat serialisation](#serial-flat), which also lacks this information. +To avoid an address collision, attempts to hash a bare executable file or symlink will result in an error (just as would happen for flat serialisation also). +Thus, Git can encode some, but not all of Nix's "File System Objects", and this sort of content-addressing is likewise partial. + +In the future, we may support a Git-like hash for such file system objects, or we may adopt another Merkle DAG format which is capable of representing all Nix file system objects. + +[file system object]: ../file-system-object.md +[store object]: ../store-object.md +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/doc/manual/src/store/store-object/content-address.md b/doc/manual/src/store/store-object/content-address.md new file mode 100644 index 000000000..02dce2836 --- /dev/null +++ b/doc/manual/src/store/store-object/content-address.md @@ -0,0 +1,95 @@ +# Content-Addressing Store Objects + +Just [like][fso-ca] [File System Objects][File System Object], +[Store Objects][Store Object] can also be [content-addressed](@docroot@/glossary.md#gloss-content-addressed), +unless they are [input-addressed](@docroot@/glossary.md#gloss-input-addressed-store-object). + +For store objects, the content address we produce will take the form of a [Store Path] rather than regular hash. +In particular, the content-addressing scheme will ensure that the digest of the store path is solely computed from the + +- file system object graph (the root one and its children, if it has any) +- references +- [store directory](../store-path.md#store-directory) +- name + +of the store object, and not any other information, which would not be an intrinsic property of that store object. + +For the full specification of the algorithms involved, see the [specification of store path digests][sp-spec]. + +[File System Object]: ../file-system-object.md +[Store Object]: ../store-object.md +[Store Path]: ../store-path.md + +## Content addressing each part of a store object + +### File System Objects + +With all currently supported store object content addressing methods, the file system object is always [content-addressed][fso-ca] first, and then that hash is incorporated into content address computation for the store object. + +### References + +With all currently supported store object content addressing methods, +other objects are referred to by their regular (string-encoded-) [store paths][Store Path]. + +Self-references however cannot be referred to by their path, because we are in the midst of describing how to compute that path! + +> The alternative would require finding as hash function fixed point, i.e. the solution to an equation in the form +> ``` +> digest = hash(..... || digest || ....) +> ``` +> which is computationally infeasible. +> As far as we know, this is equivalent to finding a hash collision. + +Instead we just have a "has self reference" boolean, which will end up affecting the digest. + +### Name and Store Directory + +These two items affect the digest in a way that is standard for store path digest computations and not specific to content-addressing. +Consult the [specification of store path digests][sp-spec] for further details. + +## Content addressing Methods + +For historical reasons, we don't support all features in all combinations. +Each currently supported method of content addressing chooses a single method of file system object hashing, and may offer some restrictions on references. +The names and store directories are unrestricted however. + +### Flat { #method-flat } + +This uses the corresponding [Flat](../file-system-object/content-address.md#serial-flat) method of file system object content addressing. + +References are not supported: store objects with flat hashing *and* references can not be created. + +### Text { #method-text } + +This also uses the corresponding [Flat](../file-system-object/content-address.md#serial-flat) method of file system object content addressing. + +References to other store objects are supported, but self references are not. + +This is the only store-object content-addressing method that is not named identically with a corresponding file system object method. +It is somewhat obscure, mainly used for "drv files" +(derivations serialized as store objects in their ["ATerm" file format](@docroot@/protocols/derivation-aterm.md)). +Prefer another method if possible. + +### Nix Archive { #method-nix-archive } + +This uses the corresponding [Nix Archive](../file-system-object/content-address.md#serial-nix-archive) method of file system object content addressing. + +References (to other store objects and self references alike) are supported so long as the hash algorithm is SHA-256, but not (neither kind) otherwise. + +### Git { #method-git } + +> **Warning** +> +> This method is part of the [`git-hashing`][xp-feature-git-hashing] experimental feature. + +This uses the corresponding [Git](../file-system-object/content-address.md#serial-git) method of file system object content addressing. + +References are not supported. + +Only SHA-1 is supported at this time. +If [SHA-256-based Git](https://git-scm.com/docs/hash-function-transition) +becomes more widespread, this restriction will be revisited. + +[fso-ca]: ../file-system-object/content-address.md +[sp-spec]: @docroot@/protocols/store-path.md +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/doc/manual/src/store/store-path.md b/doc/manual/src/store/store-path.md index b5ad0c654..beec2389b 100644 --- a/doc/manual/src/store/store-path.md +++ b/doc/manual/src/store/store-path.md @@ -1,5 +1,11 @@ # Store Path +> **Example** +> +> `/nix/store/a040m110amc4h71lds2jmr8qrkj2jhxd-git-2.38.1` +> +> A rendered store path + Nix implements references to [store objects](./index.md#store-object) as *store paths*. Think of a store path as an [opaque], [unique identifier]: @@ -37,6 +43,10 @@ A store path is rendered to a file system path as the concatenation of > store directory digest name > ``` +Exactly how the digest is calculated depends on the type of store path. +Store path digests are *supposed* to be opaque, and so for most operations, it is not necessary to know the details. +That said, the manual has a full [specification of store path digests](@docroot@/protocols/store-path.md). + ## Store Directory Every [Nix store](./index.md) has a store directory. @@ -46,7 +56,7 @@ But if the store has a file system representation, the store directory contains [file system objects]: ./file-system-object.md -This means a store path is not just derived from the referenced store object itself, but depends on the store the store object is in. +This means a store path is not just derived from the referenced store object itself, but depends on the store that the store object is in. > **Note** > diff --git a/flake.lock b/flake.lock index bb2e400c0..2ac413a69 100644 --- a/flake.lock +++ b/flake.lock @@ -3,11 +3,11 @@ "flake-compat": { "flake": false, "locked": { - "lastModified": 1673956053, - "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", "owner": "edolstra", "repo": "flake-compat", - "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", "type": "github" }, "original": { @@ -16,38 +16,100 @@ "type": "github" } }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1719994518, + "narHash": "sha256-pQMhCCHyQGRzdfAkdJ4cIWiw+JNuWsTX7f0ZYSyz0VY=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "9227223f6d922fee3c7b190b2cc238a99527bbb7", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "git-hooks-nix": { + "inputs": { + "flake-compat": [], + "gitignore": [], + "nixpkgs": [ + "nixpkgs" + ], + "nixpkgs-stable": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1721042469, + "narHash": "sha256-6FPUl7HVtvRHCCBQne7Ylp4p+dpP3P/OYuzjztZ4s70=", + "owner": "cachix", + "repo": "git-hooks.nix", + "rev": "f451c19376071a90d8c58ab1a953c6e9840527fd", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "git-hooks.nix", + "type": "github" + } + }, "libgit2": { "flake": false, "locked": { - "lastModified": 1697646580, - "narHash": "sha256-oX4Z3S9WtJlwvj0uH9HlYcWv+x1hqp8mhXl7HsLu2f0=", + "lastModified": 1715853528, + "narHash": "sha256-J2rCxTecyLbbDdsyBWn9w7r3pbKRMkI9E7RvRgAqBdY=", "owner": "libgit2", "repo": "libgit2", - "rev": "45fd9ed7ae1a9b74b957ef4f337bc3c8b3df01b5", + "rev": "36f7e21ad757a3dacc58cf7944329da6bc1d6e96", "type": "github" }, "original": { "owner": "libgit2", + "ref": "v1.8.1", "repo": "libgit2", "type": "github" } }, "nixpkgs": { "locked": { - "lastModified": 1709083642, - "narHash": "sha256-7kkJQd4rZ+vFrzWu8sTRtta5D1kBG0LSRYAfhtmMlSo=", + "lastModified": 1721548954, + "narHash": "sha256-7cCC8+Tdq1+3OPyc3+gVo9dzUNkNIQfwSDJ2HSi2u3o=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "b550fe4b4776908ac2a861124307045f8e717c8e", + "rev": "63d37ccd2d178d54e7fb691d7ec76000740ea24a", "type": "github" }, "original": { "owner": "NixOS", - "ref": "release-23.11", + "ref": "nixos-24.05", "repo": "nixpkgs", "type": "github" } }, + "nixpkgs-23-11": { + "locked": { + "lastModified": 1717159533, + "narHash": "sha256-oamiKNfr2MS6yH64rUn99mIZjc45nGJlj9eGth/3Xuw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446", + "type": "github" + } + }, "nixpkgs-regression": { "locked": { "lastModified": 1643052045, @@ -67,8 +129,11 @@ "root": { "inputs": { "flake-compat": "flake-compat", + "flake-parts": "flake-parts", + "git-hooks-nix": "git-hooks-nix", "libgit2": "libgit2", "nixpkgs": "nixpkgs", + "nixpkgs-23-11": "nixpkgs-23-11", "nixpkgs-regression": "nixpkgs-regression" } } diff --git a/flake.nix b/flake.nix index 42aaace67..9e8592e3a 100644 --- a/flake.nix +++ b/flake.nix @@ -1,18 +1,28 @@ { description = "The purely functional package manager"; - # TODO switch to nixos-23.11-small - # https://nixpk.gs/pr-tracker.html?pr=291954 - inputs.nixpkgs.url = "github:NixOS/nixpkgs/release-23.11"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; + inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446"; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; - inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; }; + inputs.libgit2 = { url = "github:libgit2/libgit2/v1.8.1"; flake = false; }; + + # dev tooling + inputs.flake-parts.url = "github:hercules-ci/flake-parts"; + inputs.git-hooks-nix.url = "github:cachix/git-hooks.nix"; + # work around https://github.com/NixOS/nix/issues/7730 + inputs.flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; + inputs.git-hooks-nix.inputs.nixpkgs.follows = "nixpkgs"; + inputs.git-hooks-nix.inputs.nixpkgs-stable.follows = "nixpkgs"; + # work around 7730 and https://github.com/NixOS/nix/issues/7807 + inputs.git-hooks-nix.inputs.flake-compat.follows = ""; + inputs.git-hooks-nix.inputs.gitignore.follows = ""; + + outputs = inputs@{ self, nixpkgs, nixpkgs-regression, libgit2, ... }: - outputs = { self, nixpkgs, nixpkgs-regression, libgit2, ... }: let inherit (nixpkgs) lib; - inherit (lib) fileset; officialRelease = false; @@ -31,14 +41,9 @@ crossSystems = [ "armv6l-unknown-linux-gnueabihf" "armv7l-unknown-linux-gnueabihf" - "x86_64-unknown-freebsd13" + "riscv64-unknown-linux-gnu" "x86_64-unknown-netbsd" - ]; - - # Nix doesn't yet build on this platform, so we put it in a - # separate list. We just use this for `devShells` and - # `nixpkgsFor`, which this depends on. - shellCrossSystems = crossSystems ++ [ + "x86_64-unknown-freebsd" "x86_64-w64-mingw32" ]; @@ -50,6 +55,18 @@ "stdenv" ]; + /** + `flatMapAttrs attrs f` applies `f` to each attribute in `attrs` and + merges the results into a single attribute set. + + This can be nested to form a build matrix where all the attributes + generated by the innermost `f` are returned as is. + (Provided that the names are unique.) + + See https://nixos.org/manual/nixpkgs/stable/index.html#function-library-lib.attrsets.concatMapAttrs + */ + flatMapAttrs = attrs: f: lib.concatMapAttrs f attrs; + forAllSystems = lib.genAttrs systems; forAllCrossSystems = lib.genAttrs crossSystems; @@ -63,6 +80,17 @@ }) stdenvs); + + # We don't apply flake-parts to the whole flake so that non-development attributes + # load without fetching any development inputs. + devFlake = inputs.flake-parts.lib.mkFlake { inherit inputs; } { + imports = [ ./maintainers/flake-module.nix ]; + systems = lib.subtractLists crossSystems systems; + perSystem = { system, ... }: { + _module.args.pkgs = nixpkgsFor.${system}.native; + }; + }; + # Memoize nixpkgs for different platforms for efficiency. nixpkgsFor = forAllSystems (system: let @@ -84,31 +112,9 @@ in { inherit stdenvs native; static = native.pkgsStatic; - cross = lib.genAttrs shellCrossSystems (crossSystem: make-pkgs crossSystem "stdenv"); + cross = forAllCrossSystems (crossSystem: make-pkgs crossSystem "stdenv"); }); - installScriptFor = tarballs: - nixpkgsFor.x86_64-linux.native.callPackage ./scripts/installer.nix { - inherit tarballs; - }; - - testNixVersions = pkgs: client: daemon: - pkgs.callPackage ./package.nix { - pname = - "nix-tests" - + lib.optionalString - (lib.versionAtLeast daemon.version "2.4pre20211005" && - lib.versionAtLeast client.version "2.4pre20211005") - "-${client.version}-against-${daemon.version}"; - - inherit fileset; - - test-client = client; - test-daemon = daemon; - - doBuild = false; - }; - binaryTarball = nix: pkgs: pkgs.callPackage ./scripts/binary-tarball.nix { inherit nix; }; @@ -120,244 +126,130 @@ { nixStable = prev.nix; - default-busybox-sandbox-shell = final.busybox.override { - useMusl = true; - enableStatic = true; - enableMinimal = true; - extraConfig = '' - CONFIG_FEATURE_FANCY_ECHO y - CONFIG_FEATURE_SH_MATH y - CONFIG_FEATURE_SH_MATH_64 y + # A new scope, so that we can use `callPackage` to inject our own interdependencies + # without "polluting" the top level "`pkgs`" attrset. + # This also has the benefit of providing us with a distinct set of packages + # we can iterate over. + nixComponents = lib.makeScope final.nixDependencies.newScope (import ./packaging/components.nix); - CONFIG_ASH y - CONFIG_ASH_OPTIMIZE_FOR_SIZE y - - CONFIG_ASH_ALIAS y - CONFIG_ASH_BASH_COMPAT y - CONFIG_ASH_CMDCMD y - CONFIG_ASH_ECHO y - CONFIG_ASH_GETOPTS y - CONFIG_ASH_INTERNAL_GLOB y - CONFIG_ASH_JOB_CONTROL y - CONFIG_ASH_PRINTF y - CONFIG_ASH_TEST y - ''; - }; - - libgit2-nix = final.libgit2.overrideAttrs (attrs: { - src = libgit2; - version = libgit2.lastModifiedDate; - cmakeFlags = attrs.cmakeFlags or [] - ++ [ "-DUSE_SSH=exec" ]; + # The dependencies are in their own scope, so that they don't have to be + # in Nixpkgs top level `pkgs` or `nixComponents`. + nixDependencies = lib.makeScope final.newScope (import ./packaging/dependencies.nix { + inherit inputs stdenv versionSuffix; + pkgs = final; }); - boehmgc-nix = (final.boehmgc.override { - enableLargeConfig = true; - }).overrideAttrs(o: { - patches = (o.patches or []) ++ [ - ./dep-patches/boehmgc-coroutine-sp-fallback.diff + nix = final.nixComponents.nix; - # https://github.com/ivmai/bdwgc/pull/586 - ./dep-patches/boehmgc-traceable_allocator-public.diff - ]; - }); - - changelog-d-nix = final.buildPackages.callPackage ./misc/changelog-d.nix { }; - - nix = - let - officialRelease = false; - versionSuffix = - if officialRelease - then "" - else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}"; - - in final.callPackage ./package.nix { - inherit - fileset - stdenv - versionSuffix - ; - officialRelease = false; - boehmgc = final.boehmgc-nix; - libgit2 = final.libgit2-nix; - busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell; - } // { - # this is a proper separate downstream package, but put - # here also for back compat reasons. - perl-bindings = final.nix-perl-bindings; - }; - - nix-perl-bindings = final.callPackage ./perl { - inherit fileset stdenv; + nix_noTests = final.nix.override { + doInstallCheck = false; + doCheck = false; }; + # See https://github.com/NixOS/nixpkgs/pull/214409 + # Remove when fixed in this flake's nixpkgs + pre-commit = + if prev.stdenv.hostPlatform.system == "i686-linux" + then (prev.pre-commit.override (o: { dotnet-sdk = ""; })).overridePythonAttrs (o: { doCheck = false; }) + else prev.pre-commit; + }; in { # A Nixpkgs overlay that overrides the 'nix' and - # 'nix.perl-bindings' packages. + # 'nix-perl-bindings' packages. overlays.default = overlayFor (p: p.stdenv); - hydraJobs = { - - # Binary package for various platforms. - build = forAllSystems (system: self.packages.${system}.nix); - - shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation); - - buildStatic = lib.genAttrs linux64BitSystems (system: self.packages.${system}.nix-static); - - buildCross = forAllCrossSystems (crossSystem: - lib.genAttrs ["x86_64-linux"] (system: self.packages.${system}."nix-${crossSystem}")); - - buildNoGc = forAllSystems (system: - self.packages.${system}.nix.override { enableGC = false; } - ); - - buildNoTests = forAllSystems (system: - self.packages.${system}.nix.override { - doCheck = false; - doInstallCheck = false; - installUnitTests = false; - } - ); - - # Toggles some settings for better coverage. Windows needs these - # library combinations, and Debian build Nix with GNU readline too. - buildReadlineNoMarkdown = forAllSystems (system: - self.packages.${system}.nix.override { - enableMarkdown = false; - readlineFlavor = "readline"; - } - ); - - # Perl bindings for various platforms. - perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nix.perl-bindings); - - # Binary tarball for various platforms, containing a Nix store - # with the closure of 'nix' package, and the second half of - # the installation script. - binaryTarball = forAllSystems (system: binaryTarball nixpkgsFor.${system}.native.nix nixpkgsFor.${system}.native); - - binaryTarballCross = lib.genAttrs ["x86_64-linux"] (system: - forAllCrossSystems (crossSystem: - binaryTarball - self.packages.${system}."nix-${crossSystem}" - nixpkgsFor.${system}.cross.${crossSystem})); - - # The first half of the installation script. This is uploaded - # to https://nixos.org/nix/install. It downloads the binary - # tarball for the user's system and calls the second half of the - # installation script. - installerScript = installScriptFor [ - # Native - self.hydraJobs.binaryTarball."x86_64-linux" - self.hydraJobs.binaryTarball."i686-linux" - self.hydraJobs.binaryTarball."aarch64-linux" - self.hydraJobs.binaryTarball."x86_64-darwin" - self.hydraJobs.binaryTarball."aarch64-darwin" - # Cross - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" - ]; - installerScriptForGHA = installScriptFor [ - # Native - self.hydraJobs.binaryTarball."x86_64-linux" - self.hydraJobs.binaryTarball."x86_64-darwin" - # Cross - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" - ]; - - # docker image with Nix inside - dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); - - # Line coverage analysis. - coverage = nixpkgsFor.x86_64-linux.native.nix.override { - pname = "nix-coverage"; - withCoverageChecks = true; - }; - - # API docs for Nix's unstable internal C++ interfaces. - internal-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ./package.nix { - inherit fileset; - doBuild = false; - enableInternalAPIDocs = true; - }; - - # System tests. - tests = import ./tests/nixos { inherit lib nixpkgs nixpkgsFor; } // { - - # Make sure that nix-env still produces the exact same result - # on a particular version of Nixpkgs. - evalNixpkgs = - let - inherit (nixpkgsFor.x86_64-linux.native) runCommand nix; - in - runCommand "eval-nixos" { buildInputs = [ nix ]; } - '' - type -p nix-env - # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593. - time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages - [[ $(sha1sum < packages | cut -c1-40) = ff451c521e61e4fe72bdbe2d0ca5d1809affa733 ]] - mkdir $out - ''; - - nixpkgsLibTests = - forAllSystems (system: - import (nixpkgs + "/lib/tests/release.nix") - { pkgs = nixpkgsFor.${system}.native; - nixVersions = [ self.packages.${system}.nix ]; - } - ); - }; - - metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" { - pkgs = nixpkgsFor.x86_64-linux.native; - nixpkgs = nixpkgs-regression; - }; - - installTests = forAllSystems (system: - let pkgs = nixpkgsFor.${system}.native; in - pkgs.runCommand "install-tests" { - againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix; - againstCurrentUnstable = - # FIXME: temporarily disable this on macOS because of #3605. - if system == "x86_64-linux" - then testNixVersions pkgs pkgs.nix pkgs.nixUnstable - else null; - # Disabled because the latest stable version doesn't handle - # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work - # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable; - } "touch $out"); - - installerTests = import ./tests/installer { - binaryTarballs = self.hydraJobs.binaryTarball; - inherit nixpkgsFor; - }; - + hydraJobs = import ./packaging/hydra.nix { + inherit + inputs + binaryTarball + forAllCrossSystems + forAllSystems + lib + linux64BitSystems + nixpkgsFor + self + ; }; checks = forAllSystems (system: { binaryTarball = self.hydraJobs.binaryTarball.${system}; - perlBindings = self.hydraJobs.perlBindings.${system}; installTests = self.hydraJobs.installTests.${system}; nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system}; rl-next = let pkgs = nixpkgsFor.${system}.native; in pkgs.buildPackages.runCommand "test-rl-next-release-notes" { } '' - LANG=C.UTF-8 ${pkgs.changelog-d-nix}/bin/changelog-d ${./doc/manual/rl-next} >$out + LANG=C.UTF-8 ${pkgs.changelog-d}/bin/changelog-d ${./doc/manual/rl-next} >$out ''; + repl-completion = nixpkgsFor.${system}.native.callPackage ./tests/repl-completion.nix { }; } // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) { dockerImage = self.hydraJobs.dockerImage.${system}; - }); + } // (lib.optionalAttrs (!(builtins.elem system linux32BitSystems))) { + # Some perl dependencies are broken on i686-linux. + # Since the support is only best-effort there, disable the perl + # bindings - packages = forAllSystems (system: rec { - inherit (nixpkgsFor.${system}.native) nix changelog-d-nix; - default = nix; - } // (lib.optionalAttrs (builtins.elem system linux64BitSystems) { - nix-static = nixpkgsFor.${system}.static.nix; + # Temporarily disabled because GitHub Actions OOM issues. Once + # the old build system is gone and we are back to one build + # system, we should reenable this. + #perlBindings = self.hydraJobs.perlBindings.${system}; + } + # Add "passthru" tests + // flatMapAttrs ({ + "" = nixpkgsFor.${system}.native; + } // lib.optionalAttrs (! nixpkgsFor.${system}.native.stdenv.hostPlatform.isDarwin) { + # TODO: enable static builds for darwin, blocked on: + # https://github.com/NixOS/nixpkgs/issues/320448 + "static-" = nixpkgsFor.${system}.static; + }) + (nixpkgsPrefix: nixpkgs: + flatMapAttrs nixpkgs.nixComponents + (pkgName: pkg: + flatMapAttrs pkg.tests or {} + (testName: test: { + "${nixpkgsPrefix}${pkgName}-${testName}" = test; + }) + ) + ) + // devFlake.checks.${system} or {} + ); + + packages = forAllSystems (system: + { # Here we put attributes that map 1:1 into packages., ie + # for which we don't apply the full build matrix such as cross or static. + inherit (nixpkgsFor.${system}.native) + changelog-d; + default = self.packages.${system}.nix; + nix-internal-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-internal-api-docs; + nix-external-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-external-api-docs; + } + # We need to flatten recursive attribute sets of derivations to pass `flake check`. + // flatMapAttrs + { # Components we'll iterate over in the upcoming lambda + "nix" = { }; + # Temporarily disabled because GitHub Actions OOM issues. Once + # the old build system is gone and we are back to one build + # system, we should reenable these. + #"nix-util" = { }; + #"nix-store" = { }; + #"nix-fetchers" = { }; + } + (pkgName: {}: { + # These attributes go right into `packages.`. + "${pkgName}" = nixpkgsFor.${system}.native.nixComponents.${pkgName}; + "${pkgName}-static" = nixpkgsFor.${system}.static.nixComponents.${pkgName}; + } + // flatMapAttrs (lib.genAttrs crossSystems (_: { })) (crossSystem: {}: { + # These attributes go right into `packages.`. + "${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName}; + }) + // flatMapAttrs (lib.genAttrs stdenvs (_: { })) (stdenvName: {}: { + # These attributes go right into `packages.`. + "${pkgName}-${stdenvName}" = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nixComponents.${pkgName}; + }) + ) + // lib.optionalAttrs (builtins.elem system linux64BitSystems) { dockerImage = let pkgs = nixpkgsFor.${system}.native; @@ -372,21 +264,27 @@ ln -s ${image} $image echo "file binary-dist $image" >> $out/nix-support/hydra-build-products ''; - } // builtins.listToAttrs (map - (crossSystem: { - name = "nix-${crossSystem}"; - value = nixpkgsFor.${system}.cross.${crossSystem}.nix; - }) - crossSystems) - // builtins.listToAttrs (map - (stdenvName: { - name = "nix-${stdenvName}"; - value = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nix; - }) - stdenvs))); + }); devShells = let - makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; forDevShell = true; }).overrideAttrs (attrs: { + makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; forDevShell = true; }).overrideAttrs (attrs: + let + modular = devFlake.getSystem stdenv.buildPlatform.system; + transformFlag = prefix: flag: + assert builtins.isString flag; + let + rest = builtins.substring 2 (builtins.stringLength flag) flag; + in + "-D${prefix}:${rest}"; + havePerl = stdenv.buildPlatform == stdenv.hostPlatform && stdenv.hostPlatform.isUnix; + ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags; + in { + pname = "shell-for-" + attrs.pname; + + # Remove the version suffix to avoid unnecessary attempts to substitute in nix develop + version = lib.fileContents ./.version; + name = attrs.pname; + installFlags = "sysconfdir=$(out)/etc"; shellHook = '' PATH=$prefix/bin:$PATH @@ -397,11 +295,62 @@ XDG_DATA_DIRS+=:$out/share ''; + # We use this shell with the local checkout, not unpackPhase. + src = null; + + env = { + # Needed for Meson to find Boost. + # https://github.com/NixOS/nixpkgs/issues/86131. + BOOST_INCLUDEDIR = "${lib.getDev pkgs.nixDependencies.boost}/include"; + BOOST_LIBRARYDIR = "${lib.getLib pkgs.nixDependencies.boost}/lib"; + # For `make format`, to work without installing pre-commit + _NIX_PRE_COMMIT_HOOKS_CONFIG = + "${(pkgs.formats.yaml { }).generate "pre-commit-config.yaml" modular.pre-commit.settings.rawConfig}"; + }; + + mesonFlags = + map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents.nix-util.mesonFlags) + ++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents.nix-store.mesonFlags) + ++ map (transformFlag "libfetchers") (ignoreCrossFile pkgs.nixComponents.nix-fetchers.mesonFlags) + ++ lib.optionals havePerl (map (transformFlag "perl") (ignoreCrossFile pkgs.nixComponents.nix-perl-bindings.mesonFlags)) + ++ map (transformFlag "libexpr") (ignoreCrossFile pkgs.nixComponents.nix-expr.mesonFlags) + ++ map (transformFlag "libcmd") (ignoreCrossFile pkgs.nixComponents.nix-cmd.mesonFlags) + ; + nativeBuildInputs = attrs.nativeBuildInputs or [] + ++ pkgs.nixComponents.nix-util.nativeBuildInputs + ++ pkgs.nixComponents.nix-store.nativeBuildInputs + ++ pkgs.nixComponents.nix-fetchers.nativeBuildInputs + ++ lib.optionals havePerl pkgs.nixComponents.nix-perl-bindings.nativeBuildInputs + ++ pkgs.nixComponents.nix-internal-api-docs.nativeBuildInputs + ++ pkgs.nixComponents.nix-external-api-docs.nativeBuildInputs + ++ lib.optional + (!stdenv.buildPlatform.canExecute stdenv.hostPlatform + # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479 + && !(stdenv.hostPlatform.isWindows && stdenv.buildPlatform.isDarwin) + && stdenv.hostPlatform.emulatorAvailable pkgs.buildPackages + && lib.meta.availableOn stdenv.buildPlatform (stdenv.hostPlatform.emulator pkgs.buildPackages)) + pkgs.buildPackages.mesonEmulatorHook + ++ [ + pkgs.buildPackages.cmake + pkgs.buildPackages.shellcheck + pkgs.buildPackages.changelog-d + modular.pre-commit.settings.package + (pkgs.writeScriptBin "pre-commit-hooks-install" + modular.pre-commit.settings.installationScript) + ] # TODO: Remove the darwin check once # https://github.com/NixOS/nixpkgs/pull/291814 is available ++ lib.optional (stdenv.cc.isClang && !stdenv.buildPlatform.isDarwin) pkgs.buildPackages.bear ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) pkgs.buildPackages.clang-tools; + + buildInputs = attrs.buildInputs or [] + ++ [ + pkgs.gtest + pkgs.rapidcheck + ] + ++ lib.optional havePerl pkgs.perl + ; }); in forAllSystems (system: @@ -413,8 +362,8 @@ in (makeShells "native" nixpkgsFor.${system}.native) // (lib.optionalAttrs (!nixpkgsFor.${system}.native.stdenv.isDarwin) - (makeShells "static" nixpkgsFor.${system}.static)) // - (lib.genAttrs shellCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv)) // + (makeShells "static" nixpkgsFor.${system}.static) // + (forAllCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv))) // { default = self.devShells.${system}.native-stdenvPackages; } diff --git a/local.mk b/local.mk index 3f3abb9f0..b27c7031e 100644 --- a/local.mk +++ b/local.mk @@ -2,9 +2,14 @@ GLOBAL_CXXFLAGS += -Wno-deprecated-declarations -Werror=switch # Allow switch-enum to be overridden for files that do not support it, usually because of dependency headers. ERROR_SWITCH_ENUM = -Werror=switch-enum -$(foreach i, config.h $(wildcard src/lib*/*.hh), \ +$(foreach i, config.h $(wildcard src/lib*/*.hh) $(filter-out %_internal.h, $(wildcard src/lib*c/*.h)), \ $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) +ifdef HOST_UNIX + $(foreach i, $(wildcard src/lib*/unix/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) +endif + $(GCH): src/libutil/util.hh config.h -GCH_CXXFLAGS = -I src/libutil +GCH_CXXFLAGS = $(INCLUDE_libutil) diff --git a/m4/gcc_bug_80431.m4 b/m4/gcc_bug_80431.m4 index e42f01956..cdc4ddb40 100644 --- a/m4/gcc_bug_80431.m4 +++ b/m4/gcc_bug_80431.m4 @@ -46,11 +46,13 @@ AC_DEFUN([ENSURE_NO_GCC_BUG_80431], ]])], [status_80431=0], [status_80431=$?], - [ - # Assume we're bug-free when cross-compiling - ]) + [status_80431='']) AC_LANG_POP(C++) AS_CASE([$status_80431], + [''],[ + AC_MSG_RESULT(cannot check because cross compiling) + AC_MSG_NOTICE(assume we are bug free) + ], [0],[ AC_MSG_RESULT(yes) ], diff --git a/maintainers/README.md b/maintainers/README.md index fa321c7c0..b92833497 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -30,17 +30,18 @@ We aim to achieve this by improving the contributor experience and attracting mo ## Members - Eelco Dolstra (@edolstra) – Team lead -- Théophane Hufschmitt (@thufschmitt) - Valentin Gagarin (@fricklerhandwerk) - Thomas Bereknyei (@tomberek) - Robert Hensing (@roberth) - John Ericson (@Ericson2314) +The team is on Github as [@NixOS/nix-team](https://github.com/orgs/NixOS/teams/nix-team). + ## Meeting protocol -The team meets twice a week: +The team meets twice a week (times are denoted in the [Europe/Amsterdam](https://en.m.wikipedia.org/wiki/Time_in_the_Netherlands) time zone): -- Discussion meeting: [Fridays 13:00-14:00 CET](https://calendar.google.com/calendar/event?eid=MHNtOGVuNWtrZXNpZHR2bW1sM3QyN2ZjaGNfMjAyMjExMjVUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) +- Discussion meeting: [Wednesday 21:00-22:00 Europe/Amsterdam](https://www.google.com/calendar/event?eid=ZG5rZzNyajRjajducGV2NGY5aGkzYWIwdnJfMjAyNDA1MDhUMTkwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) 1. Triage issues and pull requests from the [No Status](#no-status) column (30 min) 2. Discuss issues and pull requests from the [To discuss](#to-discuss) column (30 min). @@ -49,15 +50,19 @@ The team meets twice a week: - mark it as draft if it is blocked on the contributor - escalate it back to the team by moving it to To discuss, and leaving a comment as to why the issue needs to be discussed again. -- Work meeting: [Mondays 13:00-15:00 CET](https://calendar.google.com/calendar/event?eid=NTM1MG1wNGJnOGpmOTZhYms3bTB1bnY5cWxfMjAyMjExMjFUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) +- Work meeting: [Mondays 14:00-16:00 Europe/Amsterdam](https://www.google.com/calendar/event?eid=Ym52NDdzYnRic2NzcDcybjZiNDhpNzhpa3NfMjAyNDA1MTNUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) 1. Code review on pull requests from [In review](#in-review). 2. Other chores and tasks. Meeting notes are collected on a [collaborative scratchpad](https://pad.lassul.us/Cv7FpYx-Ri-4VjUykQOLAw). -Notes on issues and pull requests are posted as comments and linked from the meeting notes, so they are easy to find from both places. +Notes on issues and pull requests are posted as comments and linked from the meeting notes, so they are can be found from both places. [All meeting notes](https://discourse.nixos.org/search?expanded=true&q=Nix%20team%20meeting%20minutes%20%23%20%23dev%3Anix%20in%3Atitle%20order%3Alatest_topic) are published on Discourse under the [Nix category](https://discourse.nixos.org/c/dev/nix/50). +Team meetings are generally open to anyone interested. +We can make exceptions to discuss sensitive issues, such as security incidents or people matters. +Contact any team member to get a calendar invite for reminders and updates. + ## Project board protocol The team uses a [GitHub project board](https://github.com/orgs/NixOS/projects/19/views/1) for tracking its work. diff --git a/maintainers/data/release-credits-email-to-handle.json b/maintainers/data/release-credits-email-to-handle.json new file mode 100644 index 000000000..cddc1a6e7 --- /dev/null +++ b/maintainers/data/release-credits-email-to-handle.json @@ -0,0 +1,52 @@ +{ + "bogus": "bogus", + "edolstra@gmail.com": "edolstra", + "roberth@users.noreply.github.com": "roberth", + "toscano.pino@tiscali.it": "pinotree", + "valentin@gagarin.work": "fricklerhandwerk", + "mr.trubach@icloud.com": "tie", + "robert@roberthensing.nl": "roberth", + "lix@jade.fyi": "lf-", + "cole.e.helbling@outlook.com": "cole-h", + "joerg@thalheim.io": "Mic92", + "John.Ericson@Obsidian.Systems": "Ericson2314", + "ryan.hendrickson@alum.mit.edu": "rhendric", + "67135060+poweredbypie@users.noreply.github.com": "poweredbypie", + "detroyejr@outlook.com": "detroyejr", + "silvan.mosberger@tweag.io": "infinisil", + "vcs@emily.moe": "emilazy", + "farid.m.zakaria@gmail.com": "fzakaria", + "22859658+RTUnreal@users.noreply.github.com": "RTUnreal", + "me@las.rs": "L-as", + "philip.taron@gmail.com": "philiptaron", + "root@goldstein.rs": "GoldsteinE", + "tomberek@users.noreply.github.com": "tomberek", + "lexi.mattick@neuralink.com": "kognise", + "andrew@johnandrewmarshall.com": "amarshall", + "contact@romain-neil.fr": "romain-neil", + "Mic92@users.noreply.github.com": "Mic92", + "valentin.gagarin@tweag.io": "fricklerhandwerk", + "siddhantk232@gmail.com": "siddhantk232", + "kn@openbsd.org": "klemensn", + "slyich@gmail.com": "trofi", + "theophane.hufschmitt@tweag.io": "thufschmitt", + "alicebob@lijzij.de": "alicebob", + "winter@winter.cafe": "winterqt", + "brian@brianmckenna.org": "puffnfresh", + "git@haenoe.party": "haenoe", + "peshogo@gmail.com": "pineapplehunter", + "poweredbypie@users.noreply.github.com": "poweredbypie", + "arthur200126@gmail.com": "Artoria2e5", + "tomberek@gmail.com": "tomberek", + "jaredbaur@fastmail.com": "jmbaur", + "andreas@rammhold.de": "andir", + "hamirmahal@gmail.com": "hamirmahal", + "git@JohnEricson.me": "Ericson2314", + "8763518+SkamDart@users.noreply.github.com": "SkamDart", + "kirillrdy@gmail.com": "kirillrdy", + "pennae@lix.systems": "pennae", + "delroth@gmail.com": "delroth", + "enno@nerdworks.de": "elohmeier", + "mjbauer95@gmail.com": "matthewbauer", + "MostAwesomeDude@gmail.com": "MostAwesomeDude" +} \ No newline at end of file diff --git a/maintainers/data/release-credits-handle-to-name.json b/maintainers/data/release-credits-handle-to-name.json new file mode 100644 index 000000000..abf9ed05b --- /dev/null +++ b/maintainers/data/release-credits-handle-to-name.json @@ -0,0 +1,45 @@ +{ + "fzakaria": "Farid Zakaria", + "kognise": "Lexi Mattick", + "L-as": "Las Safin", + "haenoe": "HaeNoe", + "andir": "Andreas Rammhold", + "matthewbauer": "Matthew Bauer", + "emilazy": "Emily", + "pineapplehunter": "Shogo Takata", + "RTUnreal": null, + "jmbaur": "Jared Baur", + "Ericson2314": "John Ericson", + "pinotree": "Pino Toscano", + "tie": "Ivan Trubach", + "poweredbypie": null, + "fricklerhandwerk": "Valentin Gagarin", + "Mic92": "J\u00f6rg Thalheim", + "alicebob": "Harmen", + "elohmeier": "Enno Richter", + "delroth": "Pierre Bourdon", + "kirillrdy": null, + "thufschmitt": "Th\u00e9ophane Hufschmitt", + "detroyejr": "Jonathan De Troye", + "klemensn": "Klemens Nanni", + "tomberek": null, + "rhendric": "Ryan Hendrickson", + "philiptaron": "Philip Taron", + "puffnfresh": "Brian McKenna", + "lf-": "jade", + "romain-neil": "Romain Neil", + "hamirmahal": "Hamir Mahal", + "edolstra": "Eelco Dolstra", + "Artoria2e5": "Mingye Wang", + "SkamDart": "Cameron", + "roberth": "Robert Hensing", + "amarshall": "Andrew Marshall", + "trofi": "Sergei Trofimovich", + "cole-h": "Cole Helbling", + "infinisil": "Silvan Mosberger", + "siddhantk232": "Siddhant Kumar", + "winterqt": "Winter", + "GoldsteinE": "Max \u201cGoldstein\u201d Siling", + "pennae": null, + "MostAwesomeDude": "Corbin Simpson" +} \ No newline at end of file diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix new file mode 100644 index 000000000..be91df536 --- /dev/null +++ b/maintainers/flake-module.nix @@ -0,0 +1,677 @@ +{ lib, getSystem, inputs, ... }: + +{ + imports = [ + inputs.git-hooks-nix.flakeModule + ]; + + perSystem = { config, pkgs, ... }: { + + # https://flake.parts/options/pre-commit-hooks-nix.html#options + pre-commit.settings = { + hooks = { + clang-format = { + enable = true; + excludes = [ + # We don't want to format test data + # ''tests/(?!nixos/).*\.nix'' + ''^tests/unit/[^/]*/data/.*$'' + + # Don't format vendored code + ''^doc/manual/redirects\.js$'' + ''^doc/manual/theme/highlight\.js$'' + + # We haven't applied formatting to these files yet + ''^doc/manual/redirects\.js$'' + ''^doc/manual/theme/highlight\.js$'' + ''^precompiled-headers\.h$'' + ''^src/build-remote/build-remote\.cc$'' + ''^src/libcmd/built-path\.cc$'' + ''^src/libcmd/built-path\.hh$'' + ''^src/libcmd/command\.cc$'' + ''^src/libcmd/command\.hh$'' + ''^src/libcmd/common-eval-args\.cc$'' + ''^src/libcmd/common-eval-args\.hh$'' + ''^src/libcmd/editor-for\.cc$'' + ''^src/libcmd/installable-attr-path\.cc$'' + ''^src/libcmd/installable-attr-path\.hh$'' + ''^src/libcmd/installable-derived-path\.cc$'' + ''^src/libcmd/installable-derived-path\.hh$'' + ''^src/libcmd/installable-flake\.cc$'' + ''^src/libcmd/installable-flake\.hh$'' + ''^src/libcmd/installable-value\.cc$'' + ''^src/libcmd/installable-value\.hh$'' + ''^src/libcmd/installables\.cc$'' + ''^src/libcmd/installables\.hh$'' + ''^src/libcmd/legacy\.hh$'' + ''^src/libcmd/markdown\.cc$'' + ''^src/libcmd/misc-store-flags\.cc$'' + ''^src/libcmd/repl-interacter\.cc$'' + ''^src/libcmd/repl-interacter\.hh$'' + ''^src/libcmd/repl\.cc$'' + ''^src/libcmd/repl\.hh$'' + ''^src/libexpr-c/nix_api_expr\.cc$'' + ''^src/libexpr-c/nix_api_external\.cc$'' + ''^src/libexpr/attr-path\.cc$'' + ''^src/libexpr/attr-path\.hh$'' + ''^src/libexpr/attr-set\.cc$'' + ''^src/libexpr/attr-set\.hh$'' + ''^src/libexpr/eval-cache\.cc$'' + ''^src/libexpr/eval-cache\.hh$'' + ''^src/libexpr/eval-error\.cc$'' + ''^src/libexpr/eval-inline\.hh$'' + ''^src/libexpr/eval-settings\.cc$'' + ''^src/libexpr/eval-settings\.hh$'' + ''^src/libexpr/eval\.cc$'' + ''^src/libexpr/eval\.hh$'' + ''^src/libexpr/function-trace\.cc$'' + ''^src/libexpr/gc-small-vector\.hh$'' + ''^src/libexpr/get-drvs\.cc$'' + ''^src/libexpr/get-drvs\.hh$'' + ''^src/libexpr/json-to-value\.cc$'' + ''^src/libexpr/nixexpr\.cc$'' + ''^src/libexpr/nixexpr\.hh$'' + ''^src/libexpr/parser-state\.hh$'' + ''^src/libexpr/pos-table\.hh$'' + ''^src/libexpr/primops\.cc$'' + ''^src/libexpr/primops\.hh$'' + ''^src/libexpr/primops/context\.cc$'' + ''^src/libexpr/primops/fetchClosure\.cc$'' + ''^src/libexpr/primops/fetchMercurial\.cc$'' + ''^src/libexpr/primops/fetchTree\.cc$'' + ''^src/libexpr/primops/fromTOML\.cc$'' + ''^src/libexpr/print-ambiguous\.cc$'' + ''^src/libexpr/print-ambiguous\.hh$'' + ''^src/libexpr/print-options\.hh$'' + ''^src/libexpr/print\.cc$'' + ''^src/libexpr/print\.hh$'' + ''^src/libexpr/search-path\.cc$'' + ''^src/libexpr/symbol-table\.hh$'' + ''^src/libexpr/value-to-json\.cc$'' + ''^src/libexpr/value-to-json\.hh$'' + ''^src/libexpr/value-to-xml\.cc$'' + ''^src/libexpr/value-to-xml\.hh$'' + ''^src/libexpr/value\.hh$'' + ''^src/libexpr/value/context\.cc$'' + ''^src/libexpr/value/context\.hh$'' + ''^src/libfetchers/attrs\.cc$'' + ''^src/libfetchers/cache\.cc$'' + ''^src/libfetchers/cache\.hh$'' + ''^src/libfetchers/fetch-settings\.cc$'' + ''^src/libfetchers/fetch-settings\.hh$'' + ''^src/libfetchers/fetch-to-store\.cc$'' + ''^src/libfetchers/fetchers\.cc$'' + ''^src/libfetchers/fetchers\.hh$'' + ''^src/libfetchers/filtering-source-accessor\.cc$'' + ''^src/libfetchers/filtering-source-accessor\.hh$'' + ''^src/libfetchers/fs-source-accessor\.cc$'' + ''^src/libfetchers/fs-source-accessor\.hh$'' + ''^src/libfetchers/git-utils\.cc$'' + ''^src/libfetchers/git-utils\.hh$'' + ''^src/libfetchers/github\.cc$'' + ''^src/libfetchers/indirect\.cc$'' + ''^src/libfetchers/memory-source-accessor\.cc$'' + ''^src/libfetchers/path\.cc$'' + ''^src/libfetchers/registry\.cc$'' + ''^src/libfetchers/registry\.hh$'' + ''^src/libfetchers/tarball\.cc$'' + ''^src/libfetchers/tarball\.hh$'' + ''^src/libfetchers/git\.cc$'' + ''^src/libfetchers/mercurial\.cc$'' + ''^src/libflake/flake/config\.cc$'' + ''^src/libflake/flake/flake\.cc$'' + ''^src/libflake/flake/flake\.hh$'' + ''^src/libflake/flake/flakeref\.cc$'' + ''^src/libflake/flake/flakeref\.hh$'' + ''^src/libflake/flake/lockfile\.cc$'' + ''^src/libflake/flake/lockfile\.hh$'' + ''^src/libflake/flake/url-name\.cc$'' + ''^src/libmain/common-args\.cc$'' + ''^src/libmain/common-args\.hh$'' + ''^src/libmain/loggers\.cc$'' + ''^src/libmain/loggers\.hh$'' + ''^src/libmain/progress-bar\.cc$'' + ''^src/libmain/shared\.cc$'' + ''^src/libmain/shared\.hh$'' + ''^src/libmain/unix/stack\.cc$'' + ''^src/libstore/binary-cache-store\.cc$'' + ''^src/libstore/binary-cache-store\.hh$'' + ''^src/libstore/build-result\.hh$'' + ''^src/libstore/builtins\.hh$'' + ''^src/libstore/builtins/buildenv\.cc$'' + ''^src/libstore/builtins/buildenv\.hh$'' + ''^src/libstore/common-protocol-impl\.hh$'' + ''^src/libstore/common-protocol\.cc$'' + ''^src/libstore/common-protocol\.hh$'' + ''^src/libstore/common-ssh-store-config\.hh$'' + ''^src/libstore/content-address\.cc$'' + ''^src/libstore/content-address\.hh$'' + ''^src/libstore/daemon\.cc$'' + ''^src/libstore/daemon\.hh$'' + ''^src/libstore/derivations\.cc$'' + ''^src/libstore/derivations\.hh$'' + ''^src/libstore/derived-path-map\.cc$'' + ''^src/libstore/derived-path-map\.hh$'' + ''^src/libstore/derived-path\.cc$'' + ''^src/libstore/derived-path\.hh$'' + ''^src/libstore/downstream-placeholder\.cc$'' + ''^src/libstore/downstream-placeholder\.hh$'' + ''^src/libstore/dummy-store\.cc$'' + ''^src/libstore/export-import\.cc$'' + ''^src/libstore/filetransfer\.cc$'' + ''^src/libstore/filetransfer\.hh$'' + ''^src/libstore/gc-store\.hh$'' + ''^src/libstore/globals\.cc$'' + ''^src/libstore/globals\.hh$'' + ''^src/libstore/http-binary-cache-store\.cc$'' + ''^src/libstore/legacy-ssh-store\.cc$'' + ''^src/libstore/legacy-ssh-store\.hh$'' + ''^src/libstore/length-prefixed-protocol-helper\.hh$'' + ''^src/libstore/linux/personality\.cc$'' + ''^src/libstore/linux/personality\.hh$'' + ''^src/libstore/local-binary-cache-store\.cc$'' + ''^src/libstore/local-fs-store\.cc$'' + ''^src/libstore/local-fs-store\.hh$'' + ''^src/libstore/log-store\.cc$'' + ''^src/libstore/log-store\.hh$'' + ''^src/libstore/machines\.cc$'' + ''^src/libstore/machines\.hh$'' + ''^src/libstore/make-content-addressed\.cc$'' + ''^src/libstore/make-content-addressed\.hh$'' + ''^src/libstore/misc\.cc$'' + ''^src/libstore/names\.cc$'' + ''^src/libstore/names\.hh$'' + ''^src/libstore/nar-accessor\.cc$'' + ''^src/libstore/nar-accessor\.hh$'' + ''^src/libstore/nar-info-disk-cache\.cc$'' + ''^src/libstore/nar-info-disk-cache\.hh$'' + ''^src/libstore/nar-info\.cc$'' + ''^src/libstore/nar-info\.hh$'' + ''^src/libstore/outputs-spec\.cc$'' + ''^src/libstore/outputs-spec\.hh$'' + ''^src/libstore/parsed-derivations\.cc$'' + ''^src/libstore/path-info\.cc$'' + ''^src/libstore/path-info\.hh$'' + ''^src/libstore/path-references\.cc$'' + ''^src/libstore/path-regex\.hh$'' + ''^src/libstore/path-with-outputs\.cc$'' + ''^src/libstore/path\.cc$'' + ''^src/libstore/path\.hh$'' + ''^src/libstore/pathlocks\.cc$'' + ''^src/libstore/pathlocks\.hh$'' + ''^src/libstore/profiles\.cc$'' + ''^src/libstore/profiles\.hh$'' + ''^src/libstore/realisation\.cc$'' + ''^src/libstore/realisation\.hh$'' + ''^src/libstore/remote-fs-accessor\.cc$'' + ''^src/libstore/remote-fs-accessor\.hh$'' + ''^src/libstore/remote-store-connection\.hh$'' + ''^src/libstore/remote-store\.cc$'' + ''^src/libstore/remote-store\.hh$'' + ''^src/libstore/s3-binary-cache-store\.cc$'' + ''^src/libstore/s3\.hh$'' + ''^src/libstore/serve-protocol-impl\.cc$'' + ''^src/libstore/serve-protocol-impl\.hh$'' + ''^src/libstore/serve-protocol\.cc$'' + ''^src/libstore/serve-protocol\.hh$'' + ''^src/libstore/sqlite\.cc$'' + ''^src/libstore/sqlite\.hh$'' + ''^src/libstore/ssh-store\.cc$'' + ''^src/libstore/ssh\.cc$'' + ''^src/libstore/ssh\.hh$'' + ''^src/libstore/store-api\.cc$'' + ''^src/libstore/store-api\.hh$'' + ''^src/libstore/store-dir-config\.hh$'' + ''^src/libstore/build/derivation-goal\.cc$'' + ''^src/libstore/build/derivation-goal\.hh$'' + ''^src/libstore/build/drv-output-substitution-goal\.cc$'' + ''^src/libstore/build/drv-output-substitution-goal\.hh$'' + ''^src/libstore/build/entry-points\.cc$'' + ''^src/libstore/build/goal\.cc$'' + ''^src/libstore/build/goal\.hh$'' + ''^src/libstore/unix/build/hook-instance\.cc$'' + ''^src/libstore/unix/build/local-derivation-goal\.cc$'' + ''^src/libstore/unix/build/local-derivation-goal\.hh$'' + ''^src/libstore/build/substitution-goal\.cc$'' + ''^src/libstore/build/substitution-goal\.hh$'' + ''^src/libstore/build/worker\.cc$'' + ''^src/libstore/build/worker\.hh$'' + ''^src/libstore/builtins/fetchurl\.cc$'' + ''^src/libstore/builtins/unpack-channel\.cc$'' + ''^src/libstore/gc\.cc$'' + ''^src/libstore/local-overlay-store\.cc$'' + ''^src/libstore/local-overlay-store\.hh$'' + ''^src/libstore/local-store\.cc$'' + ''^src/libstore/local-store\.hh$'' + ''^src/libstore/unix/user-lock\.cc$'' + ''^src/libstore/unix/user-lock\.hh$'' + ''^src/libstore/optimise-store\.cc$'' + ''^src/libstore/unix/pathlocks\.cc$'' + ''^src/libstore/posix-fs-canonicalise\.cc$'' + ''^src/libstore/posix-fs-canonicalise\.hh$'' + ''^src/libstore/uds-remote-store\.cc$'' + ''^src/libstore/uds-remote-store\.hh$'' + ''^src/libstore/windows/build\.cc$'' + ''^src/libstore/worker-protocol-impl\.hh$'' + ''^src/libstore/worker-protocol\.cc$'' + ''^src/libstore/worker-protocol\.hh$'' + ''^src/libutil-c/nix_api_util_internal\.h$'' + ''^src/libutil/archive\.cc$'' + ''^src/libutil/archive\.hh$'' + ''^src/libutil/args\.cc$'' + ''^src/libutil/args\.hh$'' + ''^src/libutil/args/root\.hh$'' + ''^src/libutil/callback\.hh$'' + ''^src/libutil/canon-path\.cc$'' + ''^src/libutil/canon-path\.hh$'' + ''^src/libutil/chunked-vector\.hh$'' + ''^src/libutil/closure\.hh$'' + ''^src/libutil/comparator\.hh$'' + ''^src/libutil/compute-levels\.cc$'' + ''^src/libutil/config-impl\.hh$'' + ''^src/libutil/config\.cc$'' + ''^src/libutil/config\.hh$'' + ''^src/libutil/current-process\.cc$'' + ''^src/libutil/current-process\.hh$'' + ''^src/libutil/english\.cc$'' + ''^src/libutil/english\.hh$'' + ''^src/libutil/environment-variables\.cc$'' + ''^src/libutil/error\.cc$'' + ''^src/libutil/error\.hh$'' + ''^src/libutil/exit\.hh$'' + ''^src/libutil/experimental-features\.cc$'' + ''^src/libutil/experimental-features\.hh$'' + ''^src/libutil/file-content-address\.cc$'' + ''^src/libutil/file-content-address\.hh$'' + ''^src/libutil/file-descriptor\.cc$'' + ''^src/libutil/file-descriptor\.hh$'' + ''^src/libutil/file-path-impl\.hh$'' + ''^src/libutil/file-path\.hh$'' + ''^src/libutil/file-system\.cc$'' + ''^src/libutil/file-system\.hh$'' + ''^src/libutil/finally\.hh$'' + ''^src/libutil/fmt\.hh$'' + ''^src/libutil/fs-sink\.cc$'' + ''^src/libutil/fs-sink\.hh$'' + ''^src/libutil/git\.cc$'' + ''^src/libutil/git\.hh$'' + ''^src/libutil/hash\.cc$'' + ''^src/libutil/hash\.hh$'' + ''^src/libutil/hilite\.cc$'' + ''^src/libutil/hilite\.hh$'' + ''^src/libutil/source-accessor\.hh$'' + ''^src/libutil/json-impls\.hh$'' + ''^src/libutil/json-utils\.cc$'' + ''^src/libutil/json-utils\.hh$'' + ''^src/libutil/linux/cgroup\.cc$'' + ''^src/libutil/linux/namespaces\.cc$'' + ''^src/libutil/logging\.cc$'' + ''^src/libutil/logging\.hh$'' + ''^src/libutil/lru-cache\.hh$'' + ''^src/libutil/memory-source-accessor\.cc$'' + ''^src/libutil/memory-source-accessor\.hh$'' + ''^src/libutil/pool\.hh$'' + ''^src/libutil/position\.cc$'' + ''^src/libutil/position\.hh$'' + ''^src/libutil/posix-source-accessor\.cc$'' + ''^src/libutil/posix-source-accessor\.hh$'' + ''^src/libutil/processes\.hh$'' + ''^src/libutil/ref\.hh$'' + ''^src/libutil/references\.cc$'' + ''^src/libutil/references\.hh$'' + ''^src/libutil/regex-combinators\.hh$'' + ''^src/libutil/serialise\.cc$'' + ''^src/libutil/serialise\.hh$'' + ''^src/libutil/signals\.hh$'' + ''^src/libutil/signature/local-keys\.cc$'' + ''^src/libutil/signature/local-keys\.hh$'' + ''^src/libutil/signature/signer\.cc$'' + ''^src/libutil/signature/signer\.hh$'' + ''^src/libutil/source-accessor\.cc$'' + ''^src/libutil/source-accessor\.hh$'' + ''^src/libutil/source-path\.cc$'' + ''^src/libutil/source-path\.hh$'' + ''^src/libutil/split\.hh$'' + ''^src/libutil/suggestions\.cc$'' + ''^src/libutil/suggestions\.hh$'' + ''^src/libutil/sync\.hh$'' + ''^src/libutil/terminal\.cc$'' + ''^src/libutil/terminal\.hh$'' + ''^src/libutil/thread-pool\.cc$'' + ''^src/libutil/thread-pool\.hh$'' + ''^src/libutil/topo-sort\.hh$'' + ''^src/libutil/types\.hh$'' + ''^src/libutil/unix/file-descriptor\.cc$'' + ''^src/libutil/unix/file-path\.cc$'' + ''^src/libutil/unix/monitor-fd\.hh$'' + ''^src/libutil/unix/processes\.cc$'' + ''^src/libutil/unix/signals-impl\.hh$'' + ''^src/libutil/unix/signals\.cc$'' + ''^src/libutil/unix-domain-socket\.cc$'' + ''^src/libutil/unix/users\.cc$'' + ''^src/libutil/url-parts\.hh$'' + ''^src/libutil/url\.cc$'' + ''^src/libutil/url\.hh$'' + ''^src/libutil/users\.cc$'' + ''^src/libutil/users\.hh$'' + ''^src/libutil/util\.cc$'' + ''^src/libutil/util\.hh$'' + ''^src/libutil/variant-wrapper\.hh$'' + ''^src/libutil/windows/environment-variables\.cc$'' + ''^src/libutil/windows/file-descriptor\.cc$'' + ''^src/libutil/windows/file-path\.cc$'' + ''^src/libutil/windows/processes\.cc$'' + ''^src/libutil/windows/users\.cc$'' + ''^src/libutil/windows/windows-error\.cc$'' + ''^src/libutil/windows/windows-error\.hh$'' + ''^src/libutil/xml-writer\.cc$'' + ''^src/libutil/xml-writer\.hh$'' + ''^src/nix-build/nix-build\.cc$'' + ''^src/nix-channel/nix-channel\.cc$'' + ''^src/nix-collect-garbage/nix-collect-garbage\.cc$'' + ''^src/nix-env/buildenv.nix$'' + ''^src/nix-env/nix-env\.cc$'' + ''^src/nix-env/user-env\.cc$'' + ''^src/nix-env/user-env\.hh$'' + ''^src/nix-instantiate/nix-instantiate\.cc$'' + ''^src/nix-store/dotgraph\.cc$'' + ''^src/nix-store/graphml\.cc$'' + ''^src/nix-store/nix-store\.cc$'' + ''^src/nix/add-to-store\.cc$'' + ''^src/nix/app\.cc$'' + ''^src/nix/build\.cc$'' + ''^src/nix/bundle\.cc$'' + ''^src/nix/cat\.cc$'' + ''^src/nix/config-check\.cc$'' + ''^src/nix/config\.cc$'' + ''^src/nix/copy\.cc$'' + ''^src/nix/derivation-add\.cc$'' + ''^src/nix/derivation-show\.cc$'' + ''^src/nix/derivation\.cc$'' + ''^src/nix/develop\.cc$'' + ''^src/nix/diff-closures\.cc$'' + ''^src/nix/dump-path\.cc$'' + ''^src/nix/edit\.cc$'' + ''^src/nix/eval\.cc$'' + ''^src/nix/flake\.cc$'' + ''^src/nix/fmt\.cc$'' + ''^src/nix/hash\.cc$'' + ''^src/nix/log\.cc$'' + ''^src/nix/ls\.cc$'' + ''^src/nix/main\.cc$'' + ''^src/nix/make-content-addressed\.cc$'' + ''^src/nix/nar\.cc$'' + ''^src/nix/optimise-store\.cc$'' + ''^src/nix/path-from-hash-part\.cc$'' + ''^src/nix/path-info\.cc$'' + ''^src/nix/prefetch\.cc$'' + ''^src/nix/profile\.cc$'' + ''^src/nix/realisation\.cc$'' + ''^src/nix/registry\.cc$'' + ''^src/nix/repl\.cc$'' + ''^src/nix/run\.cc$'' + ''^src/nix/run\.hh$'' + ''^src/nix/search\.cc$'' + ''^src/nix/sigs\.cc$'' + ''^src/nix/store-copy-log\.cc$'' + ''^src/nix/store-delete\.cc$'' + ''^src/nix/store-gc\.cc$'' + ''^src/nix/store-info\.cc$'' + ''^src/nix/store-repair\.cc$'' + ''^src/nix/store\.cc$'' + ''^src/nix/unix/daemon\.cc$'' + ''^src/nix/upgrade-nix\.cc$'' + ''^src/nix/verify\.cc$'' + ''^src/nix/why-depends\.cc$'' + + ''^tests/functional/plugins/plugintest\.cc'' + ''^tests/functional/test-libstoreconsumer/main\.cc'' + ''^tests/nixos/ca-fd-leak/sender\.c'' + ''^tests/nixos/ca-fd-leak/smuggler\.c'' + ''^tests/nixos/user-sandboxing/attacker\.c'' + ''^tests/unit/libexpr-support/tests/libexpr\.hh'' + ''^tests/unit/libexpr-support/tests/value/context\.cc'' + ''^tests/unit/libexpr-support/tests/value/context\.hh'' + ''^tests/unit/libexpr/derived-path\.cc'' + ''^tests/unit/libexpr/error_traces\.cc'' + ''^tests/unit/libexpr/eval\.cc'' + ''^tests/unit/libexpr/json\.cc'' + ''^tests/unit/libexpr/main\.cc'' + ''^tests/unit/libexpr/primops\.cc'' + ''^tests/unit/libexpr/search-path\.cc'' + ''^tests/unit/libexpr/trivial\.cc'' + ''^tests/unit/libexpr/value/context\.cc'' + ''^tests/unit/libexpr/value/print\.cc'' + ''^tests/unit/libfetchers/public-key\.cc'' + ''^tests/unit/libflake/flakeref\.cc'' + ''^tests/unit/libflake/url-name\.cc'' + ''^tests/unit/libstore-support/tests/derived-path\.cc'' + ''^tests/unit/libstore-support/tests/derived-path\.hh'' + ''^tests/unit/libstore-support/tests/nix_api_store\.hh'' + ''^tests/unit/libstore-support/tests/outputs-spec\.cc'' + ''^tests/unit/libstore-support/tests/outputs-spec\.hh'' + ''^tests/unit/libstore-support/tests/path\.cc'' + ''^tests/unit/libstore-support/tests/path\.hh'' + ''^tests/unit/libstore-support/tests/protocol\.hh'' + ''^tests/unit/libstore/common-protocol\.cc'' + ''^tests/unit/libstore/content-address\.cc'' + ''^tests/unit/libstore/derivation\.cc'' + ''^tests/unit/libstore/derived-path\.cc'' + ''^tests/unit/libstore/downstream-placeholder\.cc'' + ''^tests/unit/libstore/machines\.cc'' + ''^tests/unit/libstore/nar-info-disk-cache\.cc'' + ''^tests/unit/libstore/nar-info\.cc'' + ''^tests/unit/libstore/outputs-spec\.cc'' + ''^tests/unit/libstore/path-info\.cc'' + ''^tests/unit/libstore/path\.cc'' + ''^tests/unit/libstore/serve-protocol\.cc'' + ''^tests/unit/libstore/worker-protocol\.cc'' + ''^tests/unit/libutil-support/tests/characterization\.hh'' + ''^tests/unit/libutil-support/tests/hash\.cc'' + ''^tests/unit/libutil-support/tests/hash\.hh'' + ''^tests/unit/libutil/args\.cc'' + ''^tests/unit/libutil/canon-path\.cc'' + ''^tests/unit/libutil/chunked-vector\.cc'' + ''^tests/unit/libutil/closure\.cc'' + ''^tests/unit/libutil/compression\.cc'' + ''^tests/unit/libutil/config\.cc'' + ''^tests/unit/libutil/file-content-address\.cc'' + ''^tests/unit/libutil/git\.cc'' + ''^tests/unit/libutil/hash\.cc'' + ''^tests/unit/libutil/hilite\.cc'' + ''^tests/unit/libutil/json-utils\.cc'' + ''^tests/unit/libutil/logging\.cc'' + ''^tests/unit/libutil/lru-cache\.cc'' + ''^tests/unit/libutil/pool\.cc'' + ''^tests/unit/libutil/references\.cc'' + ''^tests/unit/libutil/suggestions\.cc'' + ''^tests/unit/libutil/tests\.cc'' + ''^tests/unit/libutil/url\.cc'' + ''^tests/unit/libutil/xml-writer\.cc'' + ]; + }; + shellcheck = { + enable = true; + excludes = [ + # We haven't linted these files yet + ''^config/install-sh$'' + ''^misc/bash/completion\.sh$'' + ''^misc/fish/completion\.fish$'' + ''^misc/zsh/completion\.zsh$'' + ''^scripts/create-darwin-volume\.sh$'' + ''^scripts/install-darwin-multi-user\.sh$'' + ''^scripts/install-multi-user\.sh$'' + ''^scripts/install-nix-from-closure\.sh$'' + ''^scripts/install-systemd-multi-user\.sh$'' + ''^src/nix/get-env\.sh$'' + ''^tests/functional/build\.sh$'' + ''^tests/functional/ca/build-dry\.sh$'' + ''^tests/functional/ca/build-with-garbage-path\.sh$'' + ''^tests/functional/ca/common\.sh$'' + ''^tests/functional/ca/concurrent-builds\.sh$'' + ''^tests/functional/ca/eval-store\.sh$'' + ''^tests/functional/ca/gc\.sh$'' + ''^tests/functional/ca/import-derivation\.sh$'' + ''^tests/functional/ca/new-build-cmd\.sh$'' + ''^tests/functional/ca/nix-shell\.sh$'' + ''^tests/functional/ca/post-hook\.sh$'' + ''^tests/functional/ca/recursive\.sh$'' + ''^tests/functional/ca/repl\.sh$'' + ''^tests/functional/ca/selfref-gc\.sh$'' + ''^tests/functional/ca/why-depends\.sh$'' + ''^tests/functional/characterisation-test-infra\.sh$'' + ''^tests/functional/check\.sh$'' + ''^tests/functional/common/vars-and-functions\.sh$'' + ''^tests/functional/completions\.sh$'' + ''^tests/functional/compute-levels\.sh$'' + ''^tests/functional/config\.sh$'' + ''^tests/functional/db-migration\.sh$'' + ''^tests/functional/debugger\.sh$'' + ''^tests/functional/dependencies\.builder0\.sh$'' + ''^tests/functional/dependencies\.sh$'' + ''^tests/functional/dump-db\.sh$'' + ''^tests/functional/dyn-drv/build-built-drv\.sh$'' + ''^tests/functional/dyn-drv/common\.sh$'' + ''^tests/functional/dyn-drv/dep-built-drv\.sh$'' + ''^tests/functional/dyn-drv/eval-outputOf\.sh$'' + ''^tests/functional/dyn-drv/old-daemon-error-hack\.sh$'' + ''^tests/functional/dyn-drv/recursive-mod-json\.sh$'' + ''^tests/functional/eval-store\.sh$'' + ''^tests/functional/eval\.sh$'' + ''^tests/functional/export-graph\.sh$'' + ''^tests/functional/export\.sh$'' + ''^tests/functional/extra-sandbox-profile\.sh$'' + ''^tests/functional/fetchClosure\.sh$'' + ''^tests/functional/fetchGit\.sh$'' + ''^tests/functional/fetchGitRefs\.sh$'' + ''^tests/functional/fetchGitSubmodules\.sh$'' + ''^tests/functional/fetchGitVerification\.sh$'' + ''^tests/functional/fetchMercurial\.sh$'' + ''^tests/functional/fetchurl\.sh$'' + ''^tests/functional/fixed\.builder1\.sh$'' + ''^tests/functional/fixed\.builder2\.sh$'' + ''^tests/functional/fixed\.sh$'' + ''^tests/functional/flakes/absolute-paths\.sh$'' + ''^tests/functional/flakes/check\.sh$'' + ''^tests/functional/flakes/common\.sh$'' + ''^tests/functional/flakes/config\.sh$'' + ''^tests/functional/flakes/develop\.sh$'' + ''^tests/functional/flakes/flakes\.sh$'' + ''^tests/functional/flakes/follow-paths\.sh$'' + ''^tests/functional/flakes/prefetch\.sh$'' + ''^tests/functional/flakes/run\.sh$'' + ''^tests/functional/flakes/show\.sh$'' + ''^tests/functional/fmt\.sh$'' + ''^tests/functional/fmt\.simple\.sh$'' + ''^tests/functional/gc-auto\.sh$'' + ''^tests/functional/gc-concurrent\.builder\.sh$'' + ''^tests/functional/gc-concurrent\.sh$'' + ''^tests/functional/gc-concurrent2\.builder\.sh$'' + ''^tests/functional/gc-non-blocking\.sh$'' + ''^tests/functional/gc\.sh$'' + ''^tests/functional/git-hashing/common\.sh$'' + ''^tests/functional/git-hashing/simple\.sh$'' + ''^tests/functional/hash-convert\.sh$'' + ''^tests/functional/help\.sh$'' + ''^tests/functional/impure-derivations\.sh$'' + ''^tests/functional/impure-env\.sh$'' + ''^tests/functional/impure-eval\.sh$'' + ''^tests/functional/install-darwin\.sh$'' + ''^tests/functional/lang\.sh$'' + ''^tests/functional/legacy-ssh-store\.sh$'' + ''^tests/functional/linux-sandbox\.sh$'' + ''^tests/functional/local-overlay-store/add-lower-inner\.sh$'' + ''^tests/functional/local-overlay-store/add-lower\.sh$'' + ''^tests/functional/local-overlay-store/bad-uris\.sh$'' + ''^tests/functional/local-overlay-store/build-inner\.sh$'' + ''^tests/functional/local-overlay-store/build\.sh$'' + ''^tests/functional/local-overlay-store/check-post-init-inner\.sh$'' + ''^tests/functional/local-overlay-store/check-post-init\.sh$'' + ''^tests/functional/local-overlay-store/common\.sh$'' + ''^tests/functional/local-overlay-store/delete-duplicate-inner\.sh$'' + ''^tests/functional/local-overlay-store/delete-duplicate\.sh$'' + ''^tests/functional/local-overlay-store/delete-refs-inner\.sh$'' + ''^tests/functional/local-overlay-store/delete-refs\.sh$'' + ''^tests/functional/local-overlay-store/gc-inner\.sh$'' + ''^tests/functional/local-overlay-store/gc\.sh$'' + ''^tests/functional/local-overlay-store/optimise-inner\.sh$'' + ''^tests/functional/local-overlay-store/optimise\.sh$'' + ''^tests/functional/local-overlay-store/redundant-add-inner\.sh$'' + ''^tests/functional/local-overlay-store/redundant-add\.sh$'' + ''^tests/functional/local-overlay-store/remount\.sh$'' + ''^tests/functional/local-overlay-store/stale-file-handle-inner\.sh$'' + ''^tests/functional/local-overlay-store/stale-file-handle\.sh$'' + ''^tests/functional/local-overlay-store/verify-inner\.sh$'' + ''^tests/functional/local-overlay-store/verify\.sh$'' + ''^tests/functional/logging\.sh$'' + ''^tests/functional/misc\.sh$'' + ''^tests/functional/multiple-outputs\.sh$'' + ''^tests/functional/nar-access\.sh$'' + ''^tests/functional/nested-sandboxing\.sh$'' + ''^tests/functional/nested-sandboxing/command\.sh$'' + ''^tests/functional/nix-build\.sh$'' + ''^tests/functional/nix-channel\.sh$'' + ''^tests/functional/nix-collect-garbage-d\.sh$'' + ''^tests/functional/nix-copy-ssh-common\.sh$'' + ''^tests/functional/nix-copy-ssh-ng\.sh$'' + ''^tests/functional/nix-copy-ssh\.sh$'' + ''^tests/functional/nix-daemon-untrusting\.sh$'' + ''^tests/functional/nix-profile\.sh$'' + ''^tests/functional/nix-shell\.sh$'' + ''^tests/functional/nix_path\.sh$'' + ''^tests/functional/optimise-store\.sh$'' + ''^tests/functional/output-normalization\.sh$'' + ''^tests/functional/parallel\.builder\.sh$'' + ''^tests/functional/parallel\.sh$'' + ''^tests/functional/pass-as-file\.sh$'' + ''^tests/functional/path-from-hash-part\.sh$'' + ''^tests/functional/path-info\.sh$'' + ''^tests/functional/placeholders\.sh$'' + ''^tests/functional/plugins\.sh$'' + ''^tests/functional/post-hook\.sh$'' + ''^tests/functional/pure-eval\.sh$'' + ''^tests/functional/push-to-store-old\.sh$'' + ''^tests/functional/push-to-store\.sh$'' + ''^tests/functional/read-only-store\.sh$'' + ''^tests/functional/readfile-context\.sh$'' + ''^tests/functional/recursive\.sh$'' + ''^tests/functional/referrers\.sh$'' + ''^tests/functional/remote-store\.sh$'' + ''^tests/functional/repair\.sh$'' + ''^tests/functional/restricted\.sh$'' + ''^tests/functional/search\.sh$'' + ''^tests/functional/secure-drv-outputs\.sh$'' + ''^tests/functional/selfref-gc\.sh$'' + ''^tests/functional/shell\.sh$'' + ''^tests/functional/shell\.shebang\.sh$'' + ''^tests/functional/signing\.sh$'' + ''^tests/functional/simple\.builder\.sh$'' + ''^tests/functional/simple\.sh$'' + ''^tests/functional/ssh-relay\.sh$'' + ''^tests/functional/store-info\.sh$'' + ''^tests/functional/structured-attrs\.sh$'' + ''^tests/functional/substitute-with-invalid-ca\.sh$'' + ''^tests/functional/suggestions\.sh$'' + ''^tests/functional/supplementary-groups\.sh$'' + ''^tests/functional/tarball\.sh$'' + ''^tests/functional/test-infra\.sh$'' + ''^tests/functional/test-libstoreconsumer\.sh$'' + ''^tests/functional/timeout\.sh$'' + ''^tests/functional/toString-path\.sh$'' + ''^tests/functional/user-envs-migration\.sh$'' + ''^tests/functional/user-envs-test-case\.sh$'' + ''^tests/functional/user-envs\.builder\.sh$'' + ''^tests/functional/user-envs\.sh$'' + ''^tests/functional/why-depends\.sh$'' + ''^tests/functional/zstd\.sh$'' + ''^tests/unit/libutil/data/git/check-data\.sh$'' + ]; + }; + # TODO: nixfmt, https://github.com/NixOS/nixfmt/issues/153 + }; + }; + }; + + # We'll be pulling from this in the main flake + flake.getSystem = getSystem; +} diff --git a/maintainers/local.mk b/maintainers/local.mk new file mode 100644 index 000000000..88d594d67 --- /dev/null +++ b/maintainers/local.mk @@ -0,0 +1,15 @@ + +.PHONY: format +print-top-help += echo ' format: Format source code' + +# This uses the cached .pre-commit-hooks.yaml file +format: + @if ! type -p pre-commit &>/dev/null; then \ + echo "make format: pre-commit not found. Please use \`nix develop\`."; \ + exit 1; \ + fi; \ + if test -z "$$_NIX_PRE_COMMIT_HOOKS_CONFIG"; then \ + echo "make format: _NIX_PRE_COMMIT_HOOKS_CONFIG not set. Please use \`nix develop\`."; \ + exit 1; \ + fi; \ + pre-commit run --config $$_NIX_PRE_COMMIT_HOOKS_CONFIG --all-files diff --git a/maintainers/release-credits b/maintainers/release-credits new file mode 100755 index 000000000..7a5c87d7d --- /dev/null +++ b/maintainers/release-credits @@ -0,0 +1,184 @@ +#!/usr/bin/env nix +# vim: set filetype=python: +#!nix develop --impure --expr +#!nix `` +#!nix let flake = builtins.getFlake ("git+file://" + toString ../.); +#!nix pkgs = flake.inputs.nixpkgs.legacyPackages.${builtins.currentSystem}; +#!nix in pkgs.mkShell { nativeBuildInputs = [ +#!nix (pkgs.python3.withPackages (ps: with ps; [ requests ])) +#!nix ]; } +#!nix `` --command python3 + +# This script lists out the contributors for a given release. +# It must be run from the root of the Nix repository. + +import os +import sys +import json +import requests + +github_token = os.environ.get("GITHUB_TOKEN") +if not github_token: + print("GITHUB_TOKEN is not set. If you hit the rate limit, set it", file=sys.stderr) + # Might be ok, as we have a cache. + # raise ValueError("GITHUB_TOKEN must be set") + +# 1. Read the current version in .version +version = os.environ.get("VERSION") +if not version: + version = open(".version").read().strip() + +print(f"Generating release credits for Nix {version}", file=sys.stderr) + +# 2. Compute previous version +vcomponents = version.split(".") +if len(vcomponents) >= 2: + prev_version = f"{vcomponents[0]}.{int(vcomponents[1])-1}.0" +else: + raise ValueError(".version must have at least two components") + +# For unreleased versions +endref = "HEAD" +# For older releases +# endref = version + +# 2. Find the merge base between the current version and the previous version +mergeBase = os.popen(f"git merge-base {prev_version} {endref}").read().strip() +print(f"Merge base between {prev_version} and {endref} is {mergeBase}", file=sys.stderr) + +# 3. Find the date of the merge base +mergeBaseDate = os.popen(f"git show -s --format=%ci {mergeBase}").read().strip()[0:10] +print(f"Merge base date is {mergeBaseDate}", file=sys.stderr) + +# 4. Get the commits between the merge base and the current version + +def get_commits(): + raw = os.popen(f"git log --pretty=format:'%H\t%an\t%ae' {mergeBase}..{endref}").read().strip() + lines = raw.split("\n") + return [ { "hash": items[0], "author": items[1], "email": items[2] } + for line in lines + for items in (line.split("\t"),) + ] + +def commits_to_first_commit_by_email(commits): + by_email = dict() + for commit in commits: + email = commit["email"] + if email not in by_email: + by_email[email] = commit + return by_email + + +samples = commits_to_first_commit_by_email(get_commits()) + +# For quick testing, only pick two samples from the dict +# samples = dict(list(samples.items())[:2]) + +# Query the GitHub API to get handle +def get_github_commit(commit): + url = f"https://api.github.com/repos/NixOS/nix/commits/{commit['hash']}" + headers = {'Authorization': f'token {github_token}'} + response = requests.get(url, headers=headers) + response.raise_for_status() + return response.json() + +class Cache: + def __init__(self, filename, require = True): + self.filename = filename + try: + with open(filename, "r") as f: + self.values = json.load(f) + except FileNotFoundError: + if require: + raise + self.values = dict() + def save(self): + with open(self.filename, "w") as f: + json.dump(self.values, f, indent=4) + print(f"Saved cache to {self.filename}", file=sys.stderr) + +# The email to handle cache maps email addresses to either +# - a handle (string) +# - None (if no handle was found) +email_to_handle_cache = Cache("maintainers/data/release-credits-email-to-handle.json") + +handles = set() +emails = dict() + +for sample in samples: + s = samples[sample] + email = s["email"] + if not email in email_to_handle_cache.values: + print(f"Querying GitHub API for {s['hash']}, to get handle for {s['email']}") + ghc = get_github_commit(samples[sample]) + gha = ghc["author"] + if gha and gha["login"]: + handle = gha["login"] + print(f"Handle: {handle}") + email_to_handle_cache.values[email] = handle + else: + print(f"Found no handle for {s['email']}") + email_to_handle_cache.values[email] = None + handle = email_to_handle_cache.values[email] + if handle is not None: + handles.add(handle) + else: + emails[email] = s["author"] + +# print(email_to_handle_cache.values) + +email_to_handle_cache.save() + +handle_to_name_cache = Cache("maintainers/data/release-credits-handle-to-name.json") + +print(f"Found {len(handles)} handles", file=sys.stderr) + +for handle in handles: + if not handle in handle_to_name_cache.values: + print(f"Querying GitHub API for {handle}, to get name", file=sys.stderr) + url = f"https://api.github.com/users/{handle}" + headers = {'Authorization': f'token {github_token}'} + response = requests.get(url, headers=headers) + response.raise_for_status() + user = response.json() + name = user["name"] + print(f"Name: {name}", file=sys.stderr) + handle_to_name_cache.values[handle] = name + +handle_to_name_cache.save() + +entries = list() + +def is_bot(handle): + return handle.endswith("[bot]") + +for handle in handles: + name = handle_to_name_cache.values[handle] + if name is None: + # This way it looks more regular + name = handle + + # Note to bots: I'm only doing this because it would distract humans, and + # therefore delay your creation. I'm sorry; it's for your benefit. + if not is_bot(handle): + entries += [ f"- {name} [**(@{handle})**](https://github.com/{handle})" ] + +def shuffle(entries): + salt = os.urandom(16) + return sorted(entries, key=lambda x: hash((x, salt))) + +# Fair ordering is undecidable +entries = shuffle(entries) + +# For a sanity check, we could sort the entries by handle instead. +# entries = sorted(entries) + +print("") +print(f"This release was made possible by the following {len(entries)} contributors:") +print("") + +for entry in entries: + print(entry) + +for email in emails: + print(f"- {emails[email]}") diff --git a/maintainers/release-notes b/maintainers/release-notes index 2d84485c1..c0c4ee734 100755 --- a/maintainers/release-notes +++ b/maintainers/release-notes @@ -1,5 +1,6 @@ #!/usr/bin/env nix -#!nix shell .#changelog-d-nix --command bash +# vim: set filetype=bash: +#!nix shell .#changelog-d --command bash # --- CONFIGURATION --- @@ -151,6 +152,13 @@ section_title="Release $version_full ($DATE)" echo "# $section_title" echo changelog-d doc/manual/rl-next | sed -e 's/ *$//' + + if ! $IS_PATCH; then + echo + echo "# Contributors" + echo + VERSION=$version_full ./maintainers/release-credits + fi ) | tee -a $file log "Wrote $file" diff --git a/maintainers/release-process.md b/maintainers/release-process.md index da6886ea9..7a2b3c0a7 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -39,6 +39,10 @@ release: * Proof-read / edit / rearrange the release notes if needed. Breaking changes and highlights should go to the top. +* Run `maintainers/release-credits` to make sure the credits script works + and produces a sensible output. Some emails might not automatically map to + a GitHub handle. + * Push. ```console diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 4e2c379f0..731988568 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -11,6 +11,8 @@ use JSON::PP; use LWP::UserAgent; use Net::Amazon::S3; +delete $ENV{'shell'}; # shut up a LWP::UserAgent.pm warning + my $evalId = $ARGV[0] or die "Usage: $0 EVAL-ID\n"; my $releasesBucketName = "nix-releases"; @@ -36,11 +38,11 @@ sub fetch { my $evalUrl = "https://hydra.nixos.org/eval/$evalId"; my $evalInfo = decode_json(fetch($evalUrl, 'application/json')); #print Dumper($evalInfo); -my $flakeUrl = $evalInfo->{flake} or die; -my $flakeInfo = decode_json(`nix flake metadata --json "$flakeUrl"` or die); -my $nixRev = $flakeInfo->{revision} or die; +my $flakeUrl = $evalInfo->{flake}; +my $flakeInfo = decode_json(`nix flake metadata --json "$flakeUrl"` or die) if $flakeUrl; +my $nixRev = ($flakeInfo ? $flakeInfo->{revision} : $evalInfo->{jobsetevalinputs}->{nix}->{revision}) or die; -my $buildInfo = decode_json(fetch("$evalUrl/job/build.x86_64-linux", 'application/json')); +my $buildInfo = decode_json(fetch("$evalUrl/job/build.nix.x86_64-linux", 'application/json')); #print Dumper($buildInfo); my $releaseName = $buildInfo->{nixname}; @@ -83,12 +85,19 @@ my $channelsBucket = $s3_us->bucket($channelsBucketName) or die; sub getStorePath { my ($jobName, $output) = @_; my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json')); - return $buildInfo->{buildoutputs}->{$output or "out"}->{path} or die "cannot get store path for '$jobName'"; + return $buildInfo->{buildoutputs}->{$output or "out"}->{path} // die "cannot get store path for '$jobName'"; } sub copyManual { - my $manual = getStorePath("build.x86_64-linux", "doc"); - print "$manual\n"; + my $manual; + eval { + $manual = getStorePath("build.nix.x86_64-linux", "doc"); + }; + if ($@) { + warn "$@"; + return; + } + print "Manual: $manual\n"; my $manualNar = "$tmpDir/$releaseName-manual.nar.xz"; print "$manualNar\n"; @@ -154,19 +163,34 @@ downloadFile("binaryTarball.x86_64-linux", "1"); downloadFile("binaryTarball.aarch64-linux", "1"); downloadFile("binaryTarball.x86_64-darwin", "1"); downloadFile("binaryTarball.aarch64-darwin", "1"); -downloadFile("binaryTarballCross.x86_64-linux.armv6l-unknown-linux-gnueabihf", "1"); -downloadFile("binaryTarballCross.x86_64-linux.armv7l-unknown-linux-gnueabihf", "1"); +eval { + downloadFile("binaryTarballCross.x86_64-linux.armv6l-unknown-linux-gnueabihf", "1"); +}; +warn "$@" if $@; +eval { + downloadFile("binaryTarballCross.x86_64-linux.armv7l-unknown-linux-gnueabihf", "1"); +}; +warn "$@" if $@; +eval { + downloadFile("binaryTarballCross.x86_64-linux.riscv64-unknown-linux-gnu", "1"); +}; +warn "$@" if $@; downloadFile("installerScript", "1"); # Upload docker images to dockerhub. my $dockerManifest = ""; my $dockerManifestLatest = ""; +my $haveDocker = 0; for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) { my $system = $platforms->[0]; my $dockerPlatform = $platforms->[1]; my $fn = "nix-$version-docker-image-$dockerPlatform.tar.gz"; - downloadFile("dockerImage.$system", "1", $fn); + eval { + downloadFile("dockerImage.$system", "1", $fn); + }; + die "$@" if $@; + $haveDocker = 1; print STDERR "loading docker image for $dockerPlatform...\n"; system("docker load -i $tmpDir/$fn") == 0 or die; @@ -194,31 +218,34 @@ for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) { $dockerManifestLatest .= " --amend $latestTag" } -print STDERR "creating multi-platform docker manifest...\n"; -system("docker manifest rm nixos/nix:$version"); -system("docker manifest create nixos/nix:$version $dockerManifest") == 0 or die; -if ($isLatest) { - print STDERR "creating latest multi-platform docker manifest...\n"; - system("docker manifest rm nixos/nix:latest"); - system("docker manifest create nixos/nix:latest $dockerManifestLatest") == 0 or die; -} +if ($haveDocker) { + print STDERR "creating multi-platform docker manifest...\n"; + system("docker manifest rm nixos/nix:$version"); + system("docker manifest create nixos/nix:$version $dockerManifest") == 0 or die; + if ($isLatest) { + print STDERR "creating latest multi-platform docker manifest...\n"; + system("docker manifest rm nixos/nix:latest"); + system("docker manifest create nixos/nix:latest $dockerManifestLatest") == 0 or die; + } -print STDERR "pushing multi-platform docker manifest...\n"; -system("docker manifest push nixos/nix:$version") == 0 or die; + print STDERR "pushing multi-platform docker manifest...\n"; + system("docker manifest push nixos/nix:$version") == 0 or die; -if ($isLatest) { - print STDERR "pushing latest multi-platform docker manifest...\n"; - system("docker manifest push nixos/nix:latest") == 0 or die; + if ($isLatest) { + print STDERR "pushing latest multi-platform docker manifest...\n"; + system("docker manifest push nixos/nix:latest") == 0 or die; + } } # Upload nix-fallback-paths.nix. write_file("$tmpDir/fallback-paths.nix", "{\n" . - " x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" . - " i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" . - " aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" . - " x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" . - " aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" . + " x86_64-linux = \"" . getStorePath("build.nix.x86_64-linux") . "\";\n" . + " i686-linux = \"" . getStorePath("build.nix.i686-linux") . "\";\n" . + " aarch64-linux = \"" . getStorePath("build.nix.aarch64-linux") . "\";\n" . + " riscv64-linux = \"" . getStorePath("buildCross.nix.riscv64-unknown-linux-gnu.x86_64-linux") . "\";\n" . + " x86_64-darwin = \"" . getStorePath("build.nix.x86_64-darwin") . "\";\n" . + " aarch64-darwin = \"" . getStorePath("build.nix.aarch64-darwin") . "\";\n" . "}\n"); # Upload release files to S3. diff --git a/meson.build b/meson.build new file mode 100644 index 000000000..1554244ab --- /dev/null +++ b/meson.build @@ -0,0 +1,44 @@ +# This is just a stub project to include all the others as subprojects +# for development shell purposes + +project('nix-dev-shell', 'cpp', + version : files('.version'), + subproject_dir : 'src', +) + +# Internal Libraries +subproject('libutil') +subproject('libstore') +subproject('libfetchers') +subproject('libexpr') +subproject('libflake') +subproject('libmain') +subproject('libcmd') + +# Executables +subproject('nix') + +# Docs +subproject('internal-api-docs') +subproject('external-api-docs') + +# External C wrapper libraries +subproject('libutil-c') +subproject('libstore-c') +subproject('libexpr-c') +subproject('libmain-c') + +# Language Bindings +if not meson.is_cross_build() + subproject('perl') +endif + +# Testing +subproject('nix-util-test-support') +subproject('nix-util-tests') +subproject('nix-store-test-support') +subproject('nix-store-tests') +subproject('nix-fetchers-tests') +subproject('nix-expr-test-support') +subproject('nix-expr-tests') +subproject('nix-flake-tests') diff --git a/misc/changelog-d.cabal.nix b/misc/changelog-d.cabal.nix deleted file mode 100644 index 76f9353cd..000000000 --- a/misc/changelog-d.cabal.nix +++ /dev/null @@ -1,31 +0,0 @@ -{ mkDerivation, aeson, base, bytestring, cabal-install-parsers -, Cabal-syntax, containers, directory, filepath, frontmatter -, generic-lens-lite, lib, mtl, optparse-applicative, parsec, pretty -, regex-applicative, text, pkgs -}: -let rev = "f30f6969e9cd8b56242309639d58acea21c99d06"; -in -mkDerivation { - pname = "changelog-d"; - version = "0.1"; - src = pkgs.fetchurl { - name = "changelog-d-${rev}.tar.gz"; - url = "https://codeberg.org/roberth/changelog-d/archive/${rev}.tar.gz"; - hash = "sha256-8a2+i5u7YoszAgd5OIEW0eYUcP8yfhtoOIhLJkylYJ4="; - } // { inherit rev; }; - isLibrary = false; - isExecutable = true; - libraryHaskellDepends = [ - aeson base bytestring cabal-install-parsers Cabal-syntax containers - directory filepath frontmatter generic-lens-lite mtl parsec pretty - regex-applicative text - ]; - executableHaskellDepends = [ - base bytestring Cabal-syntax directory filepath - optparse-applicative - ]; - doHaddock = false; - description = "Concatenate changelog entries into a single one"; - license = lib.licenses.gpl3Plus; - mainProgram = "changelog-d"; -} diff --git a/misc/changelog-d.nix b/misc/changelog-d.nix deleted file mode 100644 index 1b20f4596..000000000 --- a/misc/changelog-d.nix +++ /dev/null @@ -1,31 +0,0 @@ -# Taken temporarily from -{ - callPackage, - lib, - haskell, - haskellPackages, -}: - -let - hsPkg = haskellPackages.callPackage ./changelog-d.cabal.nix { }; - - addCompletions = haskellPackages.generateOptparseApplicativeCompletions ["changelog-d"]; - - haskellModifications = - lib.flip lib.pipe [ - addCompletions - haskell.lib.justStaticExecutables - ]; - - mkDerivationOverrides = finalAttrs: oldAttrs: { - - version = oldAttrs.version + "-git-${lib.strings.substring 0 7 oldAttrs.src.rev}"; - - meta = oldAttrs.meta // { - homepage = "https://codeberg.org/roberth/changelog-d"; - maintainers = [ lib.maintainers.roberth ]; - }; - - }; -in - (haskellModifications hsPkg).overrideAttrs mkDerivationOverrides diff --git a/misc/systemv/nix-daemon b/misc/systemv/nix-daemon index fea537167..e8326f947 100755 --- a/misc/systemv/nix-daemon +++ b/misc/systemv/nix-daemon @@ -34,6 +34,7 @@ else fi # Source function library. +# shellcheck source=/dev/null . /etc/init.d/functions LOCKFILE=/var/lock/subsys/nix-daemon @@ -41,14 +42,20 @@ RUNDIR=/var/run/nix PIDFILE=${RUNDIR}/nix-daemon.pid RETVAL=0 -base=${0##*/} +# https://www.shellcheck.net/wiki/SC3004 +# Check if gettext exists +if ! type gettext > /dev/null 2>&1 +then + # If not, create a dummy function that returns the input verbatim + gettext() { printf '%s' "$1"; } +fi start() { mkdir -p ${RUNDIR} chown ${NIX_DAEMON_USER}:${NIX_DAEMON_USER} ${RUNDIR} - echo -n $"Starting nix daemon... " + printf '%s' "$(gettext 'Starting nix daemon... ')" daemonize -u $NIX_DAEMON_USER -p ${PIDFILE} $NIX_DAEMON_BIN $NIX_DAEMON_OPTS RETVAL=$? @@ -58,7 +65,7 @@ start() { } stop() { - echo -n $"Shutting down nix daemon: " + printf '%s' "$(gettext 'Shutting down nix daemon: ')" killproc -p ${PIDFILE} $NIX_DAEMON_BIN RETVAL=$? [ $RETVAL -eq 0 ] && rm -f ${LOCKFILE} ${PIDFILE} @@ -67,7 +74,7 @@ stop() { } reload() { - echo -n $"Reloading nix daemon... " + printf '%s' "$(gettext 'Reloading nix daemon... ')" killproc -p ${PIDFILE} $NIX_DAEMON_BIN -HUP RETVAL=$? echo @@ -105,7 +112,7 @@ case "$1" in fi ;; *) - echo $"Usage: $0 {start|stop|status|restart|condrestart}" + printf '%s' "$(gettext "Usage: $0 {start|stop|status|restart|condrestart}")" exit 2 ;; esac diff --git a/mk/common-test.sh b/mk/common-test.sh index 2783d293b..817422c40 100644 --- a/mk/common-test.sh +++ b/mk/common-test.sh @@ -1,27 +1,23 @@ +# shellcheck shell=bash + # Remove overall test dir (at most one of the two should match) and # remove file extension. -test_name=$(echo -n "$test" | sed \ - -e "s|^tests/unit/[^/]*/data/||" \ + +test_name=$(echo -n "${test?must be defined by caller (test runner)}" | sed \ + -e "s|^src/[^/]*-test/data/||" \ -e "s|^tests/functional/||" \ -e "s|\.sh$||" \ ) +# shellcheck disable=SC2016 TESTS_ENVIRONMENT=( "TEST_NAME=$test_name" 'NIX_REMOTE=' 'PS4=+(${BASH_SOURCE[0]-$0}:$LINENO) ' ) -: ${BASH:=/usr/bin/env bash} +read -r -a bash <<< "${BASH:-/usr/bin/env bash}" run () { - cd "$(dirname $1)" && env "${TESTS_ENVIRONMENT[@]}" $BASH -x -e -u -o pipefail $(basename $1) -} - -init_test () { - run "$init" 2>/dev/null > /dev/null -} - -run_test_proper () { - run "$test" + cd "$(dirname "$1")" && env "${TESTS_ENVIRONMENT[@]}" "${bash[@]}" -x -e -u -o pipefail "$(basename "$1")" } diff --git a/mk/compilation-database.mk b/mk/compilation-database.mk new file mode 100644 index 000000000..f69dc0de0 --- /dev/null +++ b/mk/compilation-database.mk @@ -0,0 +1,11 @@ +compile-commands-json-files := + +define write-compile-commands + _srcs := $$(sort $$(foreach src, $$($(1)_SOURCES), $$(src))) + + $(1)_COMPILE_COMMANDS_JSON := $$(addprefix $(buildprefix), $$(addsuffix .compile_commands.json, $$(basename $$(_srcs)))) + + compile-commands-json-files += $$($(1)_COMPILE_COMMANDS_JSON) + + clean-files += $$($(1)_COMPILE_COMMANDS_JSON) +endef diff --git a/mk/cxx-big-literal.mk b/mk/cxx-big-literal.mk index 85365df8e..d64a171c8 100644 --- a/mk/cxx-big-literal.mk +++ b/mk/cxx-big-literal.mk @@ -1,5 +1,5 @@ %.gen.hh: % - @echo 'R"foo(' >> $@.tmp + @echo 'R"__NIX_STR(' >> $@.tmp $(trace-gen) cat $< >> $@.tmp - @echo ')foo"' >> $@.tmp + @echo ')__NIX_STR"' >> $@.tmp @mv $@.tmp $@ diff --git a/mk/debug-test.sh b/mk/debug-test.sh index 52482c01e..0dd4406c3 100755 --- a/mk/debug-test.sh +++ b/mk/debug-test.sh @@ -3,12 +3,8 @@ set -eu -o pipefail test=$1 -init=${2-} dir="$(dirname "${BASH_SOURCE[0]}")" source "$dir/common-test.sh" -if [ -n "$init" ]; then - (init_test) -fi -run_test_proper +run "$test" diff --git a/mk/lib.mk b/mk/lib.mk index fe0add1c9..1e7af6ad5 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -68,6 +68,7 @@ include mk/patterns.mk include mk/templates.mk include mk/cxx-big-literal.mk include mk/tests.mk +include mk/compilation-database.mk # Include all sub-Makefiles. @@ -86,17 +87,23 @@ $(foreach script, $(bin-scripts), $(eval $(call install-program-in,$(script),$(b $(foreach script, $(bin-scripts), $(eval programs-list += $(script))) $(foreach script, $(noinst-scripts), $(eval programs-list += $(script))) $(foreach template, $(template-files), $(eval $(call instantiate-template,$(template)))) -install_test_init=tests/functional/init.sh $(foreach test, $(install-tests), \ - $(eval $(call run-test,$(test),$(install_test_init))) \ + $(eval $(call run-test,$(test))) \ $(eval installcheck: $(test).test)) $(foreach test-group, $(install-tests-groups), \ - $(eval $(call run-test-group,$(test-group),$(install_test_init))) \ + $(eval $(call run-test-group,$(test-group))) \ $(eval installcheck: $(test-group).test-group) \ $(foreach test, $($(test-group)-tests), \ - $(eval $(call run-test,$(test),$(install_test_init))) \ + $(eval $(call run-test,$(test))) \ $(eval $(test-group).test-group: $(test).test))) +# Compilation database. +$(foreach lib, $(libraries), $(eval $(call write-compile-commands,$(lib)))) +$(foreach prog, $(programs), $(eval $(call write-compile-commands,$(prog)))) + +compile_commands.json: $(compile-commands-json-files) + @jq --slurp '.' $^ >$@ + # Include makefiles requiring built programs. $(foreach mf, $(makefiles-late), $(eval $(call include-sub-makefile,$(mf)))) diff --git a/mk/patterns.mk b/mk/patterns.mk index c81150260..4caa2039e 100644 --- a/mk/patterns.mk +++ b/mk/patterns.mk @@ -1,11 +1,41 @@ + +# These are the complete command lines we use to compile C and C++ files. +# - $< is the source file. +# - $1 is the object file to create. +CC_CMD=$(CC) -o $1 -c $< $(CPPFLAGS) $(GLOBAL_CFLAGS) $(CFLAGS) $($1_CFLAGS) -MMD -MF $(call filename-to-dep,$1) -MP +CXX_CMD=$(CXX) -o $1 -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($1_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep,$1) -MP + +# We use COMPILE_COMMANDS_JSON_CMD to turn a compilation command (like CC_CMD +# or CXX_CMD above) into a comple_commands.json file. We rely on bash native +# word splitting to define the positional arguments. +# - $< is the source file being compiled. +COMPILE_COMMANDS_JSON_CMD=jq --null-input '{ directory: $$ENV.PWD, file: "$<", arguments: $$ARGS.positional }' --args -- + + $(buildprefix)%.o: %.cc @mkdir -p "$(dir $@)" - $(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cxx) $(call CXX_CMD,$@) $(buildprefix)%.o: %.cpp @mkdir -p "$(dir $@)" - $(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cxx) $(call CXX_CMD,$@) $(buildprefix)%.o: %.c @mkdir -p "$(dir $@)" - $(trace-cc) $(CC) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CFLAGS) $(CFLAGS) $($@_CFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cc) $(call CC_CMD,$@) + +# In the following we need to replace the .compile_commands.json extension in $@ with .o +# to make the object file. This is needed because CC_CMD and CXX_CMD do further expansions +# based on the object file name (i.e. *_CXXFLAGS and filename-to-dep). + +$(buildprefix)%.compile_commands.json: %.cc + @mkdir -p "$(dir $@)" + $(trace-jq) $(COMPILE_COMMANDS_JSON_CMD) $(call CXX_CMD,$(@:.compile_commands.json=.o)) > $@ + +$(buildprefix)%.compile_commands.json: %.cpp + @mkdir -p "$(dir $@)" + $(trace-jq) $(COMPILE_COMMANDS_JSON_CMD) $(call CXX_CMD,$(@:.compile_commands.json=.o)) > $@ + +$(buildprefix)%.compile_commands.json: %.c + @mkdir -p "$(dir $@)" + $(trace-jq) $(COMPILE_COMMANDS_JSON_CMD) $(call CC_CMD,$(@:.compile_commands.json=.o)) > $@ diff --git a/mk/platform.mk b/mk/platform.mk index fe960dedf..22c114a20 100644 --- a/mk/platform.mk +++ b/mk/platform.mk @@ -29,4 +29,8 @@ ifdef HOST_OS HOST_SOLARIS = 1 HOST_UNIX = 1 endif + ifeq ($(HOST_KERNEL), gnu) + HOST_HURD = 1 + HOST_UNIX = 1 + endif endif diff --git a/mk/precompiled-headers.mk b/mk/precompiled-headers.mk index cdd3daecd..f2803eb79 100644 --- a/mk/precompiled-headers.mk +++ b/mk/precompiled-headers.mk @@ -8,7 +8,7 @@ GCH = $(buildprefix)precompiled-headers.h.gch $(GCH): precompiled-headers.h @rm -f $@ @mkdir -p "$(dir $@)" - $(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS) + $(trace-gen) $(CXX) -c -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS) clean-files += $(GCH) diff --git a/mk/run-test.sh b/mk/run-test.sh index da9c5a473..7f9f1d5f8 100755 --- a/mk/run-test.sh +++ b/mk/run-test.sh @@ -8,7 +8,6 @@ yellow="" normal="" test=$1 -init=${2-} dir="$(dirname "${BASH_SOURCE[0]}")" source "$dir/common-test.sh" @@ -22,20 +21,18 @@ if [ -t 1 ]; then fi run_test () { - if [ -n "$init" ]; then - (init_test 2>/dev/null > /dev/null) - fi - log="$(run_test_proper 2>&1)" && status=0 || status=$? + log="$(run "$test" 2>&1)" && status=0 || status=$? } run_test -if [ $status -eq 0 ]; then +if [[ "$status" = 0 ]]; then echo "$post_run_msg [${green}PASS$normal]" -elif [ $status -eq 99 ]; then +elif [[ "$status" = 77 ]]; then echo "$post_run_msg [${yellow}SKIP$normal]" else echo "$post_run_msg [${red}FAIL$normal]" + # shellcheck disable=SC2001 echo "$log" | sed 's/^/ /' exit "$status" fi diff --git a/mk/tests.mk b/mk/tests.mk index bac9b704a..0a10f6d3b 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -12,8 +12,8 @@ endef define run-test - $(eval $(call run-bash,$1.test,$1 $(test-deps),mk/run-test.sh $1 $2)) - $(eval $(call run-bash,$1.test-debug,$1 $(test-deps),mk/debug-test.sh $1 $2)) + $(eval $(call run-bash,$1.test,$1 $(test-deps),mk/run-test.sh $1)) + $(eval $(call run-bash,$1.test-debug,$1 $(test-deps),mk/debug-test.sh $1)) endef diff --git a/mk/tracing.mk b/mk/tracing.mk index 1fc5573d7..09db1e617 100644 --- a/mk/tracing.mk +++ b/mk/tracing.mk @@ -10,6 +10,8 @@ ifeq ($(V), 0) trace-install = @echo " INST " $@; trace-mkdir = @echo " MKDIR " $@; trace-test = @echo " TEST " $@; + trace-sh = @echo " SH " $@; + trace-jq = @echo " JQ " $@; suppress = @ diff --git a/package.nix b/package.nix index 20796a386..a7c8923e8 100644 --- a/package.nix +++ b/package.nix @@ -13,17 +13,16 @@ , curl , editline , readline -, fileset , flex , git , gtest , jq -, doxygen , libarchive , libcpuid , libgit2 , libseccomp , libsodium +, man , lowdown , mdbook , mdbook-linkcheck @@ -33,7 +32,8 @@ , pkg-config , rapidcheck , sqlite -, util-linux +, toml11 +, unixtools , xz , busybox-sandbox-shell ? null @@ -48,10 +48,8 @@ , pname ? "nix" , versionSuffix ? "" -, officialRelease ? false -# Whether to build Nix. Useful to skip for tasks like (a) just -# generating API docs or (b) testing existing pre-built versions of Nix +# Whether to build Nix. Useful to skip for tasks like testing existing pre-built versions of Nix , doBuild ? true # Run the unit tests as part of the build. See `installUnitTests` for an @@ -74,7 +72,10 @@ # sounds so long as evaluation just takes places within short-lived # processes. (When the process exits, the memory is reclaimed; it is # only leaked *within* the process.) -, enableGC ? true +# +# Temporarily disabled on Windows because the `GC_throw_bad_alloc` +# symbol is missing during linking. +, enableGC ? !stdenv.hostPlatform.isWindows # Whether to enable Markdown rendering in the Nix binary. , enableMarkdown ? !stdenv.hostPlatform.isWindows @@ -87,10 +88,6 @@ # - readline , readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline" -# Whether to build the internal API docs, can be done separately from -# everything else. -, enableInternalAPIDocs ? false - # Whether to install unit tests. This is useful when cross compiling # since we cannot run them natively during the build, but can do so # later. @@ -113,6 +110,8 @@ }: let + inherit (lib) fileset; + version = lib.fileContents ./.version + versionSuffix; # selected attributes with defaults, will be used to define some @@ -161,6 +160,8 @@ in { ./m4 # TODO: do we really need README.md? It doesn't seem used in the build. ./README.md + # This could be put behind a conditional + ./maintainers/local.mk # For make, regardless of what we are building ./local.mk ./Makefile @@ -171,17 +172,11 @@ in { ./doc ./misc ./precompiled-headers.h - ./src + (fileset.difference ./src ./src/perl) ./COPYING ./scripts/local.mk - ] ++ lib.optionals buildUnitTests [ + ] ++ lib.optionals enableManual [ ./doc/manual - ] ++ lib.optionals enableInternalAPIDocs [ - ./doc/internal-api - # Source might not be compiled, but still must be available - # for Doxygen to gather comments. - ./src - ./tests/unit ] ++ lib.optionals buildUnitTests [ ./tests/unit ] ++ lib.optionals doInstallCheck [ @@ -195,8 +190,10 @@ in { ++ lib.optional doBuild "dev" # If we are doing just build or just docs, the one thing will use # "out". We only need additional outputs if we are doing both. - ++ lib.optional (doBuild && (enableManual || enableInternalAPIDocs)) "doc" - ++ lib.optional installUnitTests "check"; + ++ lib.optional (doBuild && enableManual) "doc" + ++ lib.optional installUnitTests "check" + ++ lib.optional doCheck "testresults" + ; nativeBuildInputs = [ autoconf-archive @@ -213,14 +210,14 @@ in { git mercurial openssh + man # for testing `nix-* --help` ] ++ lib.optionals (doInstallCheck || enableManual) [ jq # Also for custom mdBook preprocessor. - ] ++ lib.optional stdenv.hostPlatform.isLinux util-linux - ++ lib.optional enableInternalAPIDocs doxygen + ] ++ lib.optional stdenv.hostPlatform.isStatic unixtools.hexdump ; - buildInputs = lib.optionals doBuild [ - boost + buildInputs = lib.optionals doBuild ( + [ brotli bzip2 curl @@ -229,6 +226,7 @@ in { libsodium openssl sqlite + toml11 xz ({ inherit readline editline; }.${readlineFlavor}) ] ++ lib.optionals enableMarkdown [ @@ -240,46 +238,22 @@ in { ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid # There have been issues building these dependencies ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin)) - (aws-sdk-cpp.override { - apis = ["s3" "transfer"]; - customMemoryManagement = false; - }) - ; + aws-sdk-cpp + ); - propagatedBuildInputs = [ + propagatedBuildInputs = lib.optionals doBuild ([ + boost nlohmann_json - ] ++ lib.optional enableGC boehmgc; + ] ++ lib.optional enableGC boehmgc + ); dontBuild = !attrs.doBuild; doCheck = attrs.doCheck; - disallowedReferences = [ boost ]; - - preConfigure = lib.optionalString (doBuild && ! stdenv.hostPlatform.isStatic) ( - '' - # Copy libboost_context so we don't get all of Boost in our closure. - # https://github.com/NixOS/nixpkgs/issues/45462 - mkdir -p $out/lib - cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib - rm -f $out/lib/*.a - '' + lib.optionalString stdenv.hostPlatform.isLinux '' - chmod u+w $out/lib/*.so.* - patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* - '' + lib.optionalString stdenv.hostPlatform.isDarwin '' - for LIB in $out/lib/*.dylib; do - chmod u+w $LIB - install_name_tool -id $LIB $LIB - install_name_tool -delete_rpath ${boost}/lib/ $LIB || true - done - install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib - '' - ); - configureFlags = [ (lib.enableFeature doBuild "build") (lib.enableFeature buildUnitTests "unit-tests") (lib.enableFeature doInstallCheck "functional-tests") - (lib.enableFeature enableInternalAPIDocs "internal-api-docs") (lib.enableFeature enableManual "doc-gen") (lib.enableFeature enableGC "gc") (lib.enableFeature enableMarkdown "markdown") @@ -303,8 +277,11 @@ in { makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1"; - installTargets = lib.optional doBuild "install" - ++ lib.optional enableInternalAPIDocs "internal-api-html"; + preCheck = '' + mkdir $testresults + ''; + + installTargets = lib.optional doBuild "install"; installFlags = "sysconfdir=$(out)/etc"; @@ -319,18 +296,16 @@ in { lib.optionalString stdenv.hostPlatform.isStatic '' mkdir -p $out/nix-support echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products - '' + lib.optionalString stdenv.isDarwin '' - install_name_tool \ - -change ${boost}/lib/libboost_context.dylib \ - $out/lib/libboost_context.dylib \ - $out/lib/libnixutil.dylib '' ) + lib.optionalString enableManual '' mkdir -p ''${!outputDoc}/nix-support echo "doc manual ''${!outputDoc}/share/doc/nix/manual" >> ''${!outputDoc}/nix-support/hydra-build-products - '' + lib.optionalString enableInternalAPIDocs '' - mkdir -p ''${!outputDoc}/nix-support - echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products + ''; + + # So the check output gets links for DLLs in the out output. + preFixup = lib.optionalString (stdenv.hostPlatform.isWindows && builtins.elem "check" finalAttrs.outputs) '' + ln -s "$check/lib/"*.dll "$check/bin" + ln -s "$out/bin/"*.dll "$check/bin" ''; doInstallCheck = attrs.doInstallCheck; @@ -341,20 +316,26 @@ in { # Work around weird bug where it doesn't think there is a Makefile. installCheckPhase = if (!doBuild && doInstallCheck) then '' + runHook preInstallCheck mkdir -p src/nix-channel make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES '' else null; # Needed for tests if we are not doing a build, but testing existing # built Nix. - preInstallCheck = lib.optionalString (! doBuild) '' - mkdir -p src/nix-channel - ''; + preInstallCheck = + lib.optionalString (! doBuild) '' + mkdir -p src/nix-channel + '' + # See https://github.com/NixOS/nix/issues/2523 + # Occurs often in tests since https://github.com/NixOS/nix/pull/9900 + + lib.optionalString stdenv.hostPlatform.isDarwin '' + export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES + ''; separateDebugInfo = !stdenv.hostPlatform.isStatic; - # TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated - # to work with `strictDeps`. + # TODO Always true after https://github.com/NixOS/nixpkgs/issues/318564 strictDeps = !withCoverageChecks; hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; diff --git a/packaging/components.nix b/packaging/components.nix new file mode 100644 index 000000000..870e9ae61 --- /dev/null +++ b/packaging/components.nix @@ -0,0 +1,43 @@ +scope: +let + inherit (scope) callPackage; +in + +# This becomes the pkgs.nixComponents attribute set +{ + nix = callPackage ../package.nix { }; + + nix-util = callPackage ../src/libutil/package.nix { }; + nix-util-c = callPackage ../src/libutil-c/package.nix { }; + nix-util-test-support = callPackage ../tests/unit/libutil-support/package.nix { }; + nix-util-tests = callPackage ../tests/unit/libutil/package.nix { }; + + nix-store = callPackage ../src/libstore/package.nix { }; + nix-store-c = callPackage ../src/libstore-c/package.nix { }; + nix-store-test-support = callPackage ../tests/unit/libstore-support/package.nix { }; + nix-store-tests = callPackage ../tests/unit/libstore/package.nix { }; + + nix-fetchers = callPackage ../src/libfetchers/package.nix { }; + nix-fetchers-tests = callPackage ../tests/unit/libfetchers/package.nix { }; + + nix-expr = callPackage ../src/libexpr/package.nix { }; + nix-expr-c = callPackage ../src/libexpr-c/package.nix { }; + nix-expr-test-support = callPackage ../tests/unit/libexpr-support/package.nix { }; + nix-expr-tests = callPackage ../tests/unit/libexpr/package.nix { }; + + nix-flake = callPackage ../src/libflake/package.nix { }; + nix-flake-tests = callPackage ../tests/unit/libflake/package.nix { }; + + nix-main = callPackage ../src/libmain/package.nix { }; + nix-main-c = callPackage ../src/libmain-c/package.nix { }; + + nix-cmd = callPackage ../src/libcmd/package.nix { }; + + # Will replace `nix` once the old build system is gone. + nix-ng = callPackage ../src/nix/package.nix { }; + + nix-internal-api-docs = callPackage ../src/internal-api-docs/package.nix { }; + nix-external-api-docs = callPackage ../src/external-api-docs/package.nix { }; + + nix-perl-bindings = callPackage ../src/perl/package.nix { }; +} diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix new file mode 100644 index 000000000..e0737593f --- /dev/null +++ b/packaging/dependencies.nix @@ -0,0 +1,165 @@ +# These overrides are applied to the dependencies of the Nix components. + +{ + # Flake inputs; used for sources + inputs, + + # The raw Nixpkgs, not affected by this scope + pkgs, + + stdenv, + versionSuffix, +}: + +let + prevStdenv = stdenv; +in + +let + inherit (pkgs) lib; + + root = ../.; + + stdenv = if prevStdenv.isDarwin && prevStdenv.isx86_64 + then darwinStdenv + else prevStdenv; + + # Fix the following error with the default x86_64-darwin SDK: + # + # error: aligned allocation function of type 'void *(std::size_t, std::align_val_t)' is only available on macOS 10.13 or newer + # + # Despite the use of the 10.13 deployment target here, the aligned + # allocation function Clang uses with this setting actually works + # all the way back to 10.6. + darwinStdenv = pkgs.overrideSDK prevStdenv { darwinMinVersion = "10.13"; }; + + # Nixpkgs implements this by returning a subpath into the fetched Nix sources. + resolvePath = p: p; + + # Indirection for Nixpkgs to override when package.nix files are vendored + filesetToSource = lib.fileset.toSource; + + localSourceLayer = finalAttrs: prevAttrs: + let + workDirPath = + # Ideally we'd pick finalAttrs.workDir, but for now `mkDerivation` has + # the requirement that everything except passthru and meta must be + # serialized by mkDerivation, which doesn't work for this. + prevAttrs.workDir; + + workDirSubpath = lib.path.removePrefix root workDirPath; + sources = assert prevAttrs.fileset._type == "fileset"; prevAttrs.fileset; + src = lib.fileset.toSource { fileset = sources; inherit root; }; + + in + { + sourceRoot = "${src.name}/" + workDirSubpath; + inherit src; + + # Clear what `derivation` can't/shouldn't serialize; see prevAttrs.workDir. + fileset = null; + workDir = null; + }; + + # Work around weird `--as-needed` linker behavior with BSD, see + # https://github.com/mesonbuild/meson/issues/3593 + bsdNoLinkAsNeeded = finalAttrs: prevAttrs: + lib.optionalAttrs stdenv.hostPlatform.isBSD { + mesonFlags = [ (lib.mesonBool "b_asneeded" false) ] ++ prevAttrs.mesonFlags or []; + }; + + miscGoodPractice = finalAttrs: prevAttrs: + { + strictDeps = prevAttrs.strictDeps or true; + enableParallelBuilding = true; + }; + +in +scope: { + inherit stdenv versionSuffix; + version = lib.fileContents ../.version + versionSuffix; + + aws-sdk-cpp = (pkgs.aws-sdk-cpp.override { + apis = [ "s3" "transfer" ]; + customMemoryManagement = false; + }).overrideAttrs { + # only a stripped down version is built, which takes a lot less resources + # to build, so we don't need a "big-parallel" machine. + requiredSystemFeatures = [ ]; + }; + + libseccomp = pkgs.libseccomp.overrideAttrs (_: rec { + version = "2.5.5"; + src = pkgs.fetchurl { + url = "https://github.com/seccomp/libseccomp/releases/download/v${version}/libseccomp-${version}.tar.gz"; + hash = "sha256-JIosik2bmFiqa69ScSw0r+/PnJ6Ut23OAsHJqiX7M3U="; + }; + }); + + boehmgc = pkgs.boehmgc.override { + enableLargeConfig = true; + }; + + # TODO Hack until https://github.com/NixOS/nixpkgs/issues/45462 is fixed. + boost = (pkgs.boost.override { + extraB2Args = [ + "--with-container" + "--with-context" + "--with-coroutine" + ]; + }).overrideAttrs (old: { + # Need to remove `--with-*` to use `--with-libraries=...` + buildPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.buildPhase; + installPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.installPhase; + }); + + libgit2 = pkgs.libgit2.overrideAttrs (attrs: { + src = inputs.libgit2; + version = inputs.libgit2.lastModifiedDate; + cmakeFlags = attrs.cmakeFlags or [] + ++ [ "-DUSE_SSH=exec" ]; + }); + + busybox-sandbox-shell = pkgs.busybox-sandbox-shell or (pkgs.busybox.override { + useMusl = true; + enableStatic = true; + enableMinimal = true; + extraConfig = '' + CONFIG_FEATURE_FANCY_ECHO y + CONFIG_FEATURE_SH_MATH y + CONFIG_FEATURE_SH_MATH_64 y + + CONFIG_ASH y + CONFIG_ASH_OPTIMIZE_FOR_SIZE y + + CONFIG_ASH_ALIAS y + CONFIG_ASH_BASH_COMPAT y + CONFIG_ASH_CMDCMD y + CONFIG_ASH_ECHO y + CONFIG_ASH_GETOPTS y + CONFIG_ASH_INTERNAL_GLOB y + CONFIG_ASH_JOB_CONTROL y + CONFIG_ASH_PRINTF y + CONFIG_ASH_TEST y + ''; + }); + + # TODO change in Nixpkgs, Windows works fine. First commit of + # https://github.com/NixOS/nixpkgs/pull/322977 backported will fix. + toml11 = pkgs.toml11.overrideAttrs (old: { + meta.platforms = lib.platforms.all; + }); + + inherit resolvePath filesetToSource; + + mkMesonDerivation = f: let + exts = [ + miscGoodPractice + bsdNoLinkAsNeeded + localSourceLayer + ]; + in stdenv.mkDerivation + (lib.extends + (lib.foldr lib.composeExtensions (_: _: {}) exts) + f); +} diff --git a/packaging/hydra.nix b/packaging/hydra.nix new file mode 100644 index 000000000..dbe992476 --- /dev/null +++ b/packaging/hydra.nix @@ -0,0 +1,200 @@ +{ inputs +, binaryTarball +, forAllCrossSystems +, forAllSystems +, lib +, linux64BitSystems +, nixpkgsFor +, self +}: +let + inherit (inputs) nixpkgs nixpkgs-regression; + + installScriptFor = tarballs: + nixpkgsFor.x86_64-linux.native.callPackage ../scripts/installer.nix { + inherit tarballs; + }; + + testNixVersions = pkgs: client: daemon: + pkgs.callPackage ../package.nix { + pname = + "nix-tests" + + lib.optionalString + (lib.versionAtLeast daemon.version "2.4pre20211005" && + lib.versionAtLeast client.version "2.4pre20211005") + "-${client.version}-against-${daemon.version}"; + + test-client = client; + test-daemon = daemon; + + doBuild = false; + }; + + # Technically we could just return `pkgs.nixComponents`, but for Hydra it's + # convention to transpose it, and to transpose it efficiently, we need to + # enumerate them manually, so that we don't evaluate unnecessary package sets. + forAllPackages = lib.genAttrs [ + "nix" + "nix-util" + "nix-util-c" + "nix-util-test-support" + "nix-util-tests" + "nix-store" + "nix-store-c" + "nix-store-test-support" + "nix-store-tests" + "nix-fetchers" + "nix-fetchers-tests" + "nix-expr" + "nix-expr-c" + "nix-expr-test-support" + "nix-expr-tests" + "nix-flake" + "nix-flake-tests" + "nix-main" + "nix-main-c" + "nix-cmd" + "nix-ng" + ]; +in +{ + # Binary package for various platforms. + build = forAllPackages (pkgName: + forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.${pkgName})); + + shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation); + + buildStatic = forAllPackages (pkgName: + lib.genAttrs linux64BitSystems (system: nixpkgsFor.${system}.static.nixComponents.${pkgName})); + + buildCross = forAllPackages (pkgName: + forAllCrossSystems (crossSystem: + lib.genAttrs [ "x86_64-linux" ] (system: nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName}))); + + buildNoGc = forAllSystems (system: + self.packages.${system}.nix.override { enableGC = false; } + ); + + buildNoTests = forAllSystems (system: nixpkgsFor.${system}.native.nix_noTests); + + # Toggles some settings for better coverage. Windows needs these + # library combinations, and Debian build Nix with GNU readline too. + buildReadlineNoMarkdown = forAllSystems (system: + self.packages.${system}.nix.override { + enableMarkdown = false; + readlineFlavor = "readline"; + } + ); + + # Perl bindings for various platforms. + perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.nix-perl-bindings); + + # Binary tarball for various platforms, containing a Nix store + # with the closure of 'nix' package, and the second half of + # the installation script. + binaryTarball = forAllSystems (system: binaryTarball nixpkgsFor.${system}.native.nix nixpkgsFor.${system}.native); + + binaryTarballCross = lib.genAttrs [ "x86_64-linux" ] (system: + forAllCrossSystems (crossSystem: + binaryTarball + nixpkgsFor.${system}.cross.${crossSystem}.nix + nixpkgsFor.${system}.cross.${crossSystem})); + + # The first half of the installation script. This is uploaded + # to https://nixos.org/nix/install. It downloads the binary + # tarball for the user's system and calls the second half of the + # installation script. + installerScript = installScriptFor [ + # Native + self.hydraJobs.binaryTarball."x86_64-linux" + self.hydraJobs.binaryTarball."i686-linux" + self.hydraJobs.binaryTarball."aarch64-linux" + self.hydraJobs.binaryTarball."x86_64-darwin" + self.hydraJobs.binaryTarball."aarch64-darwin" + # Cross + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu" + ]; + installerScriptForGHA = installScriptFor [ + # Native + self.hydraJobs.binaryTarball."x86_64-linux" + self.hydraJobs.binaryTarball."aarch64-darwin" + # Cross + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu" + ]; + + # docker image with Nix inside + dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); + + # Line coverage analysis. + coverage = nixpkgsFor.x86_64-linux.native.nix.override { + pname = "nix-coverage"; + withCoverageChecks = true; + }; + + # API docs for Nix's unstable internal C++ interfaces. + internal-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-internal-api-docs; + + # API docs for Nix's C bindings. + external-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-external-api-docs; + + # System tests. + tests = import ../tests/nixos { inherit lib nixpkgs nixpkgsFor self; } // { + + # Make sure that nix-env still produces the exact same result + # on a particular version of Nixpkgs. + evalNixpkgs = + let + inherit (nixpkgsFor.x86_64-linux.native) runCommand nix; + in + runCommand "eval-nixos" { buildInputs = [ nix ]; } + '' + type -p nix-env + # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593. + ( + set -x + time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages + [[ $(sha1sum < packages | cut -c1-40) = e01b031fc9785a572a38be6bc473957e3b6faad7 ]] + ) + mkdir $out + ''; + + nixpkgsLibTests = + forAllSystems (system: + import (nixpkgs + "/lib/tests/test-with-nix.nix") + { + lib = nixpkgsFor.${system}.native.lib; + nix = self.packages.${system}.nix; + pkgs = nixpkgsFor.${system}.native; + } + ); + }; + + metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" { + pkgs = nixpkgsFor.x86_64-linux.native; + nixpkgs = nixpkgs-regression; + }; + + installTests = forAllSystems (system: + let pkgs = nixpkgsFor.${system}.native; in + pkgs.runCommand "install-tests" + { + againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix; + againstCurrentLatest = + # FIXME: temporarily disable this on macOS because of #3605. + if system == "x86_64-linux" + then testNixVersions pkgs pkgs.nix pkgs.nixVersions.latest + else null; + # Disabled because the latest stable version doesn't handle + # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work + # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable; + } "touch $out"); + + installerTests = import ../tests/installer { + binaryTarballs = self.hydraJobs.binaryTarball; + inherit nixpkgsFor; + }; +} diff --git a/perl/.yath.rc b/perl/.yath.rc deleted file mode 100644 index 118bf80c8..000000000 --- a/perl/.yath.rc +++ /dev/null @@ -1,2 +0,0 @@ -[test] --I=rel(lib/Nix) diff --git a/perl/Makefile b/perl/Makefile deleted file mode 100644 index 832668dd1..000000000 --- a/perl/Makefile +++ /dev/null @@ -1,21 +0,0 @@ -makefiles = local.mk - -GLOBAL_CXXFLAGS += -g -Wall -std=c++2a - -# A convenience for concurrent development of Nix and its Perl bindings. -# Not needed in a standalone build of the Perl bindings. -ifneq ("$(wildcard ../src)", "") - GLOBAL_CXXFLAGS += -I ../src -endif - --include Makefile.config - -OPTIMIZE = 1 - -ifeq ($(OPTIMIZE), 1) - GLOBAL_CXXFLAGS += -O3 -else - GLOBAL_CXXFLAGS += -O0 -endif - -include mk/lib.mk diff --git a/perl/Makefile.config.in b/perl/Makefile.config.in deleted file mode 100644 index d856de3ad..000000000 --- a/perl/Makefile.config.in +++ /dev/null @@ -1,18 +0,0 @@ -HOST_OS = @host_os@ -CC = @CC@ -CFLAGS = @CFLAGS@ -CXX = @CXX@ -CXXFLAGS = @CXXFLAGS@ -PACKAGE_NAME = @PACKAGE_NAME@ -PACKAGE_VERSION = @PACKAGE_VERSION@ -SODIUM_LIBS = @SODIUM_LIBS@ -NIX_CFLAGS = @NIX_CFLAGS@ -NIX_LIBS = @NIX_LIBS@ -nixbindir = @nixbindir@ -curl = @curl@ -nixlibexecdir = @nixlibexecdir@ -nixlocalstatedir = @nixlocalstatedir@ -perl = @perl@ -perllibdir = @perllibdir@ -nixstoredir = @nixstoredir@ -nixsysconfdir = @nixsysconfdir@ diff --git a/perl/configure.ac b/perl/configure.ac deleted file mode 100644 index a02cb06c9..000000000 --- a/perl/configure.ac +++ /dev/null @@ -1,84 +0,0 @@ -AC_INIT(nix-perl, m4_esyscmd([bash -c "echo -n $(cat ../.version)$VERSION_SUFFIX"])) -AC_CONFIG_SRCDIR(MANIFEST) -AC_CONFIG_AUX_DIR(../config) - -CFLAGS= -CXXFLAGS= -AC_PROG_CC -AC_PROG_CXX - -AC_CANONICAL_HOST - -# Use 64-bit file system calls so that we can support files > 2 GiB. -AC_SYS_LARGEFILE - -AC_DEFUN([NEED_PROG], -[ -AC_PATH_PROG($1, $2) -if test -z "$$1"; then - AC_MSG_ERROR([$2 is required]) -fi -]) - -NEED_PROG(perl, perl) -NEED_PROG(curl, curl) -NEED_PROG(bzip2, bzip2) -NEED_PROG(xz, xz) - -# Test that Perl has the open/fork feature (Perl 5.8.0 and beyond). -AC_MSG_CHECKING([whether Perl is recent enough]) -if ! $perl -e 'open(FOO, "-|", "true"); while () { print; }; close FOO or die;'; then - AC_MSG_RESULT(no) - AC_MSG_ERROR([Your Perl version is too old. Nix requires Perl 5.8.0 or newer.]) -fi -AC_MSG_RESULT(yes) - - -# Figure out where to install Perl modules. -AC_MSG_CHECKING([for the Perl installation prefix]) -perlversion=$($perl -e 'use Config; print $Config{version};') -perlarchname=$($perl -e 'use Config; print $Config{archname};') -AC_SUBST(perllibdir, [${libdir}/perl5/site_perl/$perlversion/$perlarchname]) -AC_MSG_RESULT($perllibdir) - -# Look for libsodium. -PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"]) - -# Check for the required Perl dependencies (DBI and DBD::SQLite). -perlFlags="-I$perllibdir" - -AC_ARG_WITH(dbi, AC_HELP_STRING([--with-dbi=PATH], - [prefix of the Perl DBI library]), - perlFlags="$perlFlags -I$withval") - -AC_ARG_WITH(dbd-sqlite, AC_HELP_STRING([--with-dbd-sqlite=PATH], - [prefix of the Perl DBD::SQLite library]), - perlFlags="$perlFlags -I$withval") - -AC_MSG_CHECKING([whether DBD::SQLite works]) -if ! $perl $perlFlags -e 'use DBI; use DBD::SQLite;' 2>&5; then - AC_MSG_RESULT(no) - AC_MSG_FAILURE([The Perl modules DBI and/or DBD::SQLite are missing.]) -fi -AC_MSG_RESULT(yes) - -AC_SUBST(perlFlags) - -PKG_CHECK_MODULES([NIX], [nix-store]) - -NEED_PROG([NIX], [nix]) - -# Expand all variables in config.status. -test "$prefix" = NONE && prefix=$ac_default_prefix -test "$exec_prefix" = NONE && exec_prefix='${prefix}' -for name in $ac_subst_vars; do - declare $name="$(eval echo "${!name}")" - declare $name="$(eval echo "${!name}")" - declare $name="$(eval echo "${!name}")" -done - -rm -f Makefile.config -ln -sfn ../mk mk - -AC_CONFIG_FILES([]) -AC_OUTPUT diff --git a/perl/default.nix b/perl/default.nix deleted file mode 100644 index 7103574c9..000000000 --- a/perl/default.nix +++ /dev/null @@ -1,61 +0,0 @@ -{ lib, fileset -, stdenv -, perl, perlPackages -, autoconf-archive, autoreconfHook, pkg-config -, nix, curl, bzip2, xz, boost, libsodium, darwin -}: - -perl.pkgs.toPerlModule (stdenv.mkDerivation (finalAttrs: { - name = "nix-perl-${nix.version}"; - - src = fileset.toSource { - root = ../.; - fileset = fileset.unions ([ - ../.version - ../m4 - ../mk - ./MANIFEST - ./Makefile - ./Makefile.config.in - ./configure.ac - ./lib - ./local.mk - ] ++ lib.optionals finalAttrs.doCheck [ - ./.yath.rc - ./t - ]); - }; - - nativeBuildInputs = - [ autoconf-archive - autoreconfHook - pkg-config - ]; - - buildInputs = - [ nix - curl - bzip2 - xz - perl - boost - ] - ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium - ++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security; - - # `perlPackages.Test2Harness` is marked broken for Darwin - doCheck = !stdenv.isDarwin; - - nativeCheckInputs = [ - perlPackages.Test2Harness - ]; - - configureFlags = [ - "--with-dbi=${perlPackages.DBI}/${perl.libPrefix}" - "--with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix}" - ]; - - enableParallelBuilding = true; - - postUnpack = "sourceRoot=$sourceRoot/perl"; -})) diff --git a/perl/local.mk b/perl/local.mk deleted file mode 100644 index ed4764eb9..000000000 --- a/perl/local.mk +++ /dev/null @@ -1,46 +0,0 @@ -nix_perl_sources := \ - lib/Nix/Store.pm \ - lib/Nix/Manifest.pm \ - lib/Nix/SSH.pm \ - lib/Nix/CopyClosure.pm \ - lib/Nix/Config.pm.in \ - lib/Nix/Utils.pm - -nix_perl_modules := $(nix_perl_sources:.in=) - -$(foreach x, $(nix_perl_modules), $(eval $(call install-data-in, $(x), $(perllibdir)/Nix))) - -lib/Nix/Store.cc: lib/Nix/Store.xs - $(trace-gen) xsubpp $^ -output $@ - -libraries += Store - -Store_DIR := lib/Nix - -Store_SOURCES := $(Store_DIR)/Store.cc - -Store_CXXFLAGS = \ - $(NIX_CFLAGS) \ - -I$(shell perl -e 'use Config; print $$Config{archlibexp};')/CORE \ - -D_FILE_OFFSET_BITS=64 \ - -Wno-unknown-warning-option -Wno-unused-variable -Wno-literal-suffix \ - -Wno-reserved-user-defined-literal -Wno-duplicate-decl-specifier -Wno-pointer-bool-conversion - -Store_LDFLAGS := $(SODIUM_LIBS) $(NIX_LIBS) - -ifdef HOST_CYGWIN - archlib = $(shell perl -E 'use Config; print $$Config{archlib};') - libperl = $(shell perl -E 'use Config; print $$Config{libperl};') - Store_LDFLAGS += $(shell find ${archlib} -name ${libperl}) -endif - -Store_ALLOW_UNDEFINED = 1 - -Store_FORCE_INSTALL = 1 - -Store_INSTALL_DIR = $(perllibdir)/auto/Nix/Store - -clean-files += lib/Nix/Config.pm lib/Nix/Store.cc Makefile.config - -check: all - yath test diff --git a/precompiled-headers.h b/precompiled-headers.h index f52f1cab8..e1a3f8cc0 100644 --- a/precompiled-headers.h +++ b/precompiled-headers.h @@ -42,19 +42,22 @@ #include #include #include -#include -#include -#include #include -#include -#include -#include #include #include #include -#include -#include -#include #include +#ifndef _WIN32 +# include +# include +# include +# include +# include +# include +# include +# include +# include +#endif + #include diff --git a/scripts/bigsur-nixbld-user-migration.sh b/scripts/bigsur-nixbld-user-migration.sh index f1619fd56..0eb312e07 100755 --- a/scripts/bigsur-nixbld-user-migration.sh +++ b/scripts/bigsur-nixbld-user-migration.sh @@ -3,7 +3,7 @@ ((NEW_NIX_FIRST_BUILD_UID=301)) id_available(){ - dscl . list /Users UniqueID | grep -E '\b'$1'\b' >/dev/null + dscl . list /Users UniqueID | grep -E '\b'"$1"'\b' >/dev/null } change_nixbld_names_and_ids(){ @@ -26,18 +26,18 @@ change_nixbld_names_and_ids(){ fi done - if [[ $name == _* ]]; then + if [[ "$name" == _* ]]; then echo " It looks like $name has already been renamed--skipping." else # first 3 are cleanup, it's OK if they aren't here - sudo dscl . delete /Users/$name dsAttrTypeNative:_writers_passwd &>/dev/null || true - sudo dscl . change /Users/$name NFSHomeDirectory "/private/var/empty 1" "/var/empty" &>/dev/null || true + sudo dscl . delete "/Users/$name" dsAttrTypeNative:_writers_passwd &>/dev/null || true + sudo dscl . change "/Users/$name" NFSHomeDirectory "/private/var/empty 1" "/var/empty" &>/dev/null || true # remove existing user from group - sudo dseditgroup -o edit -t user -d $name nixbld || true - sudo dscl . change /Users/$name UniqueID $uid $next_id - sudo dscl . change /Users/$name RecordName $name _$name + sudo dseditgroup -o edit -t user -d "$name" nixbld || true + sudo dscl . change "/Users/$name" UniqueID "$uid" "$next_id" + sudo dscl . change "/Users/$name" RecordName "$name" "_$name" # add renamed user to group - sudo dseditgroup -o edit -t user -a _$name nixbld + sudo dseditgroup -o edit -t user -a "_$name" nixbld echo " $name migrated to _$name (uid: $next_id)" fi done < <(dscl . list /Users UniqueID | grep nixbld | sort -n -k2) diff --git a/scripts/check-hydra-status.sh b/scripts/check-hydra-status.sh deleted file mode 100644 index e62705e94..000000000 --- a/scripts/check-hydra-status.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail -# set -x - - -# mapfile BUILDS_FOR_LATEST_EVAL < <( -# curl -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master/evals | \ -# jq -r '.evals[0].builds[] | @sh') -BUILDS_FOR_LATEST_EVAL=$( -curl -sS -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master/evals | \ - jq -r '.evals[0].builds[]') - -someBuildFailed=0 - -for buildId in $BUILDS_FOR_LATEST_EVAL; do - buildInfo=$(curl --fail -sS -H 'Accept: application/json' "https://hydra.nixos.org/build/$buildId") - - finished=$(echo "$buildInfo" | jq -r '.finished') - - if [[ $finished = 0 ]]; then - continue - fi - - buildStatus=$(echo "$buildInfo" | jq -r '.buildstatus') - - if [[ $buildStatus != 0 ]]; then - someBuildFailed=1 - echo "Job “$(echo "$buildInfo" | jq -r '.job')†failed on hydra: $buildInfo" - fi -done - -exit "$someBuildFailed" diff --git a/scripts/flake-regressions.sh b/scripts/flake-regressions.sh new file mode 100755 index 000000000..d76531134 --- /dev/null +++ b/scripts/flake-regressions.sh @@ -0,0 +1,27 @@ +#! /usr/bin/env bash + +set -e + +echo "Nix version:" +nix --version + +cd flake-regressions + +status=0 + +flakes=$(find tests -mindepth 3 -maxdepth 3 -type d -not -path '*/.*' | sort | head -n25) + +echo "Running flake tests..." + +for flake in $flakes; do + + if ! REGENERATE=0 ./eval-flake.sh "$flake"; then + status=1 + echo "⌠$flake" + else + echo "✅ $flake" + fi + +done + +exit "$status" diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index ad3ee8881..6aee073e3 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -754,7 +754,7 @@ I will: (if it does, I will tell you how to clean them up.) - create local users (see the list above for the users I'll make) - create a local group ($NIX_BUILD_GROUP_NAME) - - install Nix in to $NIX_ROOT + - install Nix in $NIX_ROOT - create a configuration file in /etc/nix - set up the "default profile" by creating some Nix-related files in $ROOT_HOME diff --git a/scripts/install-systemd-multi-user.sh b/scripts/install-systemd-multi-user.sh index 202a9bb54..a62ed7e3a 100755 --- a/scripts/install-systemd-multi-user.sh +++ b/scripts/install-systemd-multi-user.sh @@ -35,7 +35,7 @@ escape_systemd_env() { # Gather all non-empty proxy environment variables into a string create_systemd_proxy_env() { - vars="http_proxy https_proxy ftp_proxy no_proxy HTTP_PROXY HTTPS_PROXY FTP_PROXY NO_PROXY" + vars="http_proxy https_proxy ftp_proxy all_proxy no_proxy HTTP_PROXY HTTPS_PROXY FTP_PROXY ALL_PROXY NO_PROXY" for v in $vars; do if [ "x${!v:-}" != "x" ]; then echo "Environment=${v}=$(escape_systemd_env ${!v})" diff --git a/scripts/install.in b/scripts/install.in index 7d2e52b26..b4e808d8e 100755 --- a/scripts/install.in +++ b/scripts/install.in @@ -50,6 +50,11 @@ case "$(uname -s).$(uname -m)" in path=@tarballPath_armv7l-linux@ system=armv7l-linux ;; + Linux.riscv64) + hash=@tarballHash_riscv64-linux@ + path=@tarballPath_riscv64-linux@ + system=riscv64-linux + ;; Darwin.x86_64) hash=@tarballHash_x86_64-darwin@ path=@tarballPath_x86_64-darwin@ diff --git a/scripts/nix-profile-daemon.fish.in b/scripts/nix-profile-daemon.fish.in index c23aa64f0..346dce5dd 100644 --- a/scripts/nix-profile-daemon.fish.in +++ b/scripts/nix-profile-daemon.fish.in @@ -28,7 +28,7 @@ else end # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. -if test -n "$NIX_SSH_CERT_FILE" +if test -n "$NIX_SSL_CERT_FILE" : # Allow users to override the NIX_SSL_CERT_FILE else if test -e /etc/ssl/certs/ca-certificates.crt # NixOS, Ubuntu, Debian, Gentoo, Arch set --export NIX_SSL_CERT_FILE /etc/ssl/certs/ca-certificates.crt @@ -44,7 +44,7 @@ else if test -e "$NIX_LINK/etc/ca-bundle.crt" # old cacert in Nix profile set --export NIX_SSL_CERT_FILE "$NIX_LINK/etc/ca-bundle.crt" else # Fall back to what is in the nix profiles, favouring whatever is defined last. - for i in $NIX_PROFILES + for i in (string split ' ' $NIX_PROFILES) if test -e "$i/etc/ssl/certs/ca-bundle.crt" set --export NIX_SSL_CERT_FILE "$i/etc/ssl/certs/ca-bundle.crt" end diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index d256b24ed..eb124c0b5 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -1,4 +1,5 @@ # Only execute this file once per shell. +# This file is tested by tests/installer/default.nix. if [ -n "${__ETC_PROFILE_NIX_SOURCED:-}" ]; then return; fi __ETC_PROFILE_NIX_SOURCED=1 @@ -9,11 +10,9 @@ else NIX_LINK_NEW=$HOME/.local/state/nix/profile fi if [ -e "$NIX_LINK_NEW" ]; then - NIX_LINK="$NIX_LINK_NEW" -else - if [ -t 2 ] && [ -e "$NIX_LINK_NEW" ]; then + if [ -t 2 ] && [ -e "$NIX_LINK" ]; then warning="\033[1;35mwarning:\033[0m" - printf "$warning Both %s and legacy %s exist; using the latter.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2 + printf "$warning Both %s and legacy %s exist; using the former.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2 if [ "$(realpath "$NIX_LINK")" = "$(realpath "$NIX_LINK_NEW")" ]; then printf " Since the profiles match, you can safely delete either of them.\n" 1>&2 else @@ -26,6 +25,7 @@ else printf "$warning Profiles do not match. You should manually migrate from %s to %s.\n" "$NIX_LINK" "$NIX_LINK_NEW" 1>&2 fi fi + NIX_LINK="$NIX_LINK_NEW" fi export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK" @@ -69,4 +69,4 @@ else fi export PATH="$NIX_LINK/bin:@localstatedir@/nix/profiles/default/bin:$PATH" -unset NIX_LINK +unset NIX_LINK NIX_LINK_NEW diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index 44bc96e89..e868399b1 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -1,3 +1,4 @@ +# This file is tested by tests/installer/default.nix. if [ -n "$HOME" ] && [ -n "$USER" ]; then # Set up the per-user profile. @@ -9,11 +10,9 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then NIX_LINK_NEW="$HOME/.local/state/nix/profile" fi if [ -e "$NIX_LINK_NEW" ]; then - NIX_LINK="$NIX_LINK_NEW" - else - if [ -t 2 ] && [ -e "$NIX_LINK_NEW" ]; then + if [ -t 2 ] && [ -e "$NIX_LINK" ]; then warning="\033[1;35mwarning:\033[0m" - printf "$warning Both %s and legacy %s exist; using the latter.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2 + printf "$warning Both %s and legacy %s exist; using the former.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2 if [ "$(realpath "$NIX_LINK")" = "$(realpath "$NIX_LINK_NEW")" ]; then printf " Since the profiles match, you can safely delete either of them.\n" 1>&2 else @@ -26,6 +25,7 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then printf "$warning Profiles do not match. You should manually migrate from %s to %s.\n" "$NIX_LINK" "$NIX_LINK_NEW" 1>&2 fi fi + NIX_LINK="$NIX_LINK_NEW" fi # Set up environment. diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 118468477..82ad7d862 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -11,11 +11,13 @@ #include "machines.hh" #include "shared.hh" +#include "plugin.hh" #include "pathlocks.hh" #include "globals.hh" #include "serialise.hh" #include "build-result.hh" #include "store-api.hh" +#include "strings.hh" #include "derivations.hh" #include "local-store.hh" #include "legacy.hh" @@ -37,7 +39,7 @@ static std::string currentLoad; static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot) { - return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true); + return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri.render()), slot), true); } static bool allSupportedLocally(Store & store, const std::set& requiredFeatures) { @@ -135,7 +137,7 @@ static int main_build_remote(int argc, char * * argv) Machine * bestMachine = nullptr; uint64_t bestLoad = 0; for (auto & m : machines) { - debug("considering building on remote machine '%s'", m.storeUri); + debug("considering building on remote machine '%s'", m.storeUri.render()); if (m.enabled && m.systemSupported(neededSystem) && @@ -202,7 +204,7 @@ static int main_build_remote(int argc, char * * argv) else drvstr = ""; - auto error = HintFmt(errorText); + auto error = HintFmt::fromFormatString(errorText); error % drvstr % neededSystem @@ -232,17 +234,16 @@ static int main_build_remote(int argc, char * * argv) lock = -1; try { + storeUri = bestMachine->storeUri.render(); - Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri)); + Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", storeUri)); sshStore = bestMachine->openStore(); sshStore->connect(); - storeUri = bestMachine->storeUri; - } catch (std::exception & e) { auto msg = chomp(drainFD(5, false)); printError("cannot build on '%s': %s%s", - bestMachine->storeUri, e.what(), + storeUri, e.what(), msg.empty() ? "" : ": " + msg); bestMachine->enabled = false; continue; @@ -262,7 +263,20 @@ connected: auto inputs = readStrings(source); auto wantedOutputs = readStrings(source); - AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true); + AutoCloseFD uploadLock; + { + auto setUpdateLock = [&](auto && fileName){ + uploadLock = openLockFile(currentLoad + "/" + escapeUri(fileName) + ".upload-lock", true); + }; + try { + setUpdateLock(storeUri); + } catch (SysError & e) { + if (e.errNo != ENAMETOOLONG) throw; + // Try again hashing the store URL so we have a shorter path + auto h = hashString(HashAlgorithm::MD5, storeUri); + setUpdateLock(h.to_string(HashFormat::Base64, false)); + } + } { Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri)); diff --git a/doc/internal-api/.gitignore b/src/external-api-docs/.gitignore similarity index 100% rename from doc/internal-api/.gitignore rename to src/external-api-docs/.gitignore diff --git a/src/external-api-docs/.version b/src/external-api-docs/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/external-api-docs/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/external-api-docs/README.md b/src/external-api-docs/README.md new file mode 100644 index 000000000..8760ac88b --- /dev/null +++ b/src/external-api-docs/README.md @@ -0,0 +1,121 @@ +# Getting started + +> **Warning** These bindings are **experimental**, which means they can change +> at any time or be removed outright; nevertheless the plan is to provide a +> stable external C API to the Nix language and the Nix store. + +The language library allows evaluating Nix expressions and interacting with Nix +language values. The Nix store API is still rudimentary, and only allows +initialising and connecting to a store for the Nix language evaluator to +interact with. + +Currently there are two ways to interface with the Nix language evaluator +programmatically: + +1. Embedding the evaluator +2. Writing language plug-ins + +Embedding means you link the Nix C libraries in your program and use them from +there. Adding a plug-in means you make a library that gets loaded by the Nix +language evaluator, specified through a configuration option. + +Many of the components and mechanisms involved are not yet documented, therefore +please refer to the [Nix source code](https://github.com/NixOS/nix/) for +details. Additions to in-code documentation and the reference manual are highly +appreciated. + +The following examples, for simplicity, don't include error handling. See the +[Handling errors](@ref errors) section for more information. + +# Embedding the Nix Evaluator{#nix_evaluator_example} + +In this example we programmatically start the Nix language evaluator with a +dummy store (that has no store paths and cannot be written to), and evaluate the +Nix expression `builtins.nixVersion`. + +**main.c:** + +```C +#include +#include +#include +#include +#include +#include + +// NOTE: This example lacks all error handling. Production code must check for +// errors, as some return values will be undefined. + +void my_get_string_cb(const char * start, unsigned int n, void * user_data) +{ + *((char **) user_data) = strdup(start); +} + +int main() +{ + nix_libexpr_init(NULL); + + Store * store = nix_store_open(NULL, "dummy://", NULL); + EvalState * state = nix_state_create(NULL, NULL, store); // empty search path (NIX_PATH) + Value * value = nix_alloc_value(NULL, state); + + nix_expr_eval_from_string(NULL, state, "builtins.nixVersion", ".", value); + nix_value_force(NULL, state, value); + + char * version; + nix_get_string(NULL, value, my_get_string_cb, &version); + printf("Nix version: %s\n", version); + + free(version); + nix_gc_decref(NULL, value); + nix_state_free(state); + nix_store_free(store); + return 0; +} +``` + +**Usage:** + +```ShellSession +$ gcc main.c $(pkg-config nix-expr-c --libs --cflags) -o main +$ ./main +Nix version: 2.17 +``` + +# Writing a Nix language plug-in + +In this example we add a custom primitive operation (_primop_) to `builtins`. It +will increment the argument if it is an integer and throw an error otherwise. + +**plugin.c:** + +```C +#include +#include +#include + +void increment(void* user_data, nix_c_context* ctx, EvalState* state, Value** args, Value* v) { + nix_value_force(NULL, state, args[0]); + if (nix_get_type(NULL, args[0]) == NIX_TYPE_INT) { + nix_init_int(NULL, v, nix_get_int(NULL, args[0]) + 1); + } else { + nix_set_err_msg(ctx, NIX_ERR_UNKNOWN, "First argument should be an integer."); + } +} + +void nix_plugin_entry() { + const char* args[] = {"n", NULL}; + PrimOp *p = nix_alloc_primop(NULL, increment, 1, "increment", args, "Example custom built-in function: increments an integer", NULL); + nix_register_primop(NULL, p); + nix_gc_decref(NULL, p); +} +``` + +**Usage:** + +```ShellSession +$ gcc plugin.c $(pkg-config nix-expr-c --libs --cflags) -shared -o plugin.so +$ nix --plugin-files ./plugin.so repl +nix-repl> builtins.increment 1 +2 +``` diff --git a/src/external-api-docs/doxygen.cfg.in b/src/external-api-docs/doxygen.cfg.in new file mode 100644 index 000000000..1be71d895 --- /dev/null +++ b/src/external-api-docs/doxygen.cfg.in @@ -0,0 +1,58 @@ +# Doxyfile 1.9.5 + +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by +# double-quotes, unless you are using Doxywizard) that should identify the +# project for which the documentation is generated. This name is used in the +# title of most generated pages and in a few other places. +# The default value is: My Project. + +PROJECT_NAME = "Nix" + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. This +# could be handy for archiving the generated documentation or if some version +# control system is used. + +PROJECT_NUMBER = @PROJECT_NUMBER@ + +OUTPUT_DIRECTORY = @OUTPUT_DIRECTORY@ + +# Using the PROJECT_BRIEF tag one can provide an optional one line description +# for a project that appears at the top of each page and should give viewer a +# quick idea about the purpose of the project. Keep the description short. + +PROJECT_BRIEF = "Nix, the purely functional package manager: C API (experimental)" + +# If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output. +# The default value is: YES. + +GENERATE_LATEX = NO + +# The INPUT tag is used to specify the files and/or directories that contain +# documented source files. You may enter file names like myfile.cpp or +# directories like /usr/src/myproject. Separate the files or directories with +# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING +# Note: If this tag is empty the current directory is searched. + +# FIXME Make this list more maintainable somehow. We could maybe generate this +# in the Makefile, but we would need to change how `.in` files are preprocessed +# so they can expand variables despite configure variables. + +INPUT = \ + @src@/src/libutil-c \ + @src@/src/libexpr-c \ + @src@/src/libstore-c \ + @src@/doc/external-api/README.md + +FILE_PATTERNS = nix_api_*.h *.md + +# The INCLUDE_PATH tag can be used to specify one or more directories that +# contain include files that are not input files but should be processed by the +# preprocessor. Note that the INCLUDE_PATH is not recursive, so the setting of +# RECURSIVE has no effect here. +# This tag requires that the tag SEARCH_INCLUDES is set to YES. + +EXCLUDE_PATTERNS = *_internal.h +GENERATE_TREEVIEW = YES +OPTIMIZE_OUTPUT_FOR_C = YES + +USE_MDFILE_AS_MAINPAGE = doc/external-api/README.md diff --git a/src/external-api-docs/meson.build b/src/external-api-docs/meson.build new file mode 100644 index 000000000..62474ffe4 --- /dev/null +++ b/src/external-api-docs/meson.build @@ -0,0 +1,31 @@ +project('nix-external-api-docs', + version : files('.version'), + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +fs = import('fs') + +doxygen_cfg = configure_file( + input : 'doxygen.cfg.in', + output : 'doxygen.cfg', + configuration : { + 'PROJECT_NUMBER': meson.project_version(), + 'OUTPUT_DIRECTORY' : meson.current_build_dir(), + 'src' : fs.parent(fs.parent(meson.project_source_root())), + }, +) + +doxygen = find_program('doxygen', native : true, required : true) + +custom_target( + 'external-api-docs', + command : [ doxygen , doxygen_cfg ], + input : [ + doxygen_cfg, + ], + output : 'html', + install : true, + install_dir : get_option('datadir') / 'doc/nix/external-api', + build_always_stale : true, +) diff --git a/src/external-api-docs/package.nix b/src/external-api-docs/package.nix new file mode 100644 index 000000000..743b3e9b7 --- /dev/null +++ b/src/external-api-docs/package.nix @@ -0,0 +1,59 @@ +{ lib +, mkMesonDerivation + +, meson +, ninja +, doxygen + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-external-api-docs"; + inherit version; + + workDir = ./.; + fileset = + let + cpp = fileset.fileFilter (file: file.hasExt "cc" || file.hasExt "h"); + in + fileset.unions [ + ./.version + ../../.version + ./meson.build + ./doxygen.cfg.in + ./README.md + # Source is not compiled, but still must be available for Doxygen + # to gather comments. + (cpp ../libexpr-c) + (cpp ../libstore-c) + (cpp ../libutil-c) + ]; + + nativeBuildInputs = [ + meson + ninja + doxygen + ]; + + preConfigure = + '' + chmod u+w ./.version + echo ${finalAttrs.version} > ./.version + ''; + + postInstall = '' + mkdir -p ''${!outputDoc}/nix-support + echo "doc external-api-docs $out/share/doc/nix/external-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products + ''; + + meta = { + platforms = lib.platforms.all; + }; +}) diff --git a/src/internal-api-docs/.gitignore b/src/internal-api-docs/.gitignore new file mode 100644 index 000000000..dab28b6b0 --- /dev/null +++ b/src/internal-api-docs/.gitignore @@ -0,0 +1,3 @@ +/doxygen.cfg +/html +/latex diff --git a/src/internal-api-docs/.version b/src/internal-api-docs/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/internal-api-docs/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/doc/internal-api/doxygen.cfg.in b/src/internal-api-docs/doxygen.cfg.in similarity index 84% rename from doc/internal-api/doxygen.cfg.in rename to src/internal-api-docs/doxygen.cfg.in index 6c6c325bd..f1ef75b38 100644 --- a/doc/internal-api/doxygen.cfg.in +++ b/src/internal-api-docs/doxygen.cfg.in @@ -12,7 +12,9 @@ PROJECT_NAME = "Nix" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = @PACKAGE_VERSION@ +PROJECT_NUMBER = @PROJECT_NUMBER@ + +OUTPUT_DIRECTORY = @OUTPUT_DIRECTORY@ # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a @@ -36,27 +38,27 @@ GENERATE_LATEX = NO # so they can expand variables despite configure variables. INPUT = \ - src/libcmd \ - src/libexpr \ - src/libexpr/flake \ - tests/unit/libexpr \ - tests/unit/libexpr/value \ - tests/unit/libexpr/test \ - tests/unit/libexpr/test/value \ - src/libexpr/value \ - src/libfetchers \ - src/libmain \ - src/libstore \ - src/libstore/build \ - src/libstore/builtins \ - tests/unit/libstore \ - tests/unit/libstore/test \ - src/libutil \ - tests/unit/libutil \ - tests/unit/libutil/test \ - src/nix \ - src/nix-env \ - src/nix-store + @src@/libcmd \ + @src@/libexpr \ + @src@/libexpr/flake \ + @src@/nix-expr-tests \ + @src@/nix-expr-tests/value \ + @src@/nix-expr-test-support/test \ + @src@/nix-expr-test-support/test/value \ + @src@/libexpr/value \ + @src@/libfetchers \ + @src@/libmain \ + @src@/libstore \ + @src@/libstore/build \ + @src@/libstore/builtins \ + @src@/nix-store-tests \ + @src@/nix-store-test-support/test \ + @src@/libutil \ + @src@/nix-util-tests \ + @src@/nix-util-test-support/test \ + @src@/nix \ + @src@/nix-env \ + @src@/nix-store # If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names # in the source code. If set to NO, only conditional compilation will be diff --git a/src/internal-api-docs/meson.build b/src/internal-api-docs/meson.build new file mode 100644 index 000000000..54eb7e5dd --- /dev/null +++ b/src/internal-api-docs/meson.build @@ -0,0 +1,31 @@ +project('nix-internal-api-docs', + version : files('.version'), + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +fs = import('fs') + +doxygen_cfg = configure_file( + input : 'doxygen.cfg.in', + output : 'doxygen.cfg', + configuration : { + 'PROJECT_NUMBER': meson.project_version(), + 'OUTPUT_DIRECTORY' : meson.current_build_dir(), + 'src' : fs.parent(fs.parent(meson.project_source_root())) / 'src', + }, +) + +doxygen = find_program('doxygen', native : true, required : true) + +custom_target( + 'internal-api-docs', + command : [ doxygen , doxygen_cfg ], + input : [ + doxygen_cfg, + ], + output : 'html', + install : true, + install_dir : get_option('datadir') / 'doc/nix/internal-api', + build_always_stale : true, +) diff --git a/src/internal-api-docs/package.nix b/src/internal-api-docs/package.nix new file mode 100644 index 000000000..07ca6d4d9 --- /dev/null +++ b/src/internal-api-docs/package.nix @@ -0,0 +1,54 @@ +{ lib +, mkMesonDerivation + +, meson +, ninja +, doxygen + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-internal-api-docs"; + inherit version; + + workDir = ./.; + fileset = let + cpp = fileset.fileFilter (file: file.hasExt "cc" || file.hasExt "hh"); + in fileset.unions [ + ./.version + ../../.version + ./meson.build + ./doxygen.cfg.in + # Source is not compiled, but still must be available for Doxygen + # to gather comments. + (cpp ../.) + ]; + + nativeBuildInputs = [ + meson + ninja + doxygen + ]; + + preConfigure = + '' + chmod u+w ./.version + echo ${finalAttrs.version} > ./.version + ''; + + postInstall = '' + mkdir -p ''${!outputDoc}/nix-support + echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products + ''; + + meta = { + platforms = lib.platforms.all; + }; +}) diff --git a/src/libcmd/.version b/src/libcmd/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libcmd/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libcmd/build-utils-meson b/src/libcmd/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libcmd/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc index c5eb93c5d..905e70f32 100644 --- a/src/libcmd/built-path.cc +++ b/src/libcmd/built-path.cc @@ -1,6 +1,7 @@ #include "built-path.hh" #include "derivations.hh" #include "store-api.hh" +#include "comparator.hh" #include @@ -8,30 +9,24 @@ namespace nix { -#define CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, COMPARATOR) \ - bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \ - { \ - const MY_TYPE* me = this; \ - auto fields1 = std::tie(*me->drvPath, me->FIELD); \ - me = &other; \ - auto fields2 = std::tie(*me->drvPath, me->FIELD); \ - return fields1 COMPARATOR fields2; \ - } -#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, ==) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <) +// Custom implementation to avoid `ref` ptr equality +GENERATE_CMP_EXT( + , + std::strong_ordering, + SingleBuiltPathBuilt, + *me->drvPath, + me->output); -#define FIELD_TYPE std::pair -CMP(SingleBuiltPath, SingleBuiltPathBuilt, output) -#undef FIELD_TYPE +// Custom implementation to avoid `ref` ptr equality -#define FIELD_TYPE std::map -CMP(SingleBuiltPath, BuiltPathBuilt, outputs) -#undef FIELD_TYPE - -#undef CMP -#undef CMP_ONE +// TODO no `GENERATE_CMP_EXT` because no `std::set::operator<=>` on +// Darwin, per header. +GENERATE_EQUAL( + , + BuiltPathBuilt ::, + BuiltPathBuilt, + *me->drvPath, + me->outputs); StorePath SingleBuiltPath::outPath() const { diff --git a/src/libcmd/built-path.hh b/src/libcmd/built-path.hh index 99917e0ee..dc78d3e59 100644 --- a/src/libcmd/built-path.hh +++ b/src/libcmd/built-path.hh @@ -18,7 +18,8 @@ struct SingleBuiltPathBuilt { static SingleBuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view); nlohmann::json toJSON(const StoreDirConfig & store) const; - DECLARE_CMP(SingleBuiltPathBuilt); + bool operator ==(const SingleBuiltPathBuilt &) const noexcept; + std::strong_ordering operator <=>(const SingleBuiltPathBuilt &) const noexcept; }; using _SingleBuiltPathRaw = std::variant< @@ -33,6 +34,9 @@ struct SingleBuiltPath : _SingleBuiltPathRaw { using Opaque = DerivedPathOpaque; using Built = SingleBuiltPathBuilt; + bool operator == (const SingleBuiltPath &) const = default; + auto operator <=> (const SingleBuiltPath &) const = default; + inline const Raw & raw() const { return static_cast(*this); } @@ -59,11 +63,13 @@ struct BuiltPathBuilt { ref drvPath; std::map outputs; + bool operator == (const BuiltPathBuilt &) const noexcept; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //std::strong_ordering operator <=> (const BuiltPathBuilt &) const noexcept; + std::string to_string(const StoreDirConfig & store) const; static BuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view); nlohmann::json toJSON(const StoreDirConfig & store) const; - - DECLARE_CMP(BuiltPathBuilt); }; using _BuiltPathRaw = std::variant< @@ -82,6 +88,10 @@ struct BuiltPath : _BuiltPathRaw { using Opaque = DerivedPathOpaque; using Built = BuiltPathBuilt; + bool operator == (const BuiltPath &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=> (const BuiltPath &) const = default; + inline const Raw & raw() const { return static_cast(*this); } diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 369fa6004..67fef1909 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -1,3 +1,5 @@ +#include + #include "command.hh" #include "markdown.hh" #include "store-api.hh" @@ -6,8 +8,7 @@ #include "nixexpr.hh" #include "profiles.hh" #include "repl.hh" - -#include +#include "strings.hh" extern char * * environ __attribute__((weak)); @@ -127,12 +128,12 @@ ref EvalCommand::getEvalState() if (!evalState) { evalState = #if HAVE_BOEHMGC - std::allocate_shared(traceable_allocator(), - searchPath, getEvalStore(), getStore()) + std::allocate_shared( + traceable_allocator(), #else std::make_shared( - searchPath, getEvalStore(), getStore()) #endif + lookupPath, getEvalStore(), fetchSettings, evalSettings, getStore()) ; evalState->repair = repair; @@ -148,7 +149,7 @@ MixOperateOnOptions::MixOperateOnOptions() { addFlag({ .longName = "derivation", - .description = "Operate on the [store derivation](../../glossary.md#gloss-store-derivation) rather than its outputs.", + .description = "Operate on the [store derivation](@docroot@/glossary.md#gloss-store-derivation) rather than its outputs.", .category = installablesCategory, .handler = {&operateOn, OperateOn::Derivation}, }); diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 444ff81c9..fcef92487 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -1,18 +1,57 @@ +#include "fetch-settings.hh" #include "eval-settings.hh" #include "common-eval-args.hh" #include "shared.hh" +#include "config-global.hh" #include "filetransfer.hh" #include "eval.hh" #include "fetchers.hh" #include "registry.hh" #include "flake/flakeref.hh" +#include "flake/settings.hh" #include "store-api.hh" #include "command.hh" #include "tarball.hh" #include "fetch-to-store.hh" +#include "compatibility-settings.hh" +#include "eval-settings.hh" namespace nix { +fetchers::Settings fetchSettings; + +static GlobalConfig::Register rFetchSettings(&fetchSettings); + +EvalSettings evalSettings { + settings.readOnlyMode, + { + { + "flake", + [](ref store, std::string_view rest) { + experimentalFeatureSettings.require(Xp::Flakes); + // FIXME `parseFlakeRef` should take a `std::string_view`. + auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false); + debug("fetching flake search path element '%s''", rest); + auto storePath = flakeRef.resolve(store).fetchTree(store).first; + return store->toRealPath(storePath); + }, + }, + }, +}; + +static GlobalConfig::Register rEvalSettings(&evalSettings); + + +flake::Settings flakeSettings; + +static GlobalConfig::Register rFlakeSettings(&flakeSettings); + + +CompatibilitySettings compatibilitySettings {}; + +static GlobalConfig::Register rCompatibilitySettings(&compatibilitySettings); + + MixEvalArgs::MixEvalArgs() { addFlag({ @@ -20,7 +59,7 @@ MixEvalArgs::MixEvalArgs() .description = "Pass the value *expr* as the argument *name* to Nix functions.", .category = category, .labels = {"name", "expr"}, - .handler = {[&](std::string name, std::string expr) { autoArgs[name] = 'E' + expr; }} + .handler = {[&](std::string name, std::string expr) { autoArgs.insert_or_assign(name, AutoArg{AutoArgExpr{expr}}); }} }); addFlag({ @@ -28,87 +67,40 @@ MixEvalArgs::MixEvalArgs() .description = "Pass the string *string* as the argument *name* to Nix functions.", .category = category, .labels = {"name", "string"}, - .handler = {[&](std::string name, std::string s) { autoArgs[name] = 'S' + s; }}, + .handler = {[&](std::string name, std::string s) { autoArgs.insert_or_assign(name, AutoArg{AutoArgString{s}}); }}, + }); + + addFlag({ + .longName = "arg-from-file", + .description = "Pass the contents of file *path* as the argument *name* to Nix functions.", + .category = category, + .labels = {"name", "path"}, + .handler = {[&](std::string name, std::string path) { autoArgs.insert_or_assign(name, AutoArg{AutoArgFile{path}}); }}, + .completer = completePath + }); + + addFlag({ + .longName = "arg-from-stdin", + .description = "Pass the contents of stdin as the argument *name* to Nix functions.", + .category = category, + .labels = {"name"}, + .handler = {[&](std::string name) { autoArgs.insert_or_assign(name, AutoArg{AutoArgStdin{}}); }}, }); addFlag({ .longName = "include", .shortName = 'I', .description = R"( - Add *path* to the Nix search path. The Nix search path is - initialized from the colon-separated [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) environment - variable, and is used to look up the location of Nix expressions using [paths](@docroot@/language/values.md#type-path) enclosed in angle - brackets (i.e., ``). + Add *path* to search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md) - For instance, passing + This option may be given multiple times. - ``` - -I /home/eelco/Dev - -I /etc/nixos - ``` - - will cause Nix to look for paths relative to `/home/eelco/Dev` and - `/etc/nixos`, in that order. This is equivalent to setting the - `NIX_PATH` environment variable to - - ``` - /home/eelco/Dev:/etc/nixos - ``` - - It is also possible to match paths against a prefix. For example, - passing - - ``` - -I nixpkgs=/home/eelco/Dev/nixpkgs-branch - -I /etc/nixos - ``` - - will cause Nix to search for `` in - `/home/eelco/Dev/nixpkgs-branch/path` and `/etc/nixos/nixpkgs/path`. - - If a path in the Nix search path starts with `http://` or `https://`, - it is interpreted as the URL of a tarball that will be downloaded and - unpacked to a temporary location. The tarball must consist of a single - top-level directory. For example, passing - - ``` - -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/master.tar.gz - ``` - - tells Nix to download and use the current contents of the `master` - branch in the `nixpkgs` repository. - - The URLs of the tarballs from the official `nixos.org` channels - (see [the manual page for `nix-channel`](../nix-channel.md)) can be - abbreviated as `channel:`. For instance, the - following two flags are equivalent: - - ``` - -I nixpkgs=channel:nixos-21.05 - -I nixpkgs=https://nixos.org/channels/nixos-21.05/nixexprs.tar.xz - ``` - - You can also fetch source trees using [flake URLs](./nix3-flake.md#url-like-syntax) and add them to the - search path. For instance, - - ``` - -I nixpkgs=flake:nixpkgs - ``` - - specifies that the prefix `nixpkgs` shall refer to the source tree - downloaded from the `nixpkgs` entry in the flake registry. Similarly, - - ``` - -I nixpkgs=flake:github:NixOS/nixpkgs/nixos-22.05 - ``` - - makes `` refer to a particular branch of the - `NixOS/nixpkgs` repository on GitHub. + Paths added through `-I` take precedence over the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path) and the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH). )", .category = category, .labels = {"path"}, .handler = {[&](std::string s) { - searchPath.elements.emplace_back(SearchPath::Elem::parse(s)); + lookupPath.elements.emplace_back(LookupPath::Elem::parse(s)); }} }); @@ -127,8 +119,8 @@ MixEvalArgs::MixEvalArgs() .category = category, .labels = {"original-ref", "resolved-ref"}, .handler = {[&](std::string _from, std::string _to) { - auto from = parseFlakeRef(_from, absPath(".")); - auto to = parseFlakeRef(_to, absPath(".")); + auto from = parseFlakeRef(fetchSettings, _from, absPath(".")); + auto to = parseFlakeRef(fetchSettings, _to, absPath(".")); fetchers::Attrs extraAttrs; if (to.subdir != "") extraAttrs["dir"] = to.subdir; fetchers::overrideRegistry(from.input, to.input, extraAttrs); @@ -154,13 +146,23 @@ MixEvalArgs::MixEvalArgs() Bindings * MixEvalArgs::getAutoArgs(EvalState & state) { auto res = state.buildBindings(autoArgs.size()); - for (auto & i : autoArgs) { + for (auto & [name, arg] : autoArgs) { auto v = state.allocValue(); - if (i.second[0] == 'E') - state.mkThunk_(*v, state.parseExprFromString(i.second.substr(1), state.rootPath("."))); - else - v->mkString(((std::string_view) i.second).substr(1)); - res.insert(state.symbols.create(i.first), v); + std::visit(overloaded { + [&](const AutoArgExpr & arg) { + state.mkThunk_(*v, state.parseExprFromString(arg.expr, compatibilitySettings.nixShellShebangArgumentsRelativeToScript ? state.rootPath(absPath(getCommandBaseDir())) : state.rootPath("."))); + }, + [&](const AutoArgString & arg) { + v->mkString(arg.s); + }, + [&](const AutoArgFile & arg) { + v->mkString(readFile(arg.path.string())); + }, + [&](const AutoArgStdin & arg) { + v->mkString(readFile(STDIN_FILENO)); + } + }, arg); + res.insert(state.symbols.create(name), v); } return res.finish(); } @@ -176,7 +178,7 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas else if (hasPrefix(s, "flake:")) { experimentalFeatureSettings.require(Xp::Flakes); - auto flakeRef = parseFlakeRef(std::string(s.substr(6)), {}, true, false); + auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false); auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first; return state.rootPath(CanonPath(state.store->toRealPath(storePath))); } diff --git a/src/libcmd/common-eval-args.hh b/src/libcmd/common-eval-args.hh index 2eb63e15d..c62365b32 100644 --- a/src/libcmd/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh @@ -6,13 +6,42 @@ #include "common-args.hh" #include "search-path.hh" +#include + namespace nix { class Store; + +namespace fetchers { struct Settings; } + class EvalState; +struct EvalSettings; +struct CompatibilitySettings; class Bindings; struct SourcePath; +namespace flake { struct Settings; } + +/** + * @todo Get rid of global setttings variables + */ +extern fetchers::Settings fetchSettings; + +/** + * @todo Get rid of global setttings variables + */ +extern EvalSettings evalSettings; + +/** + * @todo Get rid of global setttings variables + */ +extern flake::Settings flakeSettings; + +/** + * Settings that control behaviors that have changed since Nix 2.3. + */ +extern CompatibilitySettings compatibilitySettings; + struct MixEvalArgs : virtual Args, virtual MixRepair { static constexpr auto category = "Common evaluation options"; @@ -21,14 +50,24 @@ struct MixEvalArgs : virtual Args, virtual MixRepair Bindings * getAutoArgs(EvalState & state); - SearchPath searchPath; + LookupPath lookupPath; std::optional evalStoreUrl; private: - std::map autoArgs; + struct AutoArgExpr { std::string expr; }; + struct AutoArgString { std::string s; }; + struct AutoArgFile { std::filesystem::path path; }; + struct AutoArgStdin { }; + + using AutoArg = std::variant; + + std::map autoArgs; }; +/** + * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) + */ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * baseDir = nullptr); } diff --git a/src/libcmd/compatibility-settings.hh b/src/libcmd/compatibility-settings.hh new file mode 100644 index 000000000..a129a957a --- /dev/null +++ b/src/libcmd/compatibility-settings.hh @@ -0,0 +1,36 @@ +#pragma once +#include "config.hh" + +namespace nix { +struct CompatibilitySettings : public Config +{ + + CompatibilitySettings() = default; + + // Added in Nix 2.24, July 2024. + Setting nixShellAlwaysLooksForShellNix{this, true, "nix-shell-always-looks-for-shell-nix", R"( + Before Nix 2.24, [`nix-shell`](@docroot@/command-ref/nix-shell.md) would only look at `shell.nix` if it was in the working directory - when no file was specified. + + Since Nix 2.24, `nix-shell` always looks for a `shell.nix`, whether that's in the working directory, or in a directory that was passed as an argument. + + You may set this to `false` to temporarily revert to the behavior of Nix 2.23 and older. + + Using this setting is not recommended. + It will be deprecated and removed. + )"}; + + // Added in Nix 2.24, July 2024. + Setting nixShellShebangArgumentsRelativeToScript{ + this, true, "nix-shell-shebang-arguments-relative-to-script", R"( + Before Nix 2.24, relative file path expressions in arguments in a `nix-shell` shebang were resolved relative to the working directory. + + Since Nix 2.24, `nix-shell` resolves these paths in a manner that is relative to the [base directory](@docroot@/glossary.md#gloss-base-directory), defined as the script's directory. + + You may set this to `false` to temporarily revert to the behavior of Nix 2.23 and older. + + Using this setting is not recommended. + It will be deprecated and removed. + )"}; +}; + +}; diff --git a/src/libcmd/installable-attr-path.cc b/src/libcmd/installable-attr-path.cc index 3ec1c1614..8917e7a01 100644 --- a/src/libcmd/installable-attr-path.cc +++ b/src/libcmd/installable-attr-path.cc @@ -75,6 +75,8 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths() std::set outputsToInstall; for (auto & output : packageInfo.queryOutputs(false, true)) outputsToInstall.insert(output.first); + if (outputsToInstall.empty()) + outputsToInstall.insert("out"); return OutputsSpec::Names { std::move(outputsToInstall) }; }, [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec { diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index ddec7537b..8796ad5ba 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -43,20 +43,6 @@ std::vector InstallableFlake::getActualAttrPaths() return res; } -Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake) -{ - auto vFlake = state.allocValue(); - - callFlake(state, lockedFlake, *vFlake); - - auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); - assert(aOutputs); - - state.forceValue(*aOutputs->value, aOutputs->value->determinePos(noPos)); - - return aOutputs->value; -} - static std::string showAttrPaths(const std::vector & paths) { std::string s; @@ -106,9 +92,14 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() fmt("while evaluating the flake output attribute '%s'", attrPath))) { return { *derivedPathWithInfo }; + } else { + throw Error( + "expected flake output attribute '%s' to be a derivation or path but found %s: %s", + attrPath, + showType(v), + ValuePrinter(*this->state, v, errorPrintOptions) + ); } - else - throw Error("flake output attribute '%s' is not a derivation or path", attrPath); } auto drvPath = attr->forceDerivation(); @@ -205,7 +196,8 @@ std::shared_ptr InstallableFlake::getLockedFlake() const flake::LockFlags lockFlagsApplyConfig = lockFlags; // FIXME why this side effect? lockFlagsApplyConfig.applyNixConfig = true; - _lockedFlake = std::make_shared(lockFlake(*state, flakeRef, lockFlagsApplyConfig)); + _lockedFlake = std::make_shared(lockFlake( + flakeSettings, *state, flakeRef, lockFlagsApplyConfig)); } return _lockedFlake; } diff --git a/src/libcmd/installable-flake.hh b/src/libcmd/installable-flake.hh index 314918c14..8e0a232ef 100644 --- a/src/libcmd/installable-flake.hh +++ b/src/libcmd/installable-flake.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "common-eval-args.hh" #include "installable-value.hh" namespace nix { @@ -52,8 +53,6 @@ struct InstallableFlake : InstallableValue std::vector getActualAttrPaths(); - Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake); - DerivedPathsWithInfo toDerivedPaths() override; std::pair toValue(EvalState & state) override; @@ -80,7 +79,7 @@ struct InstallableFlake : InstallableValue */ static inline FlakeRef defaultNixpkgsFlakeRef() { - return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}}); + return FlakeRef::fromAttrs(fetchSettings, {{"type","indirect"}, {"id", "nixpkgs"}}); } ref openEvalCache( diff --git a/src/libcmd/installable-value.hh b/src/libcmd/installable-value.hh index f300d392b..798cb5e1a 100644 --- a/src/libcmd/installable-value.hh +++ b/src/libcmd/installable-value.hh @@ -66,7 +66,7 @@ struct ExtraPathInfoValue : ExtraPathInfo }; /** - * An Installable which corresponds a Nix langauge value, in addition to + * An Installable which corresponds a Nix language value, in addition to * a collection of \ref DerivedPath "derived paths". */ struct InstallableValue : Installable diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 16d25d3cf..0fe956ec0 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -27,6 +27,8 @@ #include +#include "strings-inline.hh" + namespace nix { void completeFlakeInputPath( @@ -129,7 +131,7 @@ MixFlakeOptions::MixFlakeOptions() lockFlags.writeLockFile = false; lockFlags.inputOverrides.insert_or_assign( flake::parseInputPath(inputPath), - parseFlakeRef(flakeRef, absPath(getCommandBaseDir()), true)); + parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir()), true)); }}, .completer = {[&](AddCompletions & completions, size_t n, std::string_view prefix) { if (n == 0) { @@ -146,7 +148,7 @@ MixFlakeOptions::MixFlakeOptions() .category = category, .labels = {"flake-lock-path"}, .handler = {[&](std::string lockFilePath) { - lockFlags.referenceLockFilePath = lockFilePath; + lockFlags.referenceLockFilePath = {getFSSourceAccessor(), CanonPath(absPath(lockFilePath))}; }}, .completer = completePath }); @@ -170,14 +172,15 @@ MixFlakeOptions::MixFlakeOptions() .handler = {[&](std::string flakeRef) { auto evalState = getEvalState(); auto flake = flake::lockFlake( + flakeSettings, *evalState, - parseFlakeRef(flakeRef, absPath(getCommandBaseDir())), + parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir())), { .writeLockFile = false }); for (auto & [inputName, input] : flake.lockFile.root->inputs) { auto input2 = flake.lockFile.findInput({inputName}); // resolve 'follows' nodes if (auto input3 = std::dynamic_pointer_cast(input2)) { overrideRegistry( - fetchers::Input::fromAttrs({{"type","indirect"}, {"id", inputName}}), + fetchers::Input::fromAttrs(fetchSettings, {{"type","indirect"}, {"id", inputName}}), input3->lockedRef.input, {}); } @@ -288,11 +291,11 @@ void SourceExprCommand::completeInstallable(AddCompletions & completions, std::s state->autoCallFunction(*autoArgs, v1, v2); if (v2.type() == nAttrs) { - for (auto & i : *v2.attrs) { - std::string name = state->symbols[i.name]; + for (auto & i : *v2.attrs()) { + std::string_view name = state->symbols[i.name]; if (name.find(searchWord) == 0) { if (prefix_ == "") - completions.add(name); + completions.add(std::string(name)); else completions.add(prefix_ + "." + name); } @@ -338,10 +341,11 @@ void completeFlakeRefWithFragment( auto flakeRefS = std::string(prefix.substr(0, hash)); // TODO: ideally this would use the command base directory instead of assuming ".". - auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath(".")); + auto flakeRef = parseFlakeRef(fetchSettings, expandTilde(flakeRefS), absPath(".")); auto evalCache = openEvalCache(*evalState, - std::make_shared(lockFlake(*evalState, flakeRef, lockFlags))); + std::make_shared(lockFlake( + flakeSettings, *evalState, flakeRef, lockFlags))); auto root = evalCache->getRoot(); @@ -372,6 +376,7 @@ void completeFlakeRefWithFragment( auto attrPath2 = (*attr)->getAttrPath(attr2); /* Strip the attrpath prefix. */ attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size()); + // FIXME: handle names with dots completions.add(flakeRefS + "#" + prefixRoot + concatStringsSep(".", evalState->symbols.resolve(attrPath2))); } } @@ -403,7 +408,7 @@ void completeFlakeRef(AddCompletions & completions, ref store, std::strin Args::completeDir(completions, 0, prefix); /* Look for registry entries that match the prefix. */ - for (auto & registry : fetchers::getRegistries(store)) { + for (auto & registry : fetchers::getRegistries(fetchSettings, store)) { for (auto & entry : registry->entries) { auto from = entry.from.to_string(); if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) { @@ -442,13 +447,10 @@ ref openEvalCache( EvalState & state, std::shared_ptr lockedFlake) { - auto fingerprint = lockedFlake->getFingerprint(); - return make_ref( - evalSettings.useEvalCache && evalSettings.pureEval - ? std::optional { std::cref(fingerprint) } - : std::nullopt, - state, - [&state, lockedFlake]() + auto fingerprint = evalSettings.useEvalCache && evalSettings.pureEval + ? lockedFlake->getFingerprint(state.store) + : std::nullopt; + auto rootLoader = [&state, lockedFlake]() { /* For testing whether the evaluation cache is complete. */ @@ -460,11 +462,21 @@ ref openEvalCache( state.forceAttrs(*vFlake, noPos, "while parsing cached flake data"); - auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); + auto aOutputs = vFlake->attrs()->get(state.symbols.create("outputs")); assert(aOutputs); return aOutputs->value; - }); + }; + + if (fingerprint) { + auto search = state.evalCaches.find(fingerprint.value()); + if (search == state.evalCaches.end()) { + search = state.evalCaches.emplace(fingerprint.value(), make_ref(fingerprint, state, rootLoader)).first; + } + return search->second; + } else { + return make_ref(std::nullopt, state, rootLoader); + } } Installables SourceExprCommand::parseInstallables( @@ -527,7 +539,8 @@ Installables SourceExprCommand::parseInstallables( } try { - auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, absPath(getCommandBaseDir())); + auto [flakeRef, fragment] = parseFlakeRefWithFragment( + fetchSettings, std::string { prefix }, absPath(getCommandBaseDir())); result.push_back(make_ref( this, getEvalState(), @@ -594,6 +607,37 @@ std::vector Installable::build( return res; } +static void throwBuildErrors( + std::vector & buildResults, + const Store & store) +{ + std::vector failed; + for (auto & buildResult : buildResults) { + if (!buildResult.success()) { + failed.push_back(buildResult); + } + } + + auto failedResult = failed.begin(); + if (failedResult != failed.end()) { + if (failed.size() == 1) { + failedResult->rethrow(); + } else { + StringSet failedPaths; + for (; failedResult != failed.end(); failedResult++) { + if (!failedResult->errorMsg.empty()) { + logError(ErrorInfo{ + .level = lvlError, + .msg = failedResult->errorMsg, + }); + } + failedPaths.insert(failedResult->path.to_string(store)); + } + throw Error("build of %s failed", concatStringsSep(", ", quoteStrings(failedPaths))); + } + } +} + std::vector, BuiltPathWithResult>> Installable::build2( ref evalStore, ref store, @@ -655,10 +699,9 @@ std::vector, BuiltPathWithResult>> Installable::build if (settings.printMissing) printMissing(store, pathsToBuild, lvlInfo); - for (auto & buildResult : store->buildPathsWithResults(pathsToBuild, bMode, evalStore)) { - if (!buildResult.success()) - buildResult.rethrow(); - + auto buildResults = store->buildPathsWithResults(pathsToBuild, bMode, evalStore); + throwBuildErrors(buildResults, *store); + for (auto & buildResult : buildResults) { for (auto & aux : backmap[buildResult.path]) { std::visit(overloaded { [&](const DerivedPath::Built & bfd) { @@ -814,6 +857,7 @@ std::vector RawInstallablesCommand::getFlakeRefsForCompletion() std::vector res; for (auto i : rawInstallables) res.push_back(parseFlakeRefWithFragment( + fetchSettings, expandTilde(i), absPath(getCommandBaseDir())).first); return res; @@ -836,6 +880,7 @@ std::vector InstallableCommand::getFlakeRefsForCompletion() { return { parseFlakeRefWithFragment( + fetchSettings, expandTilde(_installable), absPath(getCommandBaseDir())).first }; diff --git a/src/libcmd/local.mk b/src/libcmd/local.mk index abb7459a7..a270333f4 100644 --- a/src/libcmd/local.mk +++ b/src/libcmd/local.mk @@ -6,10 +6,10 @@ libcmd_DIR := $(d) libcmd_SOURCES := $(wildcard $(d)/*.cc) -libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers +libcmd_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) $(INCLUDE_libflake) $(INCLUDE_libmain) libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) $(THREAD_LDFLAGS) -libcmd_LIBS = libstore libutil libexpr libmain libfetchers +libcmd_LIBS = libutil libstore libfetchers libflake libexpr libmain $(eval $(call install-file-in, $(buildprefix)$(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644)) diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc index a4e3c5a77..6a0d05d9f 100644 --- a/src/libcmd/markdown.cc +++ b/src/libcmd/markdown.cc @@ -1,21 +1,23 @@ #include "markdown.hh" -#include "util.hh" +#include "environment-variables.hh" +#include "error.hh" #include "finally.hh" #include "terminal.hh" -#include #if HAVE_LOWDOWN -#include +# include +# include #endif namespace nix { -std::string renderMarkdownToTerminal(std::string_view markdown) -{ #if HAVE_LOWDOWN +static std::string doRenderMarkdownToTerminal(std::string_view markdown) +{ int windowWidth = getWindowSize().second; - struct lowdown_opts opts { + struct lowdown_opts opts + { .type = LOWDOWN_TERM, .maxdepth = 20, .cols = (size_t) std::max(windowWidth - 5, 60), @@ -50,10 +52,22 @@ std::string renderMarkdownToTerminal(std::string_view markdown) if (!rndr_res) throw Error("allocation error while rendering Markdown"); - return filterANSIEscapes(std::string(buf->data, buf->size), !shouldANSI()); -#else - return std::string(markdown); -#endif + return filterANSIEscapes(std::string(buf->data, buf->size), !isTTY()); } +std::string renderMarkdownToTerminal(std::string_view markdown) +{ + if (auto e = getEnv("_NIX_TEST_RAW_MARKDOWN"); e && *e == "1") + return std::string(markdown); + else + return doRenderMarkdownToTerminal(markdown); } + +#else +std::string renderMarkdownToTerminal(std::string_view markdown) +{ + return std::string(markdown); +} +#endif + +} // namespace nix diff --git a/src/libcmd/markdown.hh b/src/libcmd/markdown.hh index a04d32a4f..66db1736c 100644 --- a/src/libcmd/markdown.hh +++ b/src/libcmd/markdown.hh @@ -1,10 +1,17 @@ #pragma once ///@file -#include "types.hh" +#include namespace nix { +/** + * Render the given Markdown text to the terminal. + * + * If Nix is compiled without Markdown support, this function will return the input text as-is. + * + * The renderer takes into account the terminal width, and wraps text accordingly. + */ std::string renderMarkdownToTerminal(std::string_view markdown); } diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build new file mode 100644 index 000000000..c484cf998 --- /dev/null +++ b/src/libcmd/meson.build @@ -0,0 +1,130 @@ +project('nix-cmd', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ +] +deps_public_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), + dependency('nix-fetchers'), + dependency('nix-expr'), + dependency('nix-flake'), + dependency('nix-main'), +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') +deps_public += nlohmann_json + +lowdown = dependency('lowdown', version : '>= 0.9.0', required : get_option('markdown')) +deps_private += lowdown +configdata.set('HAVE_LOWDOWN', lowdown.found().to_int()) + +readline_flavor = get_option('readline-flavor') +if readline_flavor == 'editline' + editline = dependency('libeditline', 'editline', version : '>=1.14') + deps_private += editline +elif readline_flavor == 'readline' + readline = dependency('readline') + deps_private += readline + configdata.set( + 'USE_READLINE', + 1, + description: 'Use readline instead of editline', + ) +else + error('illegal editline flavor', readline_flavor) +endif + +config_h = configure_file( + configuration : configdata, + output : 'config-cmd.hh', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + # '-include', 'config-fetchers.h', + '-include', 'config-expr.hh', + '-include', 'config-main.hh', + '-include', 'config-cmd.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'built-path.cc', + 'command-installable-value.cc', + 'command.cc', + 'common-eval-args.cc', + 'editor-for.cc', + 'installable-attr-path.cc', + 'installable-derived-path.cc', + 'installable-flake.cc', + 'installable-value.cc', + 'installables.cc', + 'legacy.cc', + 'markdown.cc', + 'misc-store-flags.cc', + 'network-proxy.cc', + 'repl-interacter.cc', + 'repl.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'built-path.hh', + 'command-installable-value.hh', + 'command.hh', + 'common-eval-args.hh', + 'compatibility-settings.hh', + 'editor-for.hh', + 'installable-attr-path.hh', + 'installable-derived-path.hh', + 'installable-flake.hh', + 'installable-value.hh', + 'installables.hh', + 'legacy.hh', + 'markdown.hh', + 'misc-store-flags.hh', + 'network-proxy.hh', + 'repl-interacter.hh', + 'repl.hh', +) + +this_library = library( + 'nixcmd', + sources, + dependencies : deps_public + deps_private + deps_other, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libcmd/meson.options b/src/libcmd/meson.options new file mode 100644 index 000000000..79ae4fa55 --- /dev/null +++ b/src/libcmd/meson.options @@ -0,0 +1,15 @@ +# vim: filetype=meson + +option( + 'markdown', + type: 'feature', + description: 'Enable Markdown rendering in the Nix binary (requires lowdown)', +) + +option( + 'readline-flavor', + type : 'combo', + choices : ['editline', 'readline'], + value : 'editline', + description : 'Which library to use for nice line editing with the Nix language REPL', +) diff --git a/src/libcmd/misc-store-flags.cc b/src/libcmd/misc-store-flags.cc index e66d3f63b..06552c032 100644 --- a/src/libcmd/misc-store-flags.cc +++ b/src/libcmd/misc-store-flags.cc @@ -81,9 +81,15 @@ Args::Flag fileIngestionMethod(FileIngestionMethod * method) How to compute the hash of the input. One of: - - `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function. + - `nar` (the default): + Serialises the input as a + [Nix Archive](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) + and passes that to the hash function. - - `flat`: Assumes that the input is a single file and directly passes it to the hash function; + - `flat`: + Assumes that the input is a single file and + [directly passes](@docroot@/store/file-system-object/content-address.md#serial-flat) + it to the hash function. )", .labels = {"file-ingestion-method"}, .handler = {[method](std::string s) { @@ -101,15 +107,23 @@ Args::Flag contentAddressMethod(ContentAddressMethod * method) How to compute the content-address of the store object. One of: - - `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function. + - [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive) + (the default): + Serialises the input as a + [Nix Archive](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) + and passes that to the hash function. - - `flat`: Assumes that the input is a single file and directly passes it to the hash function; + - [`flat`](@docroot@/store/store-object/content-address.md#method-flat): + Assumes that the input is a single file and + [directly passes](@docroot@/store/file-system-object/content-address.md#serial-flat) + it to the hash function. - - `text`: Like `flat`, but used for - [derivations](@docroot@/glossary.md#store-derivation) serialized in store object and + - [`text`](@docroot@/store/store-object/content-address.md#method-text): + Like `flat`, but used for + [derivations](@docroot@/glossary.md#store-derivation) serialized in store object and [`builtins.toFile`](@docroot@/language/builtins.html#builtins-toFile). For advanced use-cases only; - for regular usage prefer `nar` and `flat. + for regular usage prefer `nar` and `flat`. )", .labels = {"content-address-method"}, .handler = {[method](std::string s) { diff --git a/src/libcmd/network-proxy.cc b/src/libcmd/network-proxy.cc new file mode 100644 index 000000000..738bf6147 --- /dev/null +++ b/src/libcmd/network-proxy.cc @@ -0,0 +1,50 @@ +#include "network-proxy.hh" + +#include + +#include "environment-variables.hh" + +namespace nix { + +static const StringSet lowercaseVariables{"http_proxy", "https_proxy", "ftp_proxy", "all_proxy", "no_proxy"}; + +static StringSet getAllVariables() +{ + StringSet variables = lowercaseVariables; + for (const auto & variable : lowercaseVariables) { + std::string upperVariable; + std::transform( + variable.begin(), variable.end(), upperVariable.begin(), [](unsigned char c) { return std::toupper(c); }); + variables.insert(std::move(upperVariable)); + } + return variables; +} + +const StringSet networkProxyVariables = getAllVariables(); + +static StringSet getExcludingNoProxyVariables() +{ + static const StringSet excludeVariables{"no_proxy", "NO_PROXY"}; + StringSet variables; + std::set_difference( + networkProxyVariables.begin(), + networkProxyVariables.end(), + excludeVariables.begin(), + excludeVariables.end(), + std::inserter(variables, variables.begin())); + return variables; +} + +static const StringSet excludingNoProxyVariables = getExcludingNoProxyVariables(); + +bool haveNetworkProxyConnection() +{ + for (const auto & variable : excludingNoProxyVariables) { + if (getEnv(variable).has_value()) { + return true; + } + } + return false; +} + +} diff --git a/src/libcmd/network-proxy.hh b/src/libcmd/network-proxy.hh new file mode 100644 index 000000000..0b6856acb --- /dev/null +++ b/src/libcmd/network-proxy.hh @@ -0,0 +1,22 @@ +#pragma once +///@file + +#include "types.hh" + +namespace nix { + +/** + * Environment variables relating to network proxying. These are used by + * a few misc commands. + * + * See the Environment section of https://curl.se/docs/manpage.html for details. + */ +extern const StringSet networkProxyVariables; + +/** + * Heuristically check if there is a proxy connection by checking for defined + * proxy variables. + */ +bool haveNetworkProxyConnection(); + +} diff --git a/src/libcmd/package.nix b/src/libcmd/package.nix new file mode 100644 index 000000000..cde494901 --- /dev/null +++ b/src/libcmd/package.nix @@ -0,0 +1,104 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-util +, nix-store +, nix-fetchers +, nix-expr +, nix-flake +, nix-main +, editline +, readline +, lowdown +, nlohmann_json + +# Configuration Options + +, version + +# Whether to enable Markdown rendering in the Nix binary. +, enableMarkdown ? !stdenv.hostPlatform.isWindows + +# Which interactive line editor library to use for Nix's repl. +# +# Currently supported choices are: +# +# - editline (default) +# - readline +, readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline" +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-cmd"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + ({ inherit editline readline; }.${readlineFlavor}) + ] ++ lib.optional enableMarkdown lowdown; + + propagatedBuildInputs = [ + nix-util + nix-store + nix-fetchers + nix-expr + nix-flake + nix-main + nlohmann_json + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + mesonFlags = [ + (lib.mesonEnable "markdown" enableMarkdown) + (lib.mesonOption "readline-flavor" readlineFlavor) + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libcmd/repl-interacter.cc b/src/libcmd/repl-interacter.cc new file mode 100644 index 000000000..187af46ea --- /dev/null +++ b/src/libcmd/repl-interacter.cc @@ -0,0 +1,206 @@ +#include + +#ifdef USE_READLINE +#include +#include +#else +// editline < 1.15.2 don't wrap their API for C++ usage +// (added in https://github.com/troglobit/editline/commit/91398ceb3427b730995357e9d120539fb9bb7461). +// This results in linker errors due to to name-mangling of editline C symbols. +// For compatibility with these versions, we wrap the API here +// (wrapping multiple times on newer versions is no problem). +extern "C" { +#include +} +#endif + +#include "signals.hh" +#include "finally.hh" +#include "repl-interacter.hh" +#include "file-system.hh" +#include "repl.hh" +#include "environment-variables.hh" + +namespace nix { + +namespace { +// Used to communicate to NixRepl::getLine whether a signal occurred in ::readline. +volatile sig_atomic_t g_signal_received = 0; + +void sigintHandler(int signo) +{ + g_signal_received = signo; +} +}; + +static detail::ReplCompleterMixin * curRepl; // ugly + +#ifndef USE_READLINE +static char * completionCallback(char * s, int * match) +{ + auto possible = curRepl->completePrefix(s); + if (possible.size() == 1) { + *match = 1; + auto * res = strdup(possible.begin()->c_str() + strlen(s)); + if (!res) + throw Error("allocation failure"); + return res; + } else if (possible.size() > 1) { + auto checkAllHaveSameAt = [&](size_t pos) { + auto & first = *possible.begin(); + for (auto & p : possible) { + if (p.size() <= pos || p[pos] != first[pos]) + return false; + } + return true; + }; + size_t start = strlen(s); + size_t len = 0; + while (checkAllHaveSameAt(start + len)) + ++len; + if (len > 0) { + *match = 1; + auto * res = strdup(std::string(*possible.begin(), start, len).c_str()); + if (!res) + throw Error("allocation failure"); + return res; + } + } + + *match = 0; + return nullptr; +} + +static int listPossibleCallback(char * s, char *** avp) +{ + auto possible = curRepl->completePrefix(s); + + if (possible.size() > (std::numeric_limits::max() / sizeof(char *))) + throw Error("too many completions"); + + int ac = 0; + char ** vp = nullptr; + + auto check = [&](auto * p) { + if (!p) { + if (vp) { + while (--ac >= 0) + free(vp[ac]); + free(vp); + } + throw Error("allocation failure"); + } + return p; + }; + + vp = check((char **) malloc(possible.size() * sizeof(char *))); + + for (auto & p : possible) + vp[ac++] = check(strdup(p.c_str())); + + *avp = vp; + + return ac; +} +#endif + +ReadlineLikeInteracter::Guard ReadlineLikeInteracter::init(detail::ReplCompleterMixin * repl) +{ + // Allow nix-repl specific settings in .inputrc + rl_readline_name = "nix-repl"; + try { + createDirs(dirOf(historyFile)); + } catch (SystemError & e) { + logWarning(e.info()); + } +#ifndef USE_READLINE + el_hist_size = 1000; +#endif + read_history(historyFile.c_str()); + auto oldRepl = curRepl; + curRepl = repl; + Guard restoreRepl([oldRepl] { curRepl = oldRepl; }); +#ifndef USE_READLINE + rl_set_complete_func(completionCallback); + rl_set_list_possib_func(listPossibleCallback); +#endif + return restoreRepl; +} + +static constexpr const char * promptForType(ReplPromptType promptType) +{ + switch (promptType) { + case ReplPromptType::ReplPrompt: + return "nix-repl> "; + case ReplPromptType::ContinuationPrompt: + return " "; + } + assert(false); +} + +bool ReadlineLikeInteracter::getLine(std::string & input, ReplPromptType promptType) +{ +#ifndef _WIN32 // TODO use more signals.hh for this + struct sigaction act, old; + sigset_t savedSignalMask, set; + + auto setupSignals = [&]() { + act.sa_handler = sigintHandler; + sigfillset(&act.sa_mask); + act.sa_flags = 0; + if (sigaction(SIGINT, &act, &old)) + throw SysError("installing handler for SIGINT"); + + sigemptyset(&set); + sigaddset(&set, SIGINT); + if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask)) + throw SysError("unblocking SIGINT"); + }; + auto restoreSignals = [&]() { + if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) + throw SysError("restoring signals"); + + if (sigaction(SIGINT, &old, 0)) + throw SysError("restoring handler for SIGINT"); + }; + + setupSignals(); +#endif + char * s = readline(promptForType(promptType)); + Finally doFree([&]() { free(s); }); +#ifndef _WIN32 // TODO use more signals.hh for this + restoreSignals(); +#endif + + if (g_signal_received) { + g_signal_received = 0; + input.clear(); + return true; + } + + // editline doesn't echo the input to the output when non-interactive, unlike readline + // this results in a different behavior when running tests. The echoing is + // quite useful for reading the test output, so we add it here. + if (auto e = getEnv("_NIX_TEST_REPL_ECHO"); s && e && *e == "1") + { +#ifndef USE_READLINE + // This is probably not right for multi-line input, but we don't use that + // in the characterisation tests, so it's fine. + std::cout << promptForType(promptType) << s << std::endl; +#endif + } + + if (!s) + return false; + input += s; + input += '\n'; + + return true; +} + +ReadlineLikeInteracter::~ReadlineLikeInteracter() +{ + write_history(historyFile.c_str()); +} + +}; diff --git a/src/libcmd/repl-interacter.hh b/src/libcmd/repl-interacter.hh new file mode 100644 index 000000000..cc70efd07 --- /dev/null +++ b/src/libcmd/repl-interacter.hh @@ -0,0 +1,48 @@ +#pragma once +/// @file + +#include "finally.hh" +#include "types.hh" +#include +#include + +namespace nix { + +namespace detail { +/** Provides the completion hooks for the repl, without exposing its complete + * internals. */ +struct ReplCompleterMixin { + virtual StringSet completePrefix(const std::string & prefix) = 0; +}; +}; + +enum class ReplPromptType { + ReplPrompt, + ContinuationPrompt, +}; + +class ReplInteracter +{ +public: + using Guard = Finally>; + + virtual Guard init(detail::ReplCompleterMixin * repl) = 0; + /** Returns a boolean of whether the interacter got EOF */ + virtual bool getLine(std::string & input, ReplPromptType promptType) = 0; + virtual ~ReplInteracter(){}; +}; + +class ReadlineLikeInteracter : public virtual ReplInteracter +{ + std::string historyFile; +public: + ReadlineLikeInteracter(std::string historyFile) + : historyFile(historyFile) + { + } + virtual Guard init(detail::ReplCompleterMixin * repl) override; + virtual bool getLine(std::string & input, ReplPromptType promptType) override; + virtual ~ReadlineLikeInteracter() override; +}; + +}; diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 8b83608fa..efc04b029 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -1,34 +1,17 @@ #include #include #include -#include - -#include - -#ifdef USE_READLINE -#include -#include -#else -// editline < 1.15.2 don't wrap their API for C++ usage -// (added in https://github.com/troglobit/editline/commit/91398ceb3427b730995357e9d120539fb9bb7461). -// This results in linker errors due to to name-mangling of editline C symbols. -// For compatibility with these versions, we wrap the API here -// (wrapping multiple times on newer versions is no problem). -extern "C" { -#include -} -#endif +#include "repl-interacter.hh" #include "repl.hh" #include "ansicolor.hh" -#include "signals.hh" #include "shared.hh" +#include "config-global.hh" #include "eval.hh" -#include "eval-cache.hh" -#include "eval-inline.hh" #include "eval-settings.hh" #include "attr-path.hh" +#include "signals.hh" #include "store-api.hh" #include "log-store.hh" #include "common-eval-args.hh" @@ -38,18 +21,21 @@ extern "C" { #include "flake/flake.hh" #include "flake/lockfile.hh" #include "users.hh" -#include "terminal.hh" #include "editor-for.hh" #include "finally.hh" #include "markdown.hh" #include "local-fs-store.hh" #include "print.hh" +#include "ref.hh" +#include "value.hh" #if HAVE_BOEHMGC #define GC_INCLUDE_NEW #include #endif +#include "strings.hh" + namespace nix { /** @@ -75,6 +61,7 @@ enum class ProcessLineResult { struct NixRepl : AbstractNixRepl + , detail::ReplCompleterMixin #if HAVE_BOEHMGC , gc #endif @@ -90,17 +77,16 @@ struct NixRepl int displ; StringSet varNames; - const Path historyFile; + std::unique_ptr interacter; - NixRepl(const SearchPath & searchPath, nix::ref store,ref state, + NixRepl(const LookupPath & lookupPath, nix::ref store,ref state, std::function getValues); - virtual ~NixRepl(); + virtual ~NixRepl() = default; ReplExitStatus mainLoop() override; void initEnv() override; - StringSet completePrefix(const std::string & prefix); - bool getLine(std::string & input, const std::string & prompt); + virtual StringSet completePrefix(const std::string & prefix) override; StorePath getDerivationPath(Value & v); ProcessLineResult processLine(std::string line); @@ -123,7 +109,8 @@ struct NixRepl .force = true, .derivationPaths = true, .maxDepth = maxDepth, - .prettyIndent = 2 + .prettyIndent = 2, + .errors = ErrorPrintBehavior::ThrowTopLevel, }); } }; @@ -137,111 +124,33 @@ std::string removeWhitespace(std::string s) } -NixRepl::NixRepl(const SearchPath & searchPath, nix::ref store, ref state, +NixRepl::NixRepl(const LookupPath & lookupPath, nix::ref store, ref state, std::function getValues) : AbstractNixRepl(state) , debugTraceIndex(0) , getValues(getValues) , staticEnv(new StaticEnv(nullptr, state->staticBaseEnv.get())) - , historyFile(getDataDir() + "/nix/repl-history") + , interacter(make_unique(getDataDir() + "/nix/repl-history")) { } - -NixRepl::~NixRepl() -{ - write_history(historyFile.c_str()); -} - void runNix(Path program, const Strings & args, const std::optional & input = {}) { auto subprocessEnv = getEnv(); subprocessEnv["NIX_CONFIG"] = globalConfig.toKeyValue(); - + //isInteractive avoid grabling interactive commands runProgram2(RunOptions { .program = settings.nixBinDir+ "/" + program, .args = args, .environment = subprocessEnv, .input = input, + .isInteractive = true, }); return; } -static NixRepl * curRepl; // ugly - -static char * completionCallback(char * s, int *match) { - auto possible = curRepl->completePrefix(s); - if (possible.size() == 1) { - *match = 1; - auto *res = strdup(possible.begin()->c_str() + strlen(s)); - if (!res) throw Error("allocation failure"); - return res; - } else if (possible.size() > 1) { - auto checkAllHaveSameAt = [&](size_t pos) { - auto &first = *possible.begin(); - for (auto &p : possible) { - if (p.size() <= pos || p[pos] != first[pos]) - return false; - } - return true; - }; - size_t start = strlen(s); - size_t len = 0; - while (checkAllHaveSameAt(start + len)) ++len; - if (len > 0) { - *match = 1; - auto *res = strdup(std::string(*possible.begin(), start, len).c_str()); - if (!res) throw Error("allocation failure"); - return res; - } - } - - *match = 0; - return nullptr; -} - -static int listPossibleCallback(char *s, char ***avp) { - auto possible = curRepl->completePrefix(s); - - if (possible.size() > (INT_MAX / sizeof(char*))) - throw Error("too many completions"); - - int ac = 0; - char **vp = nullptr; - - auto check = [&](auto *p) { - if (!p) { - if (vp) { - while (--ac >= 0) - free(vp[ac]); - free(vp); - } - throw Error("allocation failure"); - } - return p; - }; - - vp = check((char **)malloc(possible.size() * sizeof(char*))); - - for (auto & p : possible) - vp[ac++] = check(strdup(p.c_str())); - - *avp = vp; - - return ac; -} - -namespace { - // Used to communicate to NixRepl::getLine whether a signal occurred in ::readline. - volatile sig_atomic_t g_signal_received = 0; - - void sigintHandler(int signo) { - g_signal_received = signo; - } -} - static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positions, const DebugTrace & dt) { if (dt.isError) @@ -281,24 +190,7 @@ ReplExitStatus NixRepl::mainLoop() loadFiles(); - // Allow nix-repl specific settings in .inputrc - rl_readline_name = "nix-repl"; - try { - createDirs(dirOf(historyFile)); - } catch (SystemError & e) { - logWarning(e.info()); - } -#ifndef USE_READLINE - el_hist_size = 1000; -#endif - read_history(historyFile.c_str()); - auto oldRepl = curRepl; - curRepl = this; - Finally restoreRepl([&] { curRepl = oldRepl; }); -#ifndef USE_READLINE - rl_set_complete_func(completionCallback); - rl_set_list_possib_func(listPossibleCallback); -#endif + auto _guard = interacter->init(static_cast(this)); std::string input; @@ -307,7 +199,7 @@ ReplExitStatus NixRepl::mainLoop() logger->pause(); // When continuing input from previous lines, don't print a prompt, just align to the same // number of chars as the prompt. - if (!getLine(input, input.empty() ? "nix-repl> " : " ")) { + if (!interacter->getLine(input, input.empty() ? ReplPromptType::ReplPrompt : ReplPromptType::ContinuationPrompt)) { // Ctrl-D should exit the debugger. state->debugStop = false; logger->cout(""); @@ -325,7 +217,7 @@ ReplExitStatus NixRepl::mainLoop() case ProcessLineResult::PromptAgain: break; default: - abort(); + unreachable(); } } catch (ParseError & e) { if (e.msg().find("unexpected end of file") != std::string::npos) { @@ -336,13 +228,7 @@ ReplExitStatus NixRepl::mainLoop() printMsg(lvlError, e.msg()); } } catch (EvalError & e) { - // in debugger mode, an EvalError should trigger another repl session. - // when that session returns the exception will land here. No need to show it again; - // show the error for this repl session instead. - if (state->debugRepl && !state->debugTraces.empty()) - showDebugTrace(std::cout, state->positions, state->debugTraces.front()); - else - printMsg(lvlError, e.msg()); + printMsg(lvlError, e.msg()); } catch (Error & e) { printMsg(lvlError, e.msg()); } catch (Interrupted & e) { @@ -356,51 +242,6 @@ ReplExitStatus NixRepl::mainLoop() } } - -bool NixRepl::getLine(std::string & input, const std::string & prompt) -{ - struct sigaction act, old; - sigset_t savedSignalMask, set; - - auto setupSignals = [&]() { - act.sa_handler = sigintHandler; - sigfillset(&act.sa_mask); - act.sa_flags = 0; - if (sigaction(SIGINT, &act, &old)) - throw SysError("installing handler for SIGINT"); - - sigemptyset(&set); - sigaddset(&set, SIGINT); - if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask)) - throw SysError("unblocking SIGINT"); - }; - auto restoreSignals = [&]() { - if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) - throw SysError("restoring signals"); - - if (sigaction(SIGINT, &old, 0)) - throw SysError("restoring handler for SIGINT"); - }; - - setupSignals(); - char * s = readline(prompt.c_str()); - Finally doFree([&]() { free(s); }); - restoreSignals(); - - if (g_signal_received) { - g_signal_received = 0; - input.clear(); - return true; - } - - if (!s) - return false; - input += s; - input += '\n'; - return true; -} - - StringSet NixRepl::completePrefix(const std::string & prefix) { StringSet completions; @@ -421,11 +262,14 @@ StringSet NixRepl::completePrefix(const std::string & prefix) try { auto dir = std::string(cur, 0, slash); auto prefix2 = std::string(cur, slash + 1); - for (auto & entry : readDirectory(dir == "" ? "/" : dir)) { - if (entry.name[0] != '.' && hasPrefix(entry.name, prefix2)) - completions.insert(prev + dir + "/" + entry.name); + for (auto & entry : std::filesystem::directory_iterator{dir == "" ? "/" : dir}) { + checkInterrupt(); + auto name = entry.path().filename().string(); + if (name[0] != '.' && hasPrefix(name, prefix2)) + completions.insert(prev + entry.path().string()); } } catch (Error &) { + } catch (std::filesystem::filesystem_error &) { } } else if ((dot = cur.rfind('.')) == std::string::npos) { /* This is a variable name; look it up in the current scope. */ @@ -452,7 +296,7 @@ StringSet NixRepl::completePrefix(const std::string & prefix) e->eval(*state, *env, v); state->forceAttrs(v, noPos, "while evaluating an attrset for the purpose of completion (this error should not be displayed; file an issue?)"); - for (auto & i : *v.attrs) { + for (auto & i : *v.attrs()) { std::string_view name = state->symbols[i.name]; if (name.substr(0, cur2.size()) != cur2) continue; completions.insert(concatStrings(prev, expr, ".", name)); @@ -464,6 +308,8 @@ StringSet NixRepl::completePrefix(const std::string & prefix) // Quietly ignore evaluation errors. } catch (BadURL & e) { // Quietly ignore BadURL flake-related errors. + } catch (FileNotFound & e) { + // Quietly ignore non-existent file beeing `import`-ed. } } @@ -519,7 +365,7 @@ ProcessLineResult NixRepl::processLine(std::string line) if (line.empty()) return ProcessLineResult::PromptAgain; - _isInterrupted = false; + setInterrupted(false); std::string command, arg; @@ -548,6 +394,7 @@ ProcessLineResult NixRepl::processLine(std::string line) << " :l, :load Load Nix expression and add it to scope\n" << " :lf, :load-flake Load Nix flake and add it to scope\n" << " :p, :print Evaluate and print expression recursively\n" + << " Strings are printed directly, without escaping.\n" << " :q, :quit Exit nix-repl\n" << " :r, :reload Reload all files\n" << " :sh Build dependencies of derivation, then start\n" @@ -651,7 +498,7 @@ ProcessLineResult NixRepl::processLine(std::string line) auto path = state->coerceToPath(noPos, v, context, "while evaluating the filename to edit"); return {path, 0}; } else if (v.isLambda()) { - auto pos = state->positions[v.lambda.fun->pos]; + auto pos = state->positions[v.payload.lambda.fun->pos]; if (auto path = std::get_if(&pos.origin)) return {*path, pos.line}; else @@ -669,7 +516,7 @@ ProcessLineResult NixRepl::processLine(std::string line) // runProgram redirects stdout to a StringSink, // using runProgram2 to allow editors to display their UI - runProgram2(RunOptions { .program = editor, .searchPath = true, .args = args }); + runProgram2(RunOptions { .program = editor, .lookupPath = true, .args = args , .isInteractive = true }); // Reload right after exiting the editor state->resetFileCache(); @@ -755,7 +602,11 @@ ProcessLineResult NixRepl::processLine(std::string line) else if (command == ":p" || command == ":print") { Value v; evalString(arg, v); - printValue(std::cout, v); + if (v.type() == nString) { + std::cout << v.string_view(); + } else { + printValue(std::cout, v); + } std::cout << std::endl; } @@ -766,6 +617,35 @@ ProcessLineResult NixRepl::processLine(std::string line) else if (command == ":doc") { Value v; + + auto expr = parseString(arg); + std::string fallbackName; + PosIdx fallbackPos; + DocComment fallbackDoc; + if (auto select = dynamic_cast(expr)) { + Value vAttrs; + auto name = select->evalExceptFinalSelect(*state, *env, vAttrs); + fallbackName = state->symbols[name]; + + state->forceAttrs(vAttrs, noPos, "while evaluating an attribute set to look for documentation"); + auto attrs = vAttrs.attrs(); + assert(attrs); + auto attr = attrs->get(name); + if (!attr) { + // When missing, trigger the normal exception + // e.g. :doc builtins.foo + // behaves like + // nix-repl> builtins.foo + // error: attribute 'foo' missing + evalString(arg, v); + assert(false); + } + if (attr->pos) { + fallbackPos = attr->pos; + fallbackDoc = state->getDocCommentForPos(fallbackPos); + } + } + evalString(arg, v); if (auto doc = state->getDoc(v)) { std::string markdown; @@ -783,6 +663,19 @@ ProcessLineResult NixRepl::processLine(std::string line) markdown += stripIndentation(doc->doc); logger->cout(trim(renderMarkdownToTerminal(markdown))); + } else if (fallbackPos) { + std::stringstream ss; + ss << "Attribute `" << fallbackName << "`\n\n"; + ss << " … defined at " << state->positions[fallbackPos] << "\n\n"; + if (fallbackDoc) { + ss << fallbackDoc.getInnerText(state->positions); + } else { + ss << "No documentation found.\n\n"; + } + + auto markdown = ss.str(); + logger->cout(trim(renderMarkdownToTerminal(markdown))); + } else throw Error("value does not have documentation"); } @@ -840,14 +733,14 @@ void NixRepl::loadFlake(const std::string & flakeRefS) if (flakeRefS.empty()) throw Error("cannot use ':load-flake' without a path specified. (Use '.' for the current working directory.)"); - auto flakeRef = parseFlakeRef(flakeRefS, absPath("."), true); + auto flakeRef = parseFlakeRef(fetchSettings, flakeRefS, absPath("."), true); if (evalSettings.pureEval && !flakeRef.input.isLocked()) throw Error("cannot use ':load-flake' on locked flake reference '%s' (use --impure to override)", flakeRefS); Value v; flake::callFlake(*state, - flake::lockFlake(*state, flakeRef, + flake::lockFlake(flakeSettings, *state, flakeRef, flake::LockFlags { .updateLockFile = false, .useRegistries = !evalSettings.pureEval, @@ -899,17 +792,17 @@ void NixRepl::loadFiles() void NixRepl::addAttrsToScope(Value & attrs) { state->forceAttrs(attrs, [&]() { return attrs.determinePos(noPos); }, "while evaluating an attribute set to be merged in the global scope"); - if (displ + attrs.attrs->size() >= envSize) + if (displ + attrs.attrs()->size() >= envSize) throw Error("environment full; cannot add more variables"); - for (auto & i : *attrs.attrs) { + for (auto & i : *attrs.attrs()) { staticEnv->vars.emplace_back(i.name, displ); env->values[displ++] = i.value; varNames.emplace(state->symbols[i.name]); } staticEnv->sort(); staticEnv->deduplicate(); - notice("Added %1% variables.", attrs.attrs->size()); + notice("Added %1% variables.", attrs.attrs()->size()); } @@ -941,11 +834,11 @@ void NixRepl::evalString(std::string s, Value & v) std::unique_ptr AbstractNixRepl::create( - const SearchPath & searchPath, nix::ref store, ref state, + const LookupPath & lookupPath, nix::ref store, ref state, std::function getValues) { return std::make_unique( - searchPath, + lookupPath, openStore(), state, getValues @@ -961,9 +854,9 @@ ReplExitStatus AbstractNixRepl::runSimple( NixRepl::AnnotatedValues values; return values; }; - SearchPath searchPath = {}; + LookupPath lookupPath = {}; auto repl = std::make_unique( - searchPath, + lookupPath, openStore(), evalState, getValues diff --git a/src/libcmd/repl.hh b/src/libcmd/repl.hh index 21aa8bfc7..3fd4b2c39 100644 --- a/src/libcmd/repl.hh +++ b/src/libcmd/repl.hh @@ -3,11 +3,6 @@ #include "eval.hh" -#if HAVE_BOEHMGC -#define GC_INCLUDE_NEW -#include -#endif - namespace nix { struct AbstractNixRepl @@ -25,7 +20,7 @@ struct AbstractNixRepl typedef std::vector> AnnotatedValues; static std::unique_ptr create( - const SearchPath & searchPath, nix::ref store, ref state, + const LookupPath & lookupPath, nix::ref store, ref state, std::function getValues); static ReplExitStatus runSimple( diff --git a/src/libexpr-c/.version b/src/libexpr-c/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libexpr-c/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libexpr-c/build-utils-meson b/src/libexpr-c/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libexpr-c/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libexpr-c/local.mk b/src/libexpr-c/local.mk new file mode 100644 index 000000000..227a4095b --- /dev/null +++ b/src/libexpr-c/local.mk @@ -0,0 +1,25 @@ +libraries += libexprc + +libexprc_NAME = libnixexprc + +libexprc_DIR := $(d) + +libexprc_SOURCES := \ + $(wildcard $(d)/*.cc) \ + +# Not just for this library itself, but also for downstream libraries using this library + +INCLUDE_libexprc := -I $(d) +libexprc_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libutilc) \ + $(INCLUDE_libfetchers) \ + $(INCLUDE_libstore) $(INCLUDE_libstorec) \ + $(INCLUDE_libexpr) $(INCLUDE_libexprc) + +libexprc_LIBS = libutil libutilc libstore libstorec libfetchers libexpr + +libexprc_LDFLAGS += $(THREAD_LDFLAGS) + +$(eval $(call install-file-in, $(d)/nix-expr-c.pc, $(libdir)/pkgconfig, 0644)) + +libexprc_FORCE_INSTALL := 1 + diff --git a/src/libexpr-c/meson.build b/src/libexpr-c/meson.build new file mode 100644 index 000000000..6db5b83b8 --- /dev/null +++ b/src/libexpr-c/meson.build @@ -0,0 +1,93 @@ +project('nix-expr-c', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), + dependency('nix-expr'), +] +deps_public_maybe_subproject = [ + dependency('nix-util-c'), + dependency('nix-store-c'), +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +# TODO rename, because it will conflict with downstream projects +configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) + +config_h = configure_file( + configuration : configdata, + output : 'config-expr.h', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + + # From C++ libraries, only for internals + '-include', 'config-util.hh', + '-include', 'config-store.hh', + '-include', 'config-expr.hh', + + # From C libraries, for our public, installed headers too + '-include', 'config-util.h', + '-include', 'config-store.h', + '-include', 'config-expr.h', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'nix_api_expr.cc', + 'nix_api_external.cc', + 'nix_api_value.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'nix_api_expr.h', + 'nix_api_external.h', + 'nix_api_value.h', +) + +# TODO move this header to libexpr, maybe don't use it in tests? +headers += files('nix_api_expr_internal.h') + +subdir('build-utils-meson/export-all-symbols') + +this_library = library( + 'nixexprc', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + link_args: linker_export_flags, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libexpr-c/nix-expr-c.pc.in b/src/libexpr-c/nix-expr-c.pc.in new file mode 100644 index 000000000..06897064d --- /dev/null +++ b/src/libexpr-c/nix-expr-c.pc.in @@ -0,0 +1,10 @@ +prefix=@prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: Nix +Description: Nix Language Evaluator - C API +Version: @PACKAGE_VERSION@ +Requires: nix-store-c +Libs: -L${libdir} -lnixexprc +Cflags: -I${includedir}/nix diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc new file mode 100644 index 000000000..8f21d7022 --- /dev/null +++ b/src/libexpr-c/nix_api_expr.cc @@ -0,0 +1,213 @@ +#include +#include +#include + +#include "eval.hh" +#include "eval-gc.hh" +#include "globals.hh" +#include "eval-settings.hh" + +#include "nix_api_expr.h" +#include "nix_api_expr_internal.h" +#include "nix_api_store.h" +#include "nix_api_store_internal.h" +#include "nix_api_util.h" +#include "nix_api_util_internal.h" + +#if HAVE_BOEHMGC +# include +# define GC_INCLUDE_NEW 1 +# include "gc_cpp.h" +#endif + +nix_err nix_libexpr_init(nix_c_context * context) +{ + if (context) + context->last_err_code = NIX_OK; + { + auto ret = nix_libutil_init(context); + if (ret != NIX_OK) + return ret; + } + { + auto ret = nix_libstore_init(context); + if (ret != NIX_OK) + return ret; + } + try { + nix::initGC(); + } + NIXC_CATCH_ERRS +} + +nix_err nix_expr_eval_from_string( + nix_c_context * context, EvalState * state, const char * expr, const char * path, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + nix::Expr * parsedExpr = state->state.parseExprFromString(expr, state->state.rootPath(nix::CanonPath(path))); + state->state.eval(parsedExpr, value->value); + state->state.forceValue(value->value, nix::noPos); + } + NIXC_CATCH_ERRS +} + +nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, nix_value * arg, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + state->state.callFunction(fn->value, arg->value, value->value, nix::noPos); + state->state.forceValue(value->value, nix::noPos); + } + NIXC_CATCH_ERRS +} + +nix_err nix_value_call_multi(nix_c_context * context, EvalState * state, nix_value * fn, size_t nargs, nix_value ** args, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + state->state.callFunction(fn->value, nargs, (nix::Value * *)args, value->value, nix::noPos); + state->state.forceValue(value->value, nix::noPos); + } + NIXC_CATCH_ERRS +} + +nix_err nix_value_force(nix_c_context * context, EvalState * state, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + state->state.forceValue(value->value, nix::noPos); + } + NIXC_CATCH_ERRS +} + +nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + state->state.forceValueDeep(value->value); + } + NIXC_CATCH_ERRS +} + +EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c, Store * store) +{ + if (context) + context->last_err_code = NIX_OK; + try { + nix::Strings lookupPath; + if (lookupPath_c != nullptr) + for (size_t i = 0; lookupPath_c[i] != nullptr; i++) + lookupPath.push_back(lookupPath_c[i]); + + void * p = ::operator new( + sizeof(EvalState), + static_cast(alignof(EvalState))); + auto * p2 = static_cast(p); + new (p) EvalState { + .fetchSettings = nix::fetchers::Settings{}, + .settings = nix::EvalSettings{ + nix::settings.readOnlyMode, + }, + .state = nix::EvalState( + nix::LookupPath::parse(lookupPath), + store->ptr, + p2->fetchSettings, + p2->settings), + }; + loadConfFile(p2->settings); + return p2; + } + NIXC_CATCH_ERRS_NULL +} + +void nix_state_free(EvalState * state) +{ + delete state; +} + +#if HAVE_BOEHMGC +std::unordered_map< + const void *, + unsigned int, + std::hash, + std::equal_to, + traceable_allocator
; } + > ``` + > + > For human-readable output, `nix eval` (experimental) is more informative: + > + > ```console + > $ nix-instantiate --eval --expr 'a: a' + > + > $ nix eval --expr 'a: a' + > «lambda @ «string»:1:1» + > ``` + > + > For machine-readable output, the `--xml` option produces unambiguous + > output: + > + > ```console + > $ nix-instantiate --eval --xml --expr '{ foo = ; }' + > + > + > + > + > + > + > + > + > ``` - - `--json`\ - When used with `--eval`, print the resulting value as an JSON - representation of the abstract syntax tree rather than as a Nix expression. +- `--find-file` - - `--xml`\ - When used with `--eval`, print the resulting value as an XML - representation of the abstract syntax tree rather than as a Nix expression. - The schema is the same as that used by the [`toXML` - built-in](../language/builtins.md). + Look up the given files in Nix’s search path (as specified by the + `NIX_PATH` environment variable). If found, print the corresponding + absolute paths on standard output. For instance, if `NIX_PATH` is + `nixpkgs=/home/alice/nixpkgs`, then `nix-instantiate --find-file + nixpkgs/default.nix` will print `/home/alice/nixpkgs/default.nix`. - - `--read-write-mode`\ - When used with `--eval`, perform evaluation in read/write mode so - nix language features that require it will still work (at the cost - of needing to do instantiation of every evaluated derivation). If - this option is not enabled, there may be uninstantiated store paths - in the final output. +- `--strict` + + When used with `--eval`, recursively evaluate list elements and + attributes. Normally, such sub-expressions are left unevaluated + (since the Nix language is lazy). + + > **Warning** + > + > This option can cause non-termination, because lazy data + > structures can be infinitely large. + +- `--json` + + When used with `--eval`, print the resulting value as an JSON + representation of the abstract syntax tree rather than as a Nix expression. + +- `--xml` + + When used with `--eval`, print the resulting value as an XML + representation of the abstract syntax tree rather than as a Nix expression. + The schema is the same as that used by the [`toXML` + built-in](../language/builtins.md). + +- `--read-write-mode` + + When used with `--eval`, perform evaluation in read/write mode so + nix language features that require it will still work (at the cost + of needing to do instantiation of every evaluated derivation). If + this option is not enabled, there may be uninstantiated store paths + in the final output. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-prefetch-url.md b/doc/manual/src/command-ref/nix-prefetch-url.md index 45ef01e02..ffab94b8a 100644 --- a/doc/manual/src/command-ref/nix-prefetch-url.md +++ b/doc/manual/src/command-ref/nix-prefetch-url.md @@ -39,27 +39,32 @@ the path of the downloaded file in the Nix store is also printed. # Options - - `--type` *hashAlgo*\ - Use the specified cryptographic hash algorithm, - which can be one of `md5`, `sha1`, `sha256`, and `sha512`. - The default is `sha256`. +- `--type` *hashAlgo* - - `--print-path`\ - Print the store path of the downloaded file on standard output. + Use the specified cryptographic hash algorithm, + which can be one of `md5`, `sha1`, `sha256`, and `sha512`. + The default is `sha256`. - - `--unpack`\ - Unpack the archive (which must be a tarball or zip file) and add the - result to the Nix store. The resulting hash can be used with - functions such as Nixpkgs’s `fetchzip` or `fetchFromGitHub`. +- `--print-path` - - `--executable`\ - Set the executable bit on the downloaded file. + Print the store path of the downloaded file on standard output. - - `--name` *name*\ - Override the name of the file in the Nix store. By default, this is - `hash-basename`, where *basename* is the last component of *url*. - Overriding the name is necessary when *basename* contains characters - that are not allowed in Nix store paths. +- `--unpack` + + Unpack the archive (which must be a tarball or zip file) and add the + result to the Nix store. The resulting hash can be used with + functions such as Nixpkgs’s `fetchzip` or `fetchFromGitHub`. + +- `--executable` + + Set the executable bit on the downloaded file. + +- `--name` *name* + + Override the name of the file in the Nix store. By default, this is + `hash-basename`, where *basename* is the last component of *url*. + Overriding the name is necessary when *basename* contains characters + that are not allowed in Nix store paths. # Examples diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index 1eaf3c36a..69a711bd5 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -60,55 +60,63 @@ All options not listed here are passed to `nix-store --realise`, except for `--arg` and `--attr` / `-A` which are passed to `nix-instantiate`. - - `--command` *cmd*\ - In the environment of the derivation, run the shell command *cmd*. - This command is executed in an interactive shell. (Use `--run` to - use a non-interactive shell instead.) However, a call to `exit` is - implicitly added to the command, so the shell will exit after - running the command. To prevent this, add `return` at the end; - e.g. `--command "echo Hello; return"` will print `Hello` and then - drop you into the interactive shell. This can be useful for doing - any additional initialisation. +- `--command` *cmd* - - `--run` *cmd*\ - Like `--command`, but executes the command in a non-interactive - shell. This means (among other things) that if you hit Ctrl-C while - the command is running, the shell exits. + In the environment of the derivation, run the shell command *cmd*. + This command is executed in an interactive shell. (Use `--run` to + use a non-interactive shell instead.) However, a call to `exit` is + implicitly added to the command, so the shell will exit after + running the command. To prevent this, add `return` at the end; + e.g. `--command "echo Hello; return"` will print `Hello` and then + drop you into the interactive shell. This can be useful for doing + any additional initialisation. - - `--exclude` *regexp*\ - Do not build any dependencies whose store path matches the regular - expression *regexp*. This option may be specified multiple times. +- `--run` *cmd* - - `--pure`\ - If this flag is specified, the environment is almost entirely - cleared before the interactive shell is started, so you get an - environment that more closely corresponds to the “real†Nix build. A - few variables, in particular `HOME`, `USER` and `DISPLAY`, are - retained. + Like `--command`, but executes the command in a non-interactive + shell. This means (among other things) that if you hit Ctrl-C while + the command is running, the shell exits. - - `--packages` / `-p` *packages*…\ - Set up an environment in which the specified packages are present. - The command line arguments are interpreted as attribute names inside - the Nix Packages collection. Thus, `nix-shell --packages libjpeg openjdk` - will start a shell in which the packages denoted by the attribute - names `libjpeg` and `openjdk` are present. +- `--exclude` *regexp* - - `-i` *interpreter*\ - The chained script interpreter to be invoked by `nix-shell`. Only - applicable in `#!`-scripts (described below). + Do not build any dependencies whose store path matches the regular + expression *regexp*. This option may be specified multiple times. - - `--keep` *name*\ - When a `--pure` shell is started, keep the listed environment - variables. +- `--pure` + + If this flag is specified, the environment is almost entirely + cleared before the interactive shell is started, so you get an + environment that more closely corresponds to the “real†Nix build. A + few variables, in particular `HOME`, `USER` and `DISPLAY`, are + retained. + +- `--packages` / `-p` *packages*… + + Set up an environment in which the specified packages are present. + The command line arguments are interpreted as attribute names inside + the Nix Packages collection. Thus, `nix-shell --packages libjpeg openjdk` + will start a shell in which the packages denoted by the attribute + names `libjpeg` and `openjdk` are present. + +- `-i` *interpreter* + + The chained script interpreter to be invoked by `nix-shell`. Only + applicable in `#!`-scripts (described below). + +- `--keep` *name* + + When a `--pure` shell is started, keep the listed environment + variables. {{#include ./opt-common.md}} # Environment variables - - `NIX_BUILD_SHELL`\ - Shell used to start the interactive environment. Defaults to the - `bash` found in ``, falling back to the `bash` found in - `PATH` if not found. +- `NIX_BUILD_SHELL` + + Shell used to start the interactive environment. Defaults to the + `bash` found in ``, falling back to the `bash` found in + `PATH` if not found. {{#include ./env-common.md}} @@ -202,14 +210,14 @@ For example, here is a Python script that depends on Python and the ```python #! /usr/bin/env nix-shell -#! nix-shell -i python --packages python pythonPackages.prettytable +#! nix-shell -i python3 --packages python3 python3Packages.prettytable import prettytable # Print a simple table. t = prettytable.PrettyTable(["N", "N^2"]) for n in range(1, 10): t.add_row([n, n * n]) -print t +print(t) ``` Similarly, the following is a Perl script that specifies that it @@ -289,3 +297,8 @@ with import {}; runCommand "dummy" { buildInputs = [ python pythonPackages.prettytable ]; } "" ``` + +The script's file name is passed as the first argument to the interpreter specified by the `-i` flag. + +Aside from the very first line, which is a directive to the operating system, the additional `#! nix-shell` lines do not need to be at the beginning of the file. +This allows wrapping them in block comments for languages where `#` does not start a comment, such as ECMAScript, Erlang, PHP, or Ruby. diff --git a/doc/manual/src/command-ref/nix-store/add-fixed.md b/doc/manual/src/command-ref/nix-store/add-fixed.md index d25db091c..bebf15026 100644 --- a/doc/manual/src/command-ref/nix-store/add-fixed.md +++ b/doc/manual/src/command-ref/nix-store/add-fixed.md @@ -16,9 +16,10 @@ public url or broke since the download expression was written. This operation has the following options: - - `--recursive`\ - Use recursive instead of flat hashing mode, used when adding - directories to the store. +- `--recursive` + + Use recursive instead of flat hashing mode, used when adding + directories to the store. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/dump.md b/doc/manual/src/command-ref/nix-store/dump.md index c2f3c42ef..3de0e27b0 100644 --- a/doc/manual/src/command-ref/nix-store/dump.md +++ b/doc/manual/src/command-ref/nix-store/dump.md @@ -1,6 +1,6 @@ # Name -`nix-store --dump` - write a single path to a Nix Archive +`nix-store --dump` - write a single path to a [Nix Archive] ## Synopsis @@ -8,7 +8,7 @@ ## Description -The operation `--dump` produces a NAR (Nix ARchive) file containing the +The operation `--dump` produces a [Nix archive](@docroot@/glossary.md#gloss-nar) (NAR) file containing the contents of the file system tree rooted at *path*. The archive is written to standard output. @@ -30,8 +30,9 @@ NAR archives support filenames of unlimited length and 64-bit file sizes. They can contain regular files, directories, and symbolic links, but not other types of files (such as device nodes). -A Nix archive can be unpacked using `nix-store ---restore`. +A Nix archive can be unpacked using [`nix-store --restore`](@docroot@/command-ref/nix-store/restore.md). + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/export.md b/doc/manual/src/command-ref/nix-store/export.md index 1bc46f53b..ba772eb43 100644 --- a/doc/manual/src/command-ref/nix-store/export.md +++ b/doc/manual/src/command-ref/nix-store/export.md @@ -1,6 +1,6 @@ # Name -`nix-store --export` - export store paths to a Nix Archive +`nix-store --export` - export store paths to a [Nix Archive] ## Synopsis @@ -8,16 +8,22 @@ ## Description -The operation `--export` writes a serialisation of the specified store -paths to standard output in a format that can be imported into another -Nix store with `nix-store --import`. This is like `nix-store ---dump`, except that the NAR archive produced by that command doesn’t -contain the necessary meta-information to allow it to be imported into -another Nix store (namely, the set of references of the path). +The operation `--export` writes a serialisation of the given [store objects](@docroot@/glossary.md#gloss-store-object) to standard output in a format that can be imported into another [Nix store](@docroot@/store/index.md) with [`nix-store --import`](./import.md). -This command does not produce a *closure* of the specified paths, so if -a store path references other store paths that are missing in the target -Nix store, the import will fail. +> **Warning** +> +> This command *does not* produce a [closure](@docroot@/glossary.md#gloss-closure) of the specified store paths. +> Trying to import a store object that refers to store paths not available in the target Nix store will fail. +> +> Use [`nix-store --query`](@docroot@/command-ref/nix-store/query.md) to obtain the closure of a store path. + +This command is different from [`nix-store --dump`](./dump.md), which produces a [Nix archive](@docroot@/glossary.md#gloss-nar) that *does not* contain the set of [references](@docroot@/glossary.md#gloss-reference) of a given store path. + +> **Note** +> +> For efficient transfer of closures to remote machines over SSH, use [`nix-copy-closure`](@docroot@/command-ref/nix-copy-closure.md). + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive {{#include ./opt-common.md}} @@ -27,15 +33,21 @@ Nix store, the import will fail. # Examples -To copy a whole closure, do something -like: - -```console -$ nix-store --export $(nix-store --query --requisites paths) > out -``` - -To import the whole closure again, run: - -```console -$ nix-store --import < out -``` +> **Example** +> +> Deploy GNU Hello to an airgapped machine via USB stick. +> +> Write the closure to the block device on a machine with internet connection: +> +> ```shell-session +> [alice@itchy]$ storePath=$(nix-build '' -I nixpkgs=channel:nixpkgs-unstable -A hello --no-out-link) +> [alice@itchy]$ nix-store --export $(nix-store --query --requisites $storePath) | sudo dd of=/dev/usb +> ``` +> +> Read the closure from the block device on the machine without internet connection: +> +> ```shell-session +> [bob@scratchy]$ hello=$(sudo dd if=/dev/usb | nix-store --import | tail -1) +> [bob@scratchy]$ $hello/bin/hello +> Hello, world! +> ``` diff --git a/doc/manual/src/command-ref/nix-store/gc.md b/doc/manual/src/command-ref/nix-store/gc.md index 7be0d559a..f432e00eb 100644 --- a/doc/manual/src/command-ref/nix-store/gc.md +++ b/doc/manual/src/command-ref/nix-store/gc.md @@ -14,30 +14,34 @@ reachable via file system references from a set of “rootsâ€, are deleted. The following suboperations may be specified: - - `--print-roots`\ - This operation prints on standard output the set of roots used by - the garbage collector. +- `--print-roots` - - `--print-live`\ - This operation prints on standard output the set of “live†store - paths, which are all the store paths reachable from the roots. Live - paths should never be deleted, since that would break consistency — - it would become possible that applications are installed that - reference things that are no longer present in the store. + This operation prints on standard output the set of roots used by + the garbage collector. - - `--print-dead`\ - This operation prints out on standard output the set of “dead†store - paths, which is just the opposite of the set of live paths: any path - in the store that is not live (with respect to the roots) is dead. +- `--print-live` + + This operation prints on standard output the set of “live†store + paths, which are all the store paths reachable from the roots. Live + paths should never be deleted, since that would break consistency — + it would become possible that applications are installed that + reference things that are no longer present in the store. + +- `--print-dead` + + This operation prints out on standard output the set of “dead†store + paths, which is just the opposite of the set of live paths: any path + in the store that is not live (with respect to the roots) is dead. By default, all unreachable paths are deleted. The following options control what gets deleted and in what order: - - `--max-freed` *bytes*\ - Keep deleting paths until at least *bytes* bytes have been deleted, - then stop. The argument *bytes* can be followed by the - multiplicative suffix `K`, `M`, `G` or `T`, denoting KiB, MiB, GiB - or TiB units. +- `--max-freed` *bytes* + + Keep deleting paths until at least *bytes* bytes have been deleted, + then stop. The argument *bytes* can be followed by the + multiplicative suffix `K`, `M`, `G` or `T`, denoting KiB, MiB, GiB + or TiB units. The behaviour of the collector is also influenced by the `keep-outputs` and `keep-derivations` settings in the Nix diff --git a/doc/manual/src/command-ref/nix-store/import.md b/doc/manual/src/command-ref/nix-store/import.md index 2711316a7..3f6b3d076 100644 --- a/doc/manual/src/command-ref/nix-store/import.md +++ b/doc/manual/src/command-ref/nix-store/import.md @@ -1,6 +1,8 @@ # Name -`nix-store --import` - import Nix Archive into the store +`nix-store --import` - import [Nix Archive] into the store + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive # Synopsis @@ -8,14 +10,34 @@ # Description -The operation `--import` reads a serialisation of a set of store paths -produced by `nix-store --export` from standard input and adds those -store paths to the Nix store. Paths that already exist in the Nix store -are ignored. If a path refers to another path that doesn’t exist in the -Nix store, the import fails. +The operation `--import` reads a serialisation of a set of [store objects](@docroot@/glossary.md#gloss-store-object) produced by [`nix-store --export`](./export.md) from standard input, and adds those store objects to the specified [Nix store](@docroot@/store/index.md). +Paths that already exist in the target Nix store are ignored. +If a path [refers](@docroot@/glossary.md#gloss-reference) to another path that doesn’t exist in the target Nix store, the import fails. + +> **Note** +> +> For efficient transfer of closures to remote machines over SSH, use [`nix-copy-closure`](@docroot@/command-ref/nix-copy-closure.md). {{#include ./opt-common.md}} {{#include ../opt-common.md}} {{#include ../env-common.md}} + +# Examples + +> **Example** +> +> Given a closure of GNU Hello as a file: +> +> ```shell-session +> $ storePath="$(nix-build '' -I nixpkgs=channel:nixpkgs-unstable -A hello --no-out-link)" +> $ nix-store --export $(nix-store --query --requisites $storePath) > hello.closure +> ``` +> +> Import the closure into a [remote SSH store](@docroot@/store/types/ssh-store.md) using the [`--store`](@docroot@/command-ref/conf-file.md#conf-store) option: +> +> ```console +> $ nix-store --import --store ssh://alice@itchy.example.org < hello.closure +> ``` + diff --git a/doc/manual/src/command-ref/nix-store/optimise.md b/doc/manual/src/command-ref/nix-store/optimise.md index dc392aeb8..b257466b2 100644 --- a/doc/manual/src/command-ref/nix-store/optimise.md +++ b/doc/manual/src/command-ref/nix-store/optimise.md @@ -12,7 +12,7 @@ The operation `--optimise` reduces Nix store disk space usage by finding identical files in the store and hard-linking them to each other. It typically reduces the size of the store by something like 25-35%. Only regular files and symlinks are hard-linked in this manner. Files are -considered identical when they have the same NAR archive serialisation: +considered identical when they have the same [Nix Archive (NAR)][Nix Archive] serialisation: that is, regular files must have the same contents and permission (executable or non-executable), and symlinks must have the same contents. @@ -38,3 +38,4 @@ hashing files in `/nix/store/qhqx7l2f1kmwihc9bnxs7rc159hsxnf3-gcc-4.1.1' there are 114486 files with equal contents out of 215894 files in total ``` +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive diff --git a/doc/manual/src/command-ref/nix-store/query.md b/doc/manual/src/command-ref/nix-store/query.md index a158c76aa..b4efa734e 100644 --- a/doc/manual/src/command-ref/nix-store/query.md +++ b/doc/manual/src/command-ref/nix-store/query.md @@ -24,122 +24,138 @@ symlink. # Common query options - - `--use-output`; `-u`\ - For each argument to the query that is a [store derivation], apply the - query to the output path of the derivation instead. +- `--use-output` / `-u` - - `--force-realise`; `-f`\ - Realise each argument to the query first (see [`nix-store --realise`](./realise.md)). + For each argument to the query that is a [store derivation], apply the + query to the output path of the derivation instead. + +- `--force-realise` / `-f` + + Realise each argument to the query first (see [`nix-store --realise`](./realise.md)). [store derivation]: @docroot@/glossary.md#gloss-store-derivation # Queries - - `--outputs`\ - Prints out the [output paths] of the store - derivations *paths*. These are the paths that will be produced when - the derivation is built. +- `--outputs` - [output paths]: ../../glossary.md#gloss-output-path + Prints out the [output paths] of the store + derivations *paths*. These are the paths that will be produced when + the derivation is built. - - `--requisites`; `-R`\ - Prints out the [closure] of the store path *paths*. + [output paths]: @docroot@/glossary.md#gloss-output-path - [closure]: ../../glossary.md#gloss-closure +- `--requisites` / `-R` - This query has one option: + Prints out the [closure] of the store path *paths*. - - `--include-outputs` - Also include the existing output paths of [store derivation]s, - and their closures. + [closure]: @docroot@/glossary.md#gloss-closure - This query can be used to implement various kinds of deployment. A - *source deployment* is obtained by distributing the closure of a - store derivation. A *binary deployment* is obtained by distributing - the closure of an output path. A *cache deployment* (combined - source/binary deployment, including binaries of build-time-only - dependencies) is obtained by distributing the closure of a store - derivation and specifying the option `--include-outputs`. + This query has one option: - - `--references`\ - Prints the set of [references] of the store paths - *paths*, that is, their immediate dependencies. (For *all* - dependencies, use `--requisites`.) + - `--include-outputs` + Also include the existing output paths of [store derivation]s, + and their closures. - [references]: ../../glossary.md#gloss-reference + This query can be used to implement various kinds of deployment. A + *source deployment* is obtained by distributing the closure of a + store derivation. A *binary deployment* is obtained by distributing + the closure of an output path. A *cache deployment* (combined + source/binary deployment, including binaries of build-time-only + dependencies) is obtained by distributing the closure of a store + derivation and specifying the option `--include-outputs`. - - `--referrers`\ - Prints the set of *referrers* of the store paths *paths*, that is, - the store paths currently existing in the Nix store that refer to - one of *paths*. Note that contrary to the references, the set of - referrers is not constant; it can change as store paths are added or - removed. +- `--references` - - `--referrers-closure`\ - Prints the closure of the set of store paths *paths* under the - referrers relation; that is, all store paths that directly or - indirectly refer to one of *paths*. These are all the path currently - in the Nix store that are dependent on *paths*. + Prints the set of [references] of the store paths + *paths*, that is, their immediate dependencies. (For *all* + dependencies, use `--requisites`.) - - `--deriver`; `-d`\ - Prints the [deriver] that was used to build the store paths *paths*. If - the path has no deriver (e.g., if it is a source file), or if the - deriver is not known (e.g., in the case of a binary-only - deployment), the string `unknown-deriver` is printed. - The returned deriver is not guaranteed to exist in the local store, for - example when *paths* were substituted from a binary cache. - Use `--valid-derivers` instead to obtain valid paths only. + [references]: @docroot@/glossary.md#gloss-reference - [deriver]: ../../glossary.md#gloss-deriver +- `--referrers` - - `--valid-derivers`\ - Prints a set of derivation files (`.drv`) which are supposed produce - said paths when realized. Might print nothing, for example for source paths - or paths subsituted from a binary cache. + Prints the set of *referrers* of the store paths *paths*, that is, + the store paths currently existing in the Nix store that refer to + one of *paths*. Note that contrary to the references, the set of + referrers is not constant; it can change as store paths are added or + removed. - - `--graph`\ - Prints the references graph of the store paths *paths* in the format - of the `dot` tool of AT\&T's [Graphviz - package](http://www.graphviz.org/). This can be used to visualise - dependency graphs. To obtain a build-time dependency graph, apply - this to a store derivation. To obtain a runtime dependency graph, - apply it to an output path. +- `--referrers-closure` - - `--tree`\ - Prints the references graph of the store paths *paths* as a nested - ASCII tree. References are ordered by descending closure size; this - tends to flatten the tree, making it more readable. The query only - recurses into a store path when it is first encountered; this - prevents a blowup of the tree representation of the graph. + Prints the closure of the set of store paths *paths* under the + referrers relation; that is, all store paths that directly or + indirectly refer to one of *paths*. These are all the path currently + in the Nix store that are dependent on *paths*. - - `--graphml`\ - Prints the references graph of the store paths *paths* in the - [GraphML](http://graphml.graphdrawing.org/) file format. This can be - used to visualise dependency graphs. To obtain a build-time - dependency graph, apply this to a [store derivation]. To obtain a - runtime dependency graph, apply it to an output path. +- `--deriver` / `-d` - - `--binding` *name*; `-b` *name*\ - Prints the value of the attribute *name* (i.e., environment - variable) of the [store derivation]s *paths*. It is an error for a - derivation to not have the specified attribute. + Prints the [deriver] that was used to build the store paths *paths*. If + the path has no deriver (e.g., if it is a source file), or if the + deriver is not known (e.g., in the case of a binary-only + deployment), the string `unknown-deriver` is printed. + The returned deriver is not guaranteed to exist in the local store, for + example when *paths* were substituted from a binary cache. + Use `--valid-derivers` instead to obtain valid paths only. - - `--hash`\ - Prints the SHA-256 hash of the contents of the store paths *paths* - (that is, the hash of the output of `nix-store --dump` on the given - paths). Since the hash is stored in the Nix database, this is a fast - operation. + [deriver]: @docroot@/glossary.md#gloss-deriver - - `--size`\ - Prints the size in bytes of the contents of the store paths *paths* - — to be precise, the size of the output of `nix-store --dump` on - the given paths. Note that the actual disk space required by the - store paths may be higher, especially on filesystems with large - cluster sizes. +- `--valid-derivers` - - `--roots`\ - Prints the garbage collector roots that point, directly or - indirectly, at the store paths *paths*. + Prints a set of derivation files (`.drv`) which are supposed produce + said paths when realized. Might print nothing, for example for source paths + or paths subsituted from a binary cache. + +- `--graph` + + Prints the references graph of the store paths *paths* in the format + of the `dot` tool of AT\&T's [Graphviz + package](http://www.graphviz.org/). This can be used to visualise + dependency graphs. To obtain a build-time dependency graph, apply + this to a store derivation. To obtain a runtime dependency graph, + apply it to an output path. + +- `--tree` + + Prints the references graph of the store paths *paths* as a nested + ASCII tree. References are ordered by descending closure size; this + tends to flatten the tree, making it more readable. The query only + recurses into a store path when it is first encountered; this + prevents a blowup of the tree representation of the graph. + +- `--graphml` + + Prints the references graph of the store paths *paths* in the + [GraphML](http://graphml.graphdrawing.org/) file format. This can be + used to visualise dependency graphs. To obtain a build-time + dependency graph, apply this to a [store derivation]. To obtain a + runtime dependency graph, apply it to an output path. + +- `--binding` *name* / `-b` *name* + + Prints the value of the attribute *name* (i.e., environment + variable) of the [store derivation]s *paths*. It is an error for a + derivation to not have the specified attribute. + +- `--hash` + + Prints the SHA-256 hash of the contents of the store paths *paths* + (that is, the hash of the output of `nix-store --dump` on the given + paths). Since the hash is stored in the Nix database, this is a fast + operation. + +- `--size` + + Prints the size in bytes of the contents of the store paths *paths* + — to be precise, the size of the output of `nix-store --dump` on + the given paths. Note that the actual disk space required by the + store paths may be higher, especially on filesystems with large + cluster sizes. + +- `--roots` + + Prints the garbage collector roots that point, directly or + indirectly, at the store paths *paths*. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index 5428d57fa..a899758df 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -25,14 +25,14 @@ Each of *paths* is processed as follows: If no substitutes are available and no store derivation is given, realisation fails. -[store paths]: @docroot@/glossary.md#gloss-store-path +[store paths]: @docroot@/store/store-path.md [valid]: @docroot@/glossary.md#gloss-validity [store derivation]: @docroot@/glossary.md#gloss-store-derivation [output paths]: @docroot@/glossary.md#gloss-output-path -[store objects]: @docroot@/glossary.md#gloss-store-object +[store objects]: @docroot@/store/store-object.md [closure]: @docroot@/glossary.md#gloss-closure [substituters]: @docroot@/command-ref/conf-file.md#conf-substituters -[content-addressed derivations]: @docroot@/contributing/experimental-features.md#xp-feature-ca-derivations +[content-addressed derivations]: @docroot@/development/experimental-features.md#xp-feature-ca-derivations [Nix database]: @docroot@/glossary.md#gloss-nix-database The resulting paths are printed on standard output. @@ -42,23 +42,26 @@ For non-derivation arguments, the argument itself is printed. # Options - - `--dry-run`\ - Print on standard error a description of what packages would be - built or downloaded, without actually performing the operation. +- `--dry-run` - - `--ignore-unknown`\ - If a non-derivation path does not have a substitute, then silently - ignore it. + Print on standard error a description of what packages would be + built or downloaded, without actually performing the operation. - - `--check`\ - This option allows you to check whether a derivation is - deterministic. It rebuilds the specified derivation and checks - whether the result is bitwise-identical with the existing outputs, - printing an error if that’s not the case. The outputs of the - specified derivation must already exist. When used with `-K`, if an - output path is not identical to the corresponding output from the - previous build, the new output path is left in - `/nix/store/name.check.` +- `--ignore-unknown` + + If a non-derivation path does not have a substitute, then silently + ignore it. + +- `--check` + + This option allows you to check whether a derivation is + deterministic. It rebuilds the specified derivation and checks + whether the result is bitwise-identical with the existing outputs, + printing an error if that’s not the case. The outputs of the + specified derivation must already exist. When used with `-K`, if an + output path is not identical to the corresponding output from the + previous build, the new output path is left in + `/nix/store/name.check.` {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/restore.md b/doc/manual/src/command-ref/nix-store/restore.md index fcba43df4..2d0aa3127 100644 --- a/doc/manual/src/command-ref/nix-store/restore.md +++ b/doc/manual/src/command-ref/nix-store/restore.md @@ -8,9 +8,11 @@ ## Description -The operation `--restore` unpacks a NAR archive to *path*, which must +The operation `--restore` unpacks a [Nix Archive (NAR)][Nix Archive] to *path*, which must not already exist. The archive is read from standard input. +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive + {{#include ./opt-common.md}} {{#include ../opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/serve.md b/doc/manual/src/command-ref/nix-store/serve.md index 0f90f65ae..9a4cf5216 100644 --- a/doc/manual/src/command-ref/nix-store/serve.md +++ b/doc/manual/src/command-ref/nix-store/serve.md @@ -14,10 +14,11 @@ access to a restricted ssh user. The following flags are available: - - `--write`\ - Allow the connected client to request the realization of - derivations. In effect, this can be used to make the host act as a - remote builder. +- `--write` + + Allow the connected client to request the realization of + derivations. In effect, this can be used to make the host act as a + remote builder. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/verify.md b/doc/manual/src/command-ref/nix-store/verify.md index 2695b3361..40c9180db 100644 --- a/doc/manual/src/command-ref/nix-store/verify.md +++ b/doc/manual/src/command-ref/nix-store/verify.md @@ -16,18 +16,20 @@ being modified by non-Nix tools, or of bugs in Nix itself. This operation has the following options: - - `--check-contents`\ - Checks that the contents of every valid store path has not been - altered by computing a SHA-256 hash of the contents and comparing it - with the hash stored in the Nix database at build time. Paths that - have been modified are printed out. For large stores, - `--check-contents` is obviously quite slow. +- `--check-contents` - - `--repair`\ - If any valid path is missing from the store, or (if - `--check-contents` is given) the contents of a valid path has been - modified, then try to repair the path by redownloading it. See - `nix-store --repair-path` for details. + Checks that the contents of every valid store path has not been + altered by computing a SHA-256 hash of the contents and comparing it + with the hash stored in the Nix database at build time. Paths that + have been modified are printed out. For large stores, + `--check-contents` is obviously quite slow. + +- `--repair` + + If any valid path is missing from the store, or (if + `--check-contents` is given) the contents of a valid path has been + modified, then try to repair the path by redownloading it. See + `nix-store --repair-path` for details. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index 114b292f9..69a700207 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -37,7 +37,7 @@ Most Nix commands accept the following command-line options: Print even more informational messages. - `4` “Debug†- + Print debug information. - `5` “Vomit†@@ -143,7 +143,7 @@ Most Nix commands accept the following command-line options: This option is accepted by `nix-env`, `nix-instantiate`, `nix-shell` and `nix-build`. When evaluating Nix expressions, the expression evaluator will automatically try to call functions that it encounters. - It can automatically call functions for which every argument has a [default value](@docroot@/language/constructs.md#functions) (e.g., `{ argName ? defaultValue }: ...`). + It can automatically call functions for which every argument has a [default value](@docroot@/language/syntax.md#functions) (e.g., `{ argName ? defaultValue }: ...`). With `--arg`, you can also call functions that have arguments without a default value (or override a default value). That is, if the evaluator encounters a function with an argument named *name*, it will call it with value *value*. @@ -187,11 +187,12 @@ Most Nix commands accept the following command-line options: For `nix-shell`, this option is commonly used to give you a shell in which you can build the packages returned by the expression. If you want to get a shell which contain the *built* packages ready for use, give your expression to the `nix-shell --packages ` convenience flag instead. -- [`-I`](#opt-I) *path* +- [`-I` / `--include`](#opt-I) *path* - Add an entry to the [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path). + Add an entry to the list of search paths used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). This option may be given multiple times. - Paths added through `-I` take precedence over [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH). + + Paths added through `-I` take precedence over the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path) and the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH). - [`--option`](#opt-option) *name* *value* diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/development/building.md similarity index 67% rename from doc/manual/src/contributing/hacking.md rename to doc/manual/src/development/building.md index 916ec3077..5a5fb3368 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/development/building.md @@ -1,24 +1,67 @@ -# Hacking +# Building Nix -This section provides some notes on how to hack on Nix. To get the -latest version of Nix from GitHub: +This section provides some notes on how to start hacking on Nix. +To get the latest version of Nix from GitHub: ```console $ git clone https://github.com/NixOS/nix.git $ cd nix ``` -The following instructions assume you already have some version of Nix installed locally, so that you can use it to set up the development environment. If you don't have it installed, follow the [installation instructions]. +> **Note** +> +> The following instructions assume you already have some version of Nix installed locally, so that you can use it to set up the development environment. +> If you don't have it installed, follow the [installation instructions](../installation/index.md). -[installation instructions]: ../installation/index.md + +To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: + +```console +$ nix-shell +``` + +To get a shell with one of the other [supported compilation environments](#compilation-environments): + +```console +$ nix-shell --attr devShells.x86_64-linux.native-clangStdenvPackages +``` + +> **Note** +> +> You can use `native-ccacheStdenvPackages` to drastically improve rebuild time. +> By default, [ccache](https://ccache.dev) keeps artifacts in `~/.cache/ccache/`. + +To build Nix itself in this shell: + +```console +[nix-shell]$ autoreconfPhase +[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out +[nix-shell]$ make -j $NIX_BUILD_CORES +``` + +To install it in `$(pwd)/outputs` and test it: + +```console +[nix-shell]$ make install +[nix-shell]$ make installcheck -j $NIX_BUILD_CORES +[nix-shell]$ ./outputs/out/bin/nix --version +nix (Nix) 2.12 +``` + +To build a release version of Nix for the current operating system and CPU architecture: + +```console +$ nix-build +``` + +You can also build Nix for one of the [supported platforms](#platforms). ## Building Nix with flakes This section assumes you are using Nix with the [`flakes`] and [`nix-command`] experimental features enabled. -See the [Building Nix](#building-nix) section for equivalent instructions using stable Nix interfaces. -[`flakes`]: @docroot@/contributing/experimental-features.md#xp-feature-flakes -[`nix-command`]: @docroot@/contributing/experimental-features.md#xp-nix-command +[`flakes`]: @docroot@/development/experimental-features.md#xp-feature-flakes +[`nix-command`]: @docroot@/development/experimental-features.md#xp-nix-command To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: @@ -67,50 +110,6 @@ $ nix build You can also build Nix for one of the [supported platforms](#platforms). -## Building Nix - -To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: - -```console -$ nix-shell -``` - -To get a shell with one of the other [supported compilation environments](#compilation-environments): - -```console -$ nix-shell --attr devShells.x86_64-linux.native-clangStdenvPackages -``` - -> **Note** -> -> You can use `native-ccacheStdenvPackages` to drastically improve rebuild time. -> By default, [ccache](https://ccache.dev) keeps artifacts in `~/.cache/ccache/`. - -To build Nix itself in this shell: - -```console -[nix-shell]$ autoreconfPhase -[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out -[nix-shell]$ make -j $NIX_BUILD_CORES -``` - -To install it in `$(pwd)/outputs` and test it: - -```console -[nix-shell]$ make install -[nix-shell]$ make installcheck -j $NIX_BUILD_CORES -[nix-shell]$ ./outputs/out/bin/nix --version -nix (Nix) 2.12 -``` - -To build a release version of Nix for the current operating system and CPU architecture: - -```console -$ nix-build -``` - -You can also build Nix for one of the [supported platforms](#platforms). - ## Makefile variables You may need `profiledir=$out/etc/profile.d` and `sysconfdir=$out/etc` to run `make install`. @@ -122,7 +121,6 @@ Run `make` with [`-e` / `--environment-overrides`](https://www.gnu.org/software/ The docs can take a while to build, so you may want to disable this for local development. - `ENABLE_FUNCTIONAL_TESTS=yes` to enable building the functional tests. -- `ENABLE_UNIT_TESTS=yes` to enable building the unit tests. - `OPTIMIZE=1` to enable optimizations. - `libraries=libutil programs=` to only build a specific library. @@ -144,6 +142,7 @@ Nix can be built for various platforms, as specified in [`flake.nix`]: - `aarch64-darwin` - `armv6l-linux` - `armv7l-linux` +- `riscv64-linux` In order to build Nix for a different platform than the one you're currently on, you need a way for your current Nix installation to build code for that @@ -166,7 +165,10 @@ or for Nix with the [`flakes`] and [`nix-command`] experimental features enabled $ nix build .#packages.aarch64-linux.default ``` -Cross-compiled builds are available for ARMv6 (`armv6l-linux`) and ARMv7 (`armv7l-linux`). +Cross-compiled builds are available for: +- `armv6l-linux` +- `armv7l-linux` +- `riscv64-linux` Add more [system types](#system-type) to `crossSystems` in `flake.nix` to bootstrap Nix on unsupported platforms. ### Building for multiple platforms at once @@ -196,7 +198,7 @@ In order to facilitate this, Nix has some support for being built out of tree ## System type -Nix uses a string with he following format to identify the *system type* or *platform* it runs on: +Nix uses a string with the following format to identify the *system type* or *platform* it runs on: ``` -[-] @@ -258,10 +260,10 @@ See [supported compilation environments](#compilation-environments) and instruct To use the LSP with your editor, you first need to [set up `clangd`](https://clangd.llvm.org/installation#project-setup) by running: ```console -make clean && bear -- make -j$NIX_BUILD_CORES default check install +make compile_commands.json ``` -Configure your editor to use the `clangd` from the shell, either by running it inside the development shell, or by using [nix-direnv](https://github.com/nix-community/nix-direnv) and [the appropriate editor plugin](https://github.com/direnv/direnv/wiki#editor-integration). +Configure your editor to use the `clangd` from the `.#native-clangStdenvPackages` shell. You can do that either by running it inside the development shell, or by using [nix-direnv](https://github.com/nix-community/nix-direnv) and [the appropriate editor plugin](https://github.com/direnv/direnv/wiki#editor-integration). > **Note** > @@ -269,80 +271,25 @@ Configure your editor to use the `clangd` from the shell, either by running it i > Some other editors (e.g. Emacs, Vim) need a plugin to support LSP servers in general (e.g. [lsp-mode](https://github.com/emacs-lsp/lsp-mode) for Emacs and [vim-lsp](https://github.com/prabirshrestha/vim-lsp) for vim). > Editor-specific setup is typically opinionated, so we will not cover it here in more detail. -## Add a release note +## Formatting and pre-commit hooks -`doc/manual/rl-next` contains release notes entries for all unreleased changes. +You may run the formatters as a one-off using: -User-visible changes should come with a release note. - -### Add an entry - -Here's what a complete entry looks like. The file name is not incorporated in the document. - -``` ---- -synopsis: Basically a title -issues: 1234 -prs: 1238 ---- - -Here's one or more paragraphs that describe the change. - -- It's markdown -- Add references to the manual using @docroot@ +```console +make format ``` -Significant changes should add the following header, which moves them to the top. +If you'd like to run the formatters before every commit, install the hooks: ``` -significance: significant +pre-commit-hooks-install ``` - -See also the [format documentation](https://github.com/haskell/cabal/blob/master/CONTRIBUTING.md#changelog). +This installs [pre-commit](https://pre-commit.com) using [cachix/git-hooks.nix](https://github.com/cachix/git-hooks.nix). -### Build process +When making a commit, pay attention to the console output. +If it fails, run `git add --patch` to approve the suggestions _and commit again_. -Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`. - -## Branches - -- [`master`](https://github.com/NixOS/nix/commits/master) - - The main development branch. All changes are approved and merged here. - When developing a change, create a branch based on the latest `master`. - - Maintainers try to [keep it in a release-worthy state](#reverting). - -- [`maintenance-*.*`](https://github.com/NixOS/nix/branches/all?query=maintenance) - - These branches are the subject of backports only, and are - also [kept](#reverting) in a release-worthy state. - - See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) - -- [`latest-release`](https://github.com/NixOS/nix/tree/latest-release) - - The latest patch release of the latest minor version. - - See [`maintainers/release-process.md`](https://github.com/NixOS/nix/blob/master/maintainers/release-process.md) - -- [`backport-*-to-*`](https://github.com/NixOS/nix/branches/all?query=backport) - - Generally branches created by the backport action. - - See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) - -- [_other_](https://github.com/NixOS/nix/branches/all) - - Branches that do not conform to the above patterns should be feature branches. - -## Reverting - -If a change turns out to be merged by mistake, or contain a regression, it may be reverted. -A revert is not a rejection of the contribution, but merely part of an effective development process. -It makes sure that development keeps running smoothly, with minimal uncertainty, and less overhead. -If maintainers have to worry too much about avoiding reverts, they would not be able to merge as much. -By embracing reverts as a good part of the development process, everyone wins. - -However, taking a step back may be frustrating, so maintainers will be extra supportive on the next try. +To refresh pre-commit hook's config file, do the following: +1. Exit the development shell and start it again by running `nix develop`. +2. If you also use the pre-commit hook, also run `pre-commit-hooks-install` again. diff --git a/doc/manual/src/contributing/cli-guideline.md b/doc/manual/src/development/cli-guideline.md similarity index 88% rename from doc/manual/src/contributing/cli-guideline.md rename to doc/manual/src/development/cli-guideline.md index e90d6de8d..23df844ec 100644 --- a/doc/manual/src/contributing/cli-guideline.md +++ b/doc/manual/src/development/cli-guideline.md @@ -389,88 +389,6 @@ colors, no emojis and using ASCII instead of Unicode symbols). The same should happen when TTY is not detected on STDERR. We should not display progress / status section, but only print warnings and errors. -## Returning future proof JSON - -The schema of JSON output should allow for backwards compatible extension. This section explains how to achieve this. - -Two definitions are helpful here, because while JSON only defines one "key-value" -object type, we use it to cover two use cases: - - - **dictionary**: a map from names to value that all have the same type. In - C++ this would be a `std::map` with string keys. - - **record**: a fixed set of attributes each with their own type. In C++, this - would be represented by a `struct`. - -It is best not to mix these use cases, as that may lead to incompatibilities when the schema changes. For example, adding a record field to a dictionary breaks consumers that assume all JSON object fields to have the same meaning and type. - -This leads to the following guidelines: - - - The top-level (root) value must be a record. - - Otherwise, one can not change the structure of a command's output. - - - The value of a dictionary item must be a record. - - Otherwise, the item type can not be extended. - - - List items should be records. - - Otherwise, one can not change the structure of the list items. - - If the order of the items does not matter, and each item has a unique key that is a string, consider representing the list as a dictionary instead. If the order of the items needs to be preserved, return a list of records. - - - Streaming JSON should return records. - - An example of a streaming JSON format is [JSON lines](https://jsonlines.org/), where each line represents a JSON value. These JSON values can be considered top-level values or list items, and they must be records. - -### Examples - - -This is bad, because all keys must be assumed to be store types: - -```json -{ - "local": { ... }, - "remote": { ... }, - "http": { ... } -} -``` - -This is good, because the it is extensible at the root, and is somewhat self-documenting: - -```json -{ - "storeTypes": { "local": { ... }, ... }, - "pluginSupport": true -} -``` - -While the dictionary of store types seems like a very complete response at first, a use case may arise that warrants returning additional information. -For example, the presence of plugin support may be crucial information for a client to proceed when their desired store type is missing. - - - -The following representation is bad because it is not extensible: - -```json -{ "outputs": [ "out" "bin" ] } -``` - -However, simply converting everything to records is not enough, because the order of outputs must be preserved: - -```json -{ "outputs": { "bin": {}, "out": {} } } -``` - -The first item is the default output. Deriving this information from the outputs ordering is not great, but this is how Nix currently happens to work. -While it is possible for a JSON parser to preserve the order of fields, we can not rely on this capability to be present in all JSON libraries. - -This representation is extensible and preserves the ordering: - -```json -{ "outputs": [ { "outputName": "out" }, { "outputName": "bin" } ] } -``` - ## Dialog with the user CLIs don't always make it clear when an action has taken place. For every diff --git a/doc/manual/src/development/contributing.md b/doc/manual/src/development/contributing.md new file mode 100644 index 000000000..7de7489dc --- /dev/null +++ b/doc/manual/src/development/contributing.md @@ -0,0 +1,79 @@ +# Contributing + +## Add a release note + +`doc/manual/rl-next` contains release notes entries for all unreleased changes. + +User-visible changes should come with a release note. + +### Add an entry + +Here's what a complete entry looks like. The file name is not incorporated in the document. + +``` +--- +synopsis: Basically a title +issues: 1234 +prs: 1238 +--- + +Here's one or more paragraphs that describe the change. + +- It's markdown +- Add references to the manual using @docroot@ +``` + +Significant changes should add the following header, which moves them to the top. + +``` +significance: significant +``` + + +See also the [format documentation](https://github.com/haskell/cabal/blob/master/CONTRIBUTING.md#changelog). + +### Build process + +Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`. + +## Branches + +- [`master`](https://github.com/NixOS/nix/commits/master) + + The main development branch. All changes are approved and merged here. + When developing a change, create a branch based on the latest `master`. + + Maintainers try to [keep it in a release-worthy state](#reverting). + +- [`maintenance-*.*`](https://github.com/NixOS/nix/branches/all?query=maintenance) + + These branches are the subject of backports only, and are + also [kept](#reverting) in a release-worthy state. + + See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) + +- [`latest-release`](https://github.com/NixOS/nix/tree/latest-release) + + The latest patch release of the latest minor version. + + See [`maintainers/release-process.md`](https://github.com/NixOS/nix/blob/master/maintainers/release-process.md) + +- [`backport-*-to-*`](https://github.com/NixOS/nix/branches/all?query=backport) + + Generally branches created by the backport action. + + See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) + +- [_other_](https://github.com/NixOS/nix/branches/all) + + Branches that do not conform to the above patterns should be feature branches. + +## Reverting + +If a change turns out to be merged by mistake, or contain a regression, it may be reverted. +A revert is not a rejection of the contribution, but merely part of an effective development process. +It makes sure that development keeps running smoothly, with minimal uncertainty, and less overhead. +If maintainers have to worry too much about avoiding reverts, they would not be able to merge as much. +By embracing reverts as a good part of the development process, everyone wins. + +However, taking a step back may be frustrating, so maintainers will be extra supportive on the next try. diff --git a/doc/manual/src/contributing/cxx.md b/doc/manual/src/development/cxx.md similarity index 100% rename from doc/manual/src/contributing/cxx.md rename to doc/manual/src/development/cxx.md diff --git a/doc/manual/src/contributing/documentation.md b/doc/manual/src/development/documentation.md similarity index 89% rename from doc/manual/src/contributing/documentation.md rename to doc/manual/src/development/documentation.md index 1dddb207c..63f574ab7 100644 --- a/doc/manual/src/contributing/documentation.md +++ b/doc/manual/src/development/documentation.md @@ -24,14 +24,12 @@ nix build .#^doc and open `./result-doc/share/doc/nix/manual/index.html`. -To build the manual incrementally, [enter the development shell](./hacking.md) and run: +To build the manual incrementally, [enter the development shell](./building.md) and run: ```console -make manual-html -j $NIX_BUILD_CORES +make manual-html-open -j $NIX_BUILD_CORES ``` -and open `./outputs/out/share/doc/nix/manual/language/index.html`. - In order to reflect changes to the [Makefile for the manual], clear all generated files before re-building: [Makefile for the manual]: https://github.com/NixOS/nix/blob/master/doc/manual/local.mk @@ -149,7 +147,7 @@ Please observe these guidelines to ease reviews: ``` A [store object] contains a [file system object] and [references] to other store objects. - [store object]: @docroot@/glossary.md#gloss-store-object + [store object]: @docroot@/store/store-object.md [file system object]: @docroot@/architecture/file-system-object.md [references]: @docroot@/glossary.md#gloss-reference ``` @@ -198,13 +196,35 @@ You can also build and view it yourself: [Doxygen API documentation]: https://hydra.nixos.org/job/nix/master/internal-api-docs/latest/download-by-type/doc/internal-api-docs ```console -# nix build .#hydraJobs.internal-api-docs -# xdg-open ./result/share/doc/nix/internal-api/html/index.html +$ nix build .#hydraJobs.internal-api-docs +$ xdg-open ./result/share/doc/nix/internal-api/html/index.html +``` + +or inside `nix-shell` or `nix develop`: + +```console +$ mesonConfigurePhase +$ ninja src/internal-api-docs/html +$ xdg-open src/internal-api-docs/html/index.html +``` + +## C API documentation + +Note that the C API is not yet stable. +[C API documentation] is available online. +You can also build and view it yourself: + +[C API documentation]: https://hydra.nixos.org/job/nix/master/external-api-docs/latest/download-by-type/doc/external-api-docs + +```console +$ nix build .#hydraJobs.external-api-docs +$ xdg-open ./result/share/doc/nix/external-api/html/index.html ``` or inside `nix-shell` or `nix develop`: ``` -# make internal-api-html -# xdg-open ./outputs/doc/share/doc/nix/internal-api/html/index.html +$ mesonConfigurePhase +$ ninja src/external-api-docs/html +$ xdg-open src/external-api-docs/html/index.html ``` diff --git a/doc/manual/src/contributing/experimental-features.md b/doc/manual/src/development/experimental-features.md similarity index 100% rename from doc/manual/src/contributing/experimental-features.md rename to doc/manual/src/development/experimental-features.md diff --git a/doc/manual/src/contributing/index.md b/doc/manual/src/development/index.md similarity index 77% rename from doc/manual/src/contributing/index.md rename to doc/manual/src/development/index.md index 4d55c17a4..6403c3e66 100644 --- a/doc/manual/src/contributing/index.md +++ b/doc/manual/src/development/index.md @@ -5,4 +5,4 @@ Check the [contributing guide](https://github.com/NixOS/nix/blob/master/CONTRIBU This chapter is a collection of guides for making changes to the code and documentation. -If you're not sure where to start, try to [compile Nix from source](./hacking.md) and consider [making improvements to documentation](./documentation.md). +If you're not sure where to start, try to [compile Nix from source](./building.md) and consider [making improvements to documentation](./documentation.md). diff --git a/doc/manual/src/development/json-guideline.md b/doc/manual/src/development/json-guideline.md new file mode 100644 index 000000000..b4bc92af9 --- /dev/null +++ b/doc/manual/src/development/json-guideline.md @@ -0,0 +1,128 @@ +# JSON guideline + +Nix consumes and produces JSON in a variety of contexts. +These guidelines ensure consistent practices for all our JSON interfaces, for ease of use, and so that experience in one part carries over to another. + +## Extensibility + +The schema of JSON input and output should allow for backwards compatible extension. +This section explains how to achieve this. + +Two definitions are helpful here, because while JSON only defines one "key-value" object type, we use it to cover two use cases: + + - **dictionary**: a map from names to value that all have the same type. + In C++ this would be a `std::map` with string keys. + + - **record**: a fixed set of attributes each with their own type. + In C++, this would be represented by a `struct`. + +It is best not to mix these use cases, as that may lead to incompatibilities when the schema changes. +For example, adding a record field to a dictionary breaks consumers that assume all JSON object fields to have the same meaning and type, and dictionary items with a colliding name can not be represented anymore. + +This leads to the following guidelines: + + - The top-level (root) value must be a record. + + Otherwise, one can not change the structure of a command's output. + + - The value of a dictionary item must be a record. + + Otherwise, the item type can not be extended. + + - List items should be records. + + Otherwise, one can not change the structure of the list items. + + If the order of the items does not matter, and each item has a unique key that is a string, consider representing the list as a dictionary instead. + If the order of the items needs to be preserved, return a list of records. + + - Streaming JSON should return records. + + An example of a streaming JSON format is [JSON lines](https://jsonlines.org/), where each line represents a JSON value. + These JSON values can be considered top-level values or list items, and they must be records. + +### Examples + +This is bad, because all keys must be assumed to be store types: + +```json +{ + "local": { ... }, + "remote": { ... }, + "http": { ... } +} +``` + +This is good, because the it is extensible at the root, and is somewhat self-documenting: + +```json +{ + "storeTypes": { "local": { ... }, ... }, + "pluginSupport": true +} +``` + +While the dictionary of store types seems like a very complete response at first, a use case may arise that warrants returning additional information. +For example, the presence of plugin support may be crucial information for a client to proceed when their desired store type is missing. + + + +The following representation is bad because it is not extensible: + +```json +{ "outputs": [ "out" "bin" ] } +``` + +However, simply converting everything to records is not enough, because the order of outputs must be preserved: + +```json +{ "outputs": { "bin": {}, "out": {} } } +``` + +The first item is the default output. Deriving this information from the outputs ordering is not great, but this is how Nix currently happens to work. +While it is possible for a JSON parser to preserve the order of fields, we can not rely on this capability to be present in all JSON libraries. + +This representation is extensible and preserves the ordering: + +```json +{ "outputs": [ { "outputName": "out" }, { "outputName": "bin" } ] } +``` + +## Self-describing values + +As described in the previous section, it's crucial that schemas can be extended with with new fields without breaking compatibility. +However, that should *not* mean we use the presence/absence of fields to indicate optional information *within* a version of the schema. +Instead, always include the field, and use `null` to indicate the "nothing" case. + +### Examples + +Here are two JSON objects: + +```json +{ + "foo": {} +} +``` +```json +{ + "foo": {}, + "bar": {} +} +``` + +Since they differ in which fields they contain, they should *not* both be valid values of the same schema. +At most, they can match two different schemas where the second (with `foo` and `bar`) is considered a newer version of the first (with just `foo`). +Within each version, all fields are mandatory (always `foo`, and always `foo` and `bar`). +Only *between* each version, `bar` gets added as a new mandatory field. + +Here are another two JSON objects: + +```json +{ "foo": null } +``` +```json +{ "foo": { "bar": 1 } } +``` + +Since they both contain a `foo` field, they could be valid values of the same schema. +The schema would have `foo` has an optional field, which is either `null` or an object where `bar` is an integer. diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/development/testing.md similarity index 81% rename from doc/manual/src/contributing/testing.md rename to doc/manual/src/development/testing.md index 31c39c16c..3949164d5 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/development/testing.md @@ -59,15 +59,15 @@ The unit tests are defined using the [googletest] and [rapidcheck] frameworks. > … > ``` -The tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `tests/unit/${library_name_without-nix}`. -Given a interface (header) and implementation pair in the original library, say, `src/libexpr/value/context.{hh,cc}`, we write tests for it in `tests/unit/libexpr/tests/value/context.cc`, and (possibly) declare/define additional interfaces for testing purposes in `tests/unit/libexpr-support/tests/value/context.{hh,cc}`. +The tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `src/${library_name_without-nix}-test`. +Given an interface (header) and implementation pair in the original library, say, `src/libexpr/value/context.{hh,cc}`, we write tests for it in `src/nix-expr-tests/value/context.cc`, and (possibly) declare/define additional interfaces for testing purposes in `src/nix-expr-test-support/tests/value/context.{hh,cc}`. Data for unit tests is stored in a `data` subdir of the directory for each unit test executable. -For example, `libnixstore` code is in `src/libstore`, and its test data is in `tests/unit/libstore/data`. -The path to the `tests/unit/data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`. +For example, `libnixstore` code is in `src/libstore`, and its test data is in `src/nix-store-tests/data`. +The path to the `src/${library_name_without-nix}-test/data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`. Note that each executable only gets the data for its tests. -The unit test libraries are in `tests/unit/${library_name_without-nix}-lib`. +The unit test libraries are in `src/${library_name_without-nix}-test-support`. All headers are in a `tests` subdirectory so they are included with `#include "tests/"`. The use of all these separate directories for the unit tests might seem inconvenient, as for example the tests are not "right next to" the part of the code they are testing. @@ -76,8 +76,25 @@ there is no risk of any build-system wildcards for the library accidentally pick ### Running tests -You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. -Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option, or the `GTEST_FILTER` environment variable, e.g. `GTEST_FILTER='ErrorTraceTest.*' make check`. +You can run the whole testsuite with `meson test` from the Meson build directory, or the tests for a specific component with `meson test nix-store-tests`. +A environment variables that Google Test accepts are also worth knowing: + +1. [`GTEST_FILTER`](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) + + This is used for finer-grained filtering of which tests to run. + + +2. [`GTEST_BRIEF`](https://google.github.io/googletest/advanced.html#suppressing-test-passes) + + This is used to avoid logging passing tests. + +Putting the two together, one might run + +```bash +GTEST_BRIEF=1 GTEST_FILTER='ErrorTraceTest.*' meson test nix-expr-tests -v +``` + +for short but comprensive output. ### Characterisation testing { #characaterisation-testing-unit } @@ -86,7 +103,7 @@ See [functional characterisation testing](#characterisation-testing-functional) Like with the functional characterisation, `_NIX_TEST_ACCEPT=1` is also used. For example: ```shell-session -$ _NIX_TEST_ACCEPT=1 make libstore-tests_RUN +$ _NIX_TEST_ACCEPT=1 meson test nix-store-tests -v ... [ SKIPPED ] WorkerProtoTest.string_read [ SKIPPED ] WorkerProtoTest.string_write @@ -114,6 +131,8 @@ On other platforms they wouldn't be run at all. The functional tests reside under the `tests/functional` directory and are listed in `tests/functional/local.mk`. Each test is a bash script. +Functional tests are run during `installCheck` in the `nix` package build, as well as separately from the build, in VM tests. + ### Running the whole test suite The whole test suite can be run with: @@ -162,14 +181,14 @@ ran test tests/functional/${testName}.sh... [PASS] or without `make`: ```shell-session -$ ./mk/run-test.sh tests/functional/${testName}.sh tests/functional/init.sh +$ ./mk/run-test.sh tests/functional/${testName}.sh ran test tests/functional/${testName}.sh... [PASS] ``` To see the complete output, one can also run: ```shell-session -$ ./mk/debug-test.sh tests/functional/${testName}.sh tests/functional/init.sh +$ ./mk/debug-test.sh tests/functional/${testName}.sh +(${testName}.sh:1) foo output from foo +(${testName}.sh:2) bar @@ -204,7 +223,7 @@ edit it like so: Then, running the test with `./mk/debug-test.sh` will drop you into GDB once the script reaches that point: ```shell-session -$ ./mk/debug-test.sh tests/functional/${testName}.sh tests/functional/init.sh +$ ./mk/debug-test.sh tests/functional/${testName}.sh ... + gdb blash blub GNU gdb (GDB) 12.1 @@ -252,13 +271,30 @@ Regressions are caught, and improvements always show up in code review. To ensure that characterisation testing doesn't make it harder to intentionally change these interfaces, there always must be an easy way to regenerate the expected output, as we do with `_NIX_TEST_ACCEPT=1`. +### Running functional tests on NixOS + +We run the functional tests not just in the build, but also in VM tests. +This helps us ensure that Nix works correctly on NixOS, and environments that have similar characteristics that are hard to reproduce in a build environment. + +The recommended way to run these tests during development is: + +```shell +nix build .#hydraJobs.tests.functional_user.quickBuild +``` + +The `quickBuild` attribute configures the test to use a `nix` package that's built without integration tests, so that you can iterate on the tests without performing recompilations due to the changed sources for `installCheck`. + +Generally, this build is sufficient, but in nightly or CI we also test the attributes `functional_root` and `functional_trusted`, in which the test suite is run with different levels of authorization. + ## Integration tests The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute. These tests include everything that needs to interact with external services or run Nix in a non-trivial distributed setup. Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on ). -You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}` +You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}`. + +If you are testing a build of `nix` that you haven't compiled yet, you may iterate faster by appending the `quickBuild` attribute: `nix build .#hydraJobs.tests.{testName}.quickBuild`. ## Installer tests diff --git a/doc/manual/src/favicon.png b/doc/manual/src/favicon.png new file mode 100644 index 000000000..1ed2b5fe0 Binary files /dev/null and b/doc/manual/src/favicon.png differ diff --git a/doc/manual/src/favicon.svg b/doc/manual/src/favicon.svg new file mode 100644 index 000000000..1d2a6e835 --- /dev/null +++ b/doc/manual/src/favicon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index a71b2e2b3..877c4668b 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -1,5 +1,24 @@ # Glossary +- [content address]{#gloss-content-address} + + A + [*content address*](https://en.wikipedia.org/wiki/Content-addressable_storage) + is a secure way to reference immutable data. + The reference is calculated directly from the content of the data being referenced, which means the reference is + [*tamper proof*](https://en.wikipedia.org/wiki/Tamperproofing) + --- variations of the data should always calculate to distinct content addresses. + + For how Nix uses content addresses, see: + + - [Content-Addressing File System Objects](@docroot@/store/file-system-object/content-address.md) + - [Content-Addressing Store Objects](@docroot@/store/store-object/content-address.md) + - [content-addressed derivation](#gloss-content-addressed-derivation) + + Software Heritage's writing on [*Intrinsic and Extrinsic identifiers*](https://www.softwareheritage.org/2020/07/09/intrinsic-vs-extrinsic-identifiers) is also a good introduction to the value of content-addressing over other referencing schemes. + + Besides content addressing, the Nix store also uses [input addressing](#gloss-input-addressed-store-object). + - [derivation]{#gloss-derivation} A description of a build task. The result of a derivation is a @@ -52,10 +71,9 @@ [`__contentAddressed`](./language/advanced-attributes.md#adv-attr-__contentAddressed) attribute set to `true`. -- [fixed-output derivation]{#gloss-fixed-output-derivation} +- [fixed-output derivation]{#gloss-fixed-output-derivation} (FOD) - A derivation which includes the - [`outputHash`](./language/advanced-attributes.md#adv-attr-outputHash) attribute. + A [derivation] where a cryptographic hash of the [output] is determined in advance using the [`outputHash`](./language/advanced-attributes.md#adv-attr-outputHash) attribute, and where the [`builder`](@docroot@/language/derivations.md#attr-builder) executable has access to the network. - [store]{#gloss-store} @@ -86,7 +104,7 @@ [store path]: #gloss-store-path -- [file system object]{#gloss-store-object} +- [file system object]{#gloss-file-system-object} The Nix data model for representing simplified file system data. @@ -118,9 +136,12 @@ - [content-addressed store object]{#gloss-content-addressed-store-object} - A [store object] whose [store path] is determined by its contents. + A [store object] which is [content-addressed](#gloss-content-address), + i.e. whose [store path] is determined by its contents. This includes derivations, the outputs of [content-addressed derivations](#gloss-content-addressed-derivation), and the outputs of [fixed-output derivations](#gloss-fixed-output-derivation). + See [Content-Addressing Store Objects](@docroot@/store/store-object/content-address.md) for details. + - [substitute]{#gloss-substitute} A substitute is a command invocation stored in the [Nix database] that @@ -147,7 +168,7 @@ - [impure derivation]{#gloss-impure-derivation} - [An experimental feature](#@docroot@/contributing/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, + [An experimental feature](#@docroot@/development/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, so that they are always rebuilt, and their outputs not reused by subsequent calls to realise them. - [Nix database]{#gloss-nix-database} @@ -215,6 +236,20 @@ [output path]: #gloss-output-path +- [output closure]{#gloss-output-closure}\ + The [closure] of an [output path]. It only contains what is [reachable] from the output. + +- [deriving path]{#gloss-deriving-path} + + Deriving paths are a way to refer to [store objects][store object] that ar not yet [realised][realise]. + This is necessary because, in general and particularly for [content-addressed derivations][content-addressed derivation], the [output path] of an [output] is not known in advance. + There are two forms: + + - *constant*: just a [store path] + It can be made [valid][validity] by copying it into the store: from the evaluator, command line interface or another store. + + - *output*: a pair of a [store path] to a [derivation] and an [output] name. + - [deriver]{#gloss-deriver} The [store derivation] that produced an [output path]. @@ -252,13 +287,15 @@ See [installables](./command-ref/new-cli/nix.md#installables) for [`nix` commands](./command-ref/new-cli/nix.md) (experimental) for details. -- [NAR]{#gloss-nar} +- [Nix Archive (NAR)]{#gloss-nar} A *N*ix *AR*chive. This is a serialisation of a path in the Nix store. It can contain regular files, directories and symbolic links. NARs are generated and unpacked using `nix-store --dump` and `nix-store --restore`. + See [Nix Archive](store/file-system-object/content-address.html#serial-nix-archive) for details. + - [`∅`]{#gloss-emtpy-set} The empty set symbol. In the context of profile history, this denotes a package is not present in a particular version of the profile. @@ -275,7 +312,7 @@ - [package attribute set]{#package-attribute-set} - An [attribute set](@docroot@/language/values.md#attribute-set) containing the attribute `type = "derivation";` (derivation for historical reasons), as well as other attributes, such as + An [attribute set](@docroot@/language/types.md#attribute-set) containing the attribute `type = "derivation";` (derivation for historical reasons), as well as other attributes, such as - attributes that refer to the files of a [package], typically in the form of [derivation outputs](#output), - attributes that declare something about how the package is supposed to be installed or used, - other metadata or arbitrary attributes. @@ -288,16 +325,35 @@ See [String interpolation](./language/string-interpolation.md) for details. - [string]: ./language/values.md#type-string - [path]: ./language/values.md#type-path - [attribute name]: ./language/values.md#attribute-set + [string]: ./language/types.md#type-string + [path]: ./language/types.md#type-path + [attribute name]: ./language/types.md#attribute-set + +- [base directory]{#gloss-base-directory} + + The location from which relative paths are resolved. + + - For expressions in a file, the base directory is the directory containing that file. + This is analogous to the directory of a [base URL](https://datatracker.ietf.org/doc/html/rfc1808#section-3.3). + + + + - For expressions written in command line arguments with [`--expr`](@docroot@/command-ref/opt-common.html#opt-expr), the base directory is the current working directory. + + [base directory]: #gloss-base-directory - [experimental feature]{#gloss-experimental-feature} Not yet stabilized functionality guarded by named experimental feature flags. These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting. - See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/contributing/experimental-features.md). + See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/development/experimental-features.md). [Nix language]: ./language/index.md diff --git a/doc/manual/src/installation/env-variables.md b/doc/manual/src/installation/env-variables.md index db98f52ff..035090421 100644 --- a/doc/manual/src/installation/env-variables.md +++ b/doc/manual/src/installation/env-variables.md @@ -53,7 +53,8 @@ ssl-cert-file = /etc/ssl/my-certificate-bundle.crt The Nix installer has special handling for these proxy-related environment variables: `http_proxy`, `https_proxy`, `ftp_proxy`, -`no_proxy`, `HTTP_PROXY`, `HTTPS_PROXY`, `FTP_PROXY`, `NO_PROXY`. +`all_proxy`, `no_proxy`, `HTTP_PROXY`, `HTTPS_PROXY`, `FTP_PROXY`, +`ALL_PROXY`, `NO_PROXY`. If any of these variables are set when running the Nix installer, then the installer will create an override file at diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index 0dc989159..6a168ff3d 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -50,7 +50,7 @@ Supported systems: To explicitly instruct the installer to perform a multi-user installation on your system: ```console -$ curl -L https://nixos.org/nix/install | sh -s -- --daemon +$ bash <(curl -L https://nixos.org/nix/install) --daemon ``` You can run this under your usual user account or `root`. @@ -61,7 +61,7 @@ The script will invoke `sudo` as needed. To explicitly select a single-user installation on your system: ```console -$ curl -L https://nixos.org/nix/install | sh -s -- --no-daemon +$ bash <(curl -L https://nixos.org/nix/install) --no-daemon ``` In a single-user installation, `/nix` is owned by the invoking user. @@ -77,7 +77,7 @@ $ su root # Installing from a binary tarball You can also download a binary tarball that contains Nix and all its dependencies: -- Choose a [version](https://releases.nixos.org/?prefix=nix/) and [system type](../contributing/hacking.md#platforms) +- Choose a [version](https://releases.nixos.org/?prefix=nix/) and [system type](../development/building.md#platforms) - Download and unpack the tarball - Run the installer diff --git a/doc/manual/src/installation/uninstall.md b/doc/manual/src/installation/uninstall.md index 9ead5e53c..590327fea 100644 --- a/doc/manual/src/installation/uninstall.md +++ b/doc/manual/src/installation/uninstall.md @@ -1,16 +1,8 @@ # Uninstalling Nix -## Single User - -If you have a [single-user installation](./installing-binary.md#single-user-installation) of Nix, uninstall it by running: - -```console -$ rm -rf /nix -``` - ## Multi User -Removing a [multi-user installation](./installing-binary.md#multi-user-installation) of Nix is more involved, and depends on the operating system. +Removing a [multi-user installation](./installing-binary.md#multi-user-installation) depends on the operating system. ### Linux @@ -51,7 +43,15 @@ which you may remove. ### macOS -1. Edit `/etc/zshrc`, `/etc/bashrc`, and `/etc/bash.bashrc` to remove the lines sourcing `nix-daemon.sh`, which should look like this: +1. If system-wide shell initialisation files haven't been altered since installing Nix, use the backups made by the installer: + + ```console + sudo mv /etc/zshrc.backup-before-nix /etc/zshrc + sudo mv /etc/bashrc.backup-before-nix /etc/bashrc + sudo mv /etc/bash.bashrc.backup-before-nix /etc/bash.bashrc + ``` + + Otherwise, edit `/etc/zshrc`, `/etc/bashrc`, and `/etc/bash.bashrc` to remove the lines sourcing `nix-daemon.sh`, which should look like this: ```bash # Nix @@ -61,18 +61,6 @@ which you may remove. # End Nix ``` - If these files haven't been altered since installing Nix you can simply put - the backups back in place: - - ```console - sudo mv /etc/zshrc.backup-before-nix /etc/zshrc - sudo mv /etc/bashrc.backup-before-nix /etc/bashrc - sudo mv /etc/bash.bashrc.backup-before-nix /etc/bash.bashrc - ``` - - This will stop shells from sourcing the file and bringing everything you - installed using Nix in scope. - 2. Stop and remove the Nix daemon services: ```console @@ -82,8 +70,7 @@ which you may remove. sudo rm /Library/LaunchDaemons/org.nixos.darwin-store.plist ``` - This stops the Nix daemon and prevents it from being started next time you - boot the system. + This stops the Nix daemon and prevents it from being started next time you boot the system. 3. Remove the `nixbld` group and the `_nixbuildN` users: @@ -94,25 +81,42 @@ which you may remove. This will remove all the build users that no longer serve a purpose. -4. Edit fstab using `sudo vifs` to remove the line mounting the Nix Store - volume on `/nix`, which looks like - `UUID= /nix apfs rw,noauto,nobrowse,suid,owners` or - `LABEL=Nix\040Store /nix apfs rw,nobrowse`. This will prevent automatic - mounting of the Nix Store volume. +4. Edit fstab using `sudo vifs` to remove the line mounting the Nix Store volume on `/nix`, which looks like -5. Edit `/etc/synthetic.conf` to remove the `nix` line. If this is the only - line in the file you can remove it entirely, `sudo rm /etc/synthetic.conf`. - This will prevent the creation of the empty `/nix` directory to provide a - mountpoint for the Nix Store volume. + ``` + UUID= /nix apfs rw,noauto,nobrowse,suid,owners + ``` + or -6. Remove the files Nix added to your system: + ``` + LABEL=Nix\040Store /nix apfs rw,nobrowse + ``` + + by setting the cursor on the respective line using the error keys, and pressing `dd`, and then `:wq` to save the file. + + This will prevent automatic mounting of the Nix Store volume. + +5. Edit `/etc/synthetic.conf` to remove the `nix` line. + If this is the only line in the file you can remove it entirely: + + ```bash + if [ -f /etc/synthetic.conf ]; then + if [ "$(cat /etc/synthetic.conf)" = "nix" ]; then + sudo rm /etc/synthetic.conf + else + sudo vi /etc/synthetic.conf + fi + fi + ``` + + This will prevent the creation of the empty `/nix` directory. + +6. Remove the files Nix added to your system, except for the store: ```console sudo rm -rf /etc/nix /var/root/.nix-profile /var/root/.nix-defexpr /var/root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels ``` - This gets rid of any data Nix may have created except for the store which is - removed next. 7. Remove the Nix Store volume: @@ -120,29 +124,32 @@ which you may remove. sudo diskutil apfs deleteVolume /nix ``` - This will remove the Nix Store volume and everything that was added to the - store. + This will remove the Nix Store volume and everything that was added to the store. - If the output indicates that the command couldn't remove the volume, you should - make sure you don't have an _unmounted_ Nix Store volume. Look for a - "Nix Store" volume in the output of the following command: + If the output indicates that the command couldn't remove the volume, you should make sure you don't have an _unmounted_ Nix Store volume. + Look for a "Nix Store" volume in the output of the following command: ```console diskutil list ``` - If you _do_ see a "Nix Store" volume, delete it by re-running the diskutil - deleteVolume command, but replace `/nix` with the store volume's `diskXsY` - identifier. + If you _do_ find a "Nix Store" volume, delete it by running `diskutil deleteVolume` with the store volume's `diskXsY` identifier. > **Note** > -> After you complete the steps here, you will still have an empty `/nix` -> directory. This is an expected sign of a successful uninstall. The empty -> `/nix` directory will disappear the next time you reboot. +> After you complete the steps here, you will still have an empty `/nix` directory. +> This is an expected sign of a successful uninstall. +> The empty `/nix` directory will disappear the next time you reboot. > -> You do not have to reboot to finish uninstalling Nix. The uninstall is -> complete. macOS (Catalina+) directly controls root directories and its -> read-only root will prevent you from manually deleting the empty `/nix` -> mountpoint. +> You do not have to reboot to finish uninstalling Nix. +> The uninstall is complete. +> macOS (Catalina+) directly controls root directories, and its read-only root will prevent you from manually deleting the empty `/nix` mountpoint. +## Single User + +To remove a [single-user installation](./installing-binary.md#single-user-installation) of Nix, run: + +```console +$ rm -rf /nix ~/.nix-channels ~/.nix-defexpr ~/.nix-profile +``` +You might also want to manually remove references to Nix from your `~/.profile`. diff --git a/doc/manual/src/installation/upgrading.md b/doc/manual/src/installation/upgrading.md index 38edcdbc5..a433f1d30 100644 --- a/doc/manual/src/installation/upgrading.md +++ b/doc/manual/src/installation/upgrading.md @@ -28,7 +28,7 @@ $ sudo su ## macOS multi-user ```console -$ sudo nix-env --install --file '' --attr nix -I nixpkgs=channel:nixpkgs-unstable +$ sudo nix-env --install --file '' --attr nix cacert -I nixpkgs=channel:nixpkgs-unstable $ sudo launchctl remove org.nixos.nix-daemon $ sudo launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist ``` diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 7306fc182..51b83fc8a 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -113,19 +113,18 @@ Derivations can declare some infrequently used optional attributes. > `nix-build`. If the [`configurable-impure-env` experimental - feature](@docroot@/contributing/experimental-features.md#xp-feature-configurable-impure-env) + feature](@docroot@/development/experimental-features.md#xp-feature-configurable-impure-env) is enabled, these environment variables can also be controlled through the [`impure-env`](@docroot@/command-ref/conf-file.md#conf-impure-env) configuration setting. - [`outputHash`]{#adv-attr-outputHash}; [`outputHashAlgo`]{#adv-attr-outputHashAlgo}; [`outputHashMode`]{#adv-attr-outputHashMode}\ - These attributes declare that the derivation is a so-called - *fixed-output derivation*, which means that a cryptographic hash of - the output is already known in advance. When the build of a - fixed-output derivation finishes, Nix computes the cryptographic - hash of the output and compares it to the hash declared with these - attributes. If there is a mismatch, the build fails. + These attributes declare that the derivation is a so-called *fixed-output derivation* (FOD), which means that a cryptographic hash of the output is already known in advance. + + As opposed to regular derivations, the [`builder`] executable of a fixed-output derivation has access to the network. + Nix computes a cryptographic hash of its output and compares that to the hash declared with these attributes. + If there is a mismatch, the derivation fails. The rationale for fixed-output derivations is derivations such as those produced by the `fetchurl` function. This function downloads a @@ -188,38 +187,49 @@ Derivations can declare some infrequently used optional attributes. } ``` - The `outputHashAlgo` attribute specifies the hash algorithm used to - compute the hash. It can currently be `"sha1"`, `"sha256"` or - `"sha512"`. + The `outputHash` attribute must be a string containing the hash in either hexadecimal or "nix32" encoding, or following the format for integrity metadata as defined by [SRI](https://www.w3.org/TR/SRI/). + The "nix32" encoding is an adaptation of base-32 encoding. + The [`convertHash`](@docroot@/language/builtins.md#builtins-convertHash) function shows how to convert between different encodings, and the [`nix-hash` command](../command-ref/nix-hash.md) has information about obtaining the hash for some contents, as well as converting to and from encodings. + + The `outputHashAlgo` attribute specifies the hash algorithm used to compute the hash. + It can currently be `"sha1"`, `"sha256"`, `"sha512"`, or `null`. + `outputHashAlgo` can only be `null` when `outputHash` follows the SRI format. The `outputHashMode` attribute determines how the hash is computed. - It must be one of the following two values: + It must be one of the following values: - - `"flat"`\ - The output must be a non-executable regular file. If it isn’t, - the build fails. The hash is simply computed over the contents - of that file (so it’s equal to what Unix commands like - `sha256sum` or `sha1sum` produce). + - [`"flat"`](@docroot@/store/store-object/content-address.md#method-flat) This is the default. - - `"recursive"`\ - The hash is computed over the NAR archive dump of the output - (i.e., the result of [`nix-store --dump`](@docroot@/command-ref/nix-store/dump.md)). In - this case, the output can be anything, including a directory - tree. + - [`"recursive"` or `"nar"`](@docroot@/store/store-object/content-address.md#method-nix-archive) - The `outputHash` attribute, finally, must be a string containing - the hash in either hexadecimal or base-32 notation. (See the - [`nix-hash` command](../command-ref/nix-hash.md) for information - about converting to and from base-32 notation.) + > **Compatibility** + > + > `"recursive"` is the traditional way of indicating this, + > and is supported since 2005 (virtually the entire history of Nix). + > `"nar"` is more clear, and consistent with other parts of Nix (such as the CLI), + > however support for it is only added in Nix version 2.21. + + - [`"text"`](@docroot@/store/store-object/content-address.md#method-text) + + > **Warning** + > + > The use of this method for derivation outputs is part of the [`dynamic-derivations`][xp-feature-dynamic-derivations] experimental feature. + + - [`"git"`](@docroot@/store/store-object/content-address.md#method-git) + + > **Warning** + > + > This method is part of the [`git-hashing`][xp-feature-git-hashing] experimental feature. - [`__contentAddressed`]{#adv-attr-__contentAddressed} + > **Warning** - > This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md). + > This attribute is part of an [experimental feature](@docroot@/development/experimental-features.md). > > To use this attribute, you must enable the - > [`ca-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-ca-derivations) experimental feature. + > [`ca-derivations`][xp-feature-ca-derivations] experimental feature. > For example, in [nix.conf](../command-ref/conf-file.md) you could add: > > ``` @@ -268,7 +278,9 @@ Derivations can declare some infrequently used optional attributes. > **Note** > - > If set to `false`, the [`builder`](./derivations.md#attr-builder) should be able to run on the system type specified in the [`system` attribute](./derivations.md#attr-system), since the derivation cannot be substituted. + > If set to `false`, the [`builder`] should be able to run on the system type specified in the [`system` attribute](./derivations.md#attr-system), since the derivation cannot be substituted. + + [`builder`]: ./derivations.md#attr-builder - [`__structuredAttrs`]{#adv-attr-structuredAttrs}\ If the special attribute `__structuredAttrs` is set to `true`, the other derivation @@ -290,6 +302,12 @@ Derivations can declare some infrequently used optional attributes. (associative) arrays. For example, the attribute `hardening.format = true` ends up as the Bash associative array element `${hardening[format]}`. + > **Warning** + > + > If set to `true`, other advanced attributes such as [`allowedReferences`](#adv-attr-allowedReferences), [`allowedReferences`](#adv-attr-allowedReferences), [`allowedRequisites`](#adv-attr-allowedRequisites), + [`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites), maxSize, and maxClosureSize. + will have no effect. + - [`outputChecks`]{#adv-attr-outputChecks}\ When using [structured attributes](#adv-attr-structuredAttrs), the `outputChecks` attribute allows defining checks per-output. @@ -299,7 +317,7 @@ Derivations can declare some infrequently used optional attributes. [`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites), the following attributes are available: - - `maxSize` defines the maximum size of the resulting [store object](../glossary.md#gloss-store-object). + - `maxSize` defines the maximum size of the resulting [store object](@docroot@/store/store-object.md). - `maxClosureSize` defines the maximum size of the output's closure. - `ignoreSelfRefs` controls whether self-references should be considered when checking for allowed references/requisites. @@ -351,3 +369,7 @@ Derivations can declare some infrequently used optional attributes. ``` ensures that the derivation can only be built on a machine with the `kvm` feature. + +[xp-feature-ca-derivations]: @docroot@/development/experimental-features.md#xp-feature-ca-derivations +[xp-feature-dynamic-derivations]: @docroot@/development/experimental-features.md#xp-feature-dynamic-derivations +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/doc/manual/src/language/builtin-constants-prefix.md b/doc/manual/src/language/builtin-constants-prefix.md deleted file mode 100644 index 50f43006d..000000000 --- a/doc/manual/src/language/builtin-constants-prefix.md +++ /dev/null @@ -1,5 +0,0 @@ -# Built-in Constants - -These constants are built into the Nix language evaluator: - - diff --git a/doc/manual/src/language/builtin-constants-suffix.md b/doc/manual/src/language/builtin-constants-suffix.md deleted file mode 100644 index a74db2857..000000000 --- a/doc/manual/src/language/builtin-constants-suffix.md +++ /dev/null @@ -1 +0,0 @@ - diff --git a/doc/manual/src/language/builtins-prefix.md b/doc/manual/src/language/builtins-prefix.md index 7b2321466..fb983bb7f 100644 --- a/doc/manual/src/language/builtins-prefix.md +++ b/doc/manual/src/language/builtins-prefix.md @@ -1,9 +1,11 @@ -# Built-in Functions +# Built-ins -This section lists the functions built into the Nix language evaluator. -All built-in functions are available through the global [`builtins`](./builtin-constants.md#builtins-builtins) constant. +This section lists the values and functions built into the Nix language evaluator. +All built-ins are available through the global [`builtins`](#builtins-builtins) constant. -For convenience, some built-ins can be accessed directly: +Some built-ins are also exposed directly in the global scope: + + - [`derivation`](#builtins-derivation) - [`import`](#builtins-import) diff --git a/doc/manual/src/language/constructs.md b/doc/manual/src/language/constructs.md deleted file mode 100644 index a82ec5960..000000000 --- a/doc/manual/src/language/constructs.md +++ /dev/null @@ -1,408 +0,0 @@ -# Language Constructs - -## Recursive sets - -Recursive sets are like normal [attribute sets](./values.md#attribute-set), but the attributes can refer to each other. - -> *rec-attrset* = `rec {` [ *name* `=` *expr* `;` `]`... `}` - -Example: - -```nix -rec { - x = y; - y = 123; -}.x -``` - -This evaluates to `123`. - -Note that without `rec` the binding `x = y;` would -refer to the variable `y` in the surrounding scope, if one exists, and -would be invalid if no such variable exists. That is, in a normal -(non-recursive) set, attributes are not added to the lexical scope; in a -recursive set, they are. - -Recursive sets of course introduce the danger of infinite recursion. For -example, the expression - -```nix -rec { - x = y; - y = x; -}.x -``` - -will crash with an `infinite recursion encountered` error message. - -## Let-expressions - -A let-expression allows you to define local variables for an expression. - -> *let-in* = `let` [ *identifier* = *expr* ]... `in` *expr* - -Example: - -```nix -let - x = "foo"; - y = "bar"; -in x + y -``` - -This evaluates to `"foobar"`. - -## Inheriting attributes - -When defining an [attribute set](./values.md#attribute-set) or in a [let-expression](#let-expressions) it is often convenient to copy variables from the surrounding lexical scope (e.g., when you want to propagate attributes). -This can be shortened using the `inherit` keyword. - -Example: - -```nix -let x = 123; in -{ - inherit x; - y = 456; -} -``` - -is equivalent to - -```nix -let x = 123; in -{ - x = x; - y = 456; -} -``` - -and both evaluate to `{ x = 123; y = 456; }`. - -> **Note** -> -> This works because `x` is added to the lexical scope by the `let` construct. - -It is also possible to inherit attributes from another attribute set. - -Example: - -In this fragment from `all-packages.nix`, - -```nix -graphviz = (import ../tools/graphics/graphviz) { - inherit fetchurl stdenv libpng libjpeg expat x11 yacc; - inherit (xorg) libXaw; -}; - -xorg = { - libX11 = ...; - libXaw = ...; - ... -} - -libpng = ...; -libjpg = ...; -... -``` - -the set used in the function call to the function defined in -`../tools/graphics/graphviz` inherits a number of variables from the -surrounding scope (`fetchurl` ... `yacc`), but also inherits `libXaw` -(the X Athena Widgets) from the `xorg` set. - -Summarizing the fragment - -```nix -... -inherit x y z; -inherit (src-set) a b c; -... -``` - -is equivalent to - -```nix -... -x = x; y = y; z = z; -a = src-set.a; b = src-set.b; c = src-set.c; -... -``` - -when used while defining local variables in a let-expression or while -defining a set. - -In a `let` expression, `inherit` can be used to selectively bring specific attributes of a set into scope. For example - - -```nix -let - x = { a = 1; b = 2; }; - inherit (builtins) attrNames; -in -{ - names = attrNames x; -} -``` - -is equivalent to - -```nix -let - x = { a = 1; b = 2; }; -in -{ - names = builtins.attrNames x; -} -``` - -both evaluate to `{ names = [ "a" "b" ]; }`. - -## Functions - -Functions have the following form: - -```nix -pattern: body -``` - -The pattern specifies what the argument of the function must look like, -and binds variables in the body to (parts of) the argument. There are -three kinds of patterns: - - - If a pattern is a single identifier, then the function matches any - argument. Example: - - ```nix - let negate = x: !x; - concat = x: y: x + y; - in if negate true then concat "foo" "bar" else "" - ``` - - Note that `concat` is a function that takes one argument and returns - a function that takes another argument. This allows partial - parameterisation (i.e., only filling some of the arguments of a - function); e.g., - - ```nix - map (concat "foo") [ "bar" "bla" "abc" ] - ``` - - evaluates to `[ "foobar" "foobla" "fooabc" ]`. - - - A *set pattern* of the form `{ name1, name2, …, nameN }` matches a - set containing the listed attributes, and binds the values of those - attributes to variables in the function body. For example, the - function - - ```nix - { x, y, z }: z + y + x - ``` - - can only be called with a set containing exactly the attributes `x`, - `y` and `z`. No other attributes are allowed. If you want to allow - additional arguments, you can use an ellipsis (`...`): - - ```nix - { x, y, z, ... }: z + y + x - ``` - - This works on any set that contains at least the three named - attributes. - - It is possible to provide *default values* for attributes, in - which case they are allowed to be missing. A default value is - specified by writing `name ? e`, where *e* is an arbitrary - expression. For example, - - ```nix - { x, y ? "foo", z ? "bar" }: z + y + x - ``` - - specifies a function that only requires an attribute named `x`, but - optionally accepts `y` and `z`. - - - An `@`-pattern provides a means of referring to the whole value - being matched: - - ```nix - args@{ x, y, z, ... }: z + y + x + args.a - ``` - - but can also be written as: - - ```nix - { x, y, z, ... } @ args: z + y + x + args.a - ``` - - Here `args` is bound to the argument *as passed*, which is further - matched against the pattern `{ x, y, z, ... }`. - The `@`-pattern makes mainly sense with an ellipsis(`...`) as - you can access attribute names as `a`, using `args.a`, which was - given as an additional attribute to the function. - - > **Warning** - > - > `args@` binds the name `args` to the attribute set that is passed to the function. - > In particular, `args` does *not* include any default values specified with `?` in the function's set pattern. - > - > For instance - > - > ```nix - > let - > f = args@{ a ? 23, ... }: [ a args ]; - > in - > f {} - > ``` - > - > is equivalent to - > - > ```nix - > let - > f = args @ { ... }: [ (args.a or 23) args ]; - > in - > f {} - > ``` - > - > and both expressions will evaluate to: - > - > ```nix - > [ 23 {} ] - > ``` - -Note that functions do not have names. If you want to give them a name, -you can bind them to an attribute, e.g., - -```nix -let concat = { x, y }: x + y; -in concat { x = "foo"; y = "bar"; } -``` - -## Conditionals - -Conditionals look like this: - -```nix -if e1 then e2 else e3 -``` - -where *e1* is an expression that should evaluate to a Boolean value -(`true` or `false`). - -## Assertions - -Assertions are generally used to check that certain requirements on or -between features and dependencies hold. They look like this: - -```nix -assert e1; e2 -``` - -where *e1* is an expression that should evaluate to a Boolean value. If -it evaluates to `true`, *e2* is returned; otherwise expression -evaluation is aborted and a backtrace is printed. - -Here is a Nix expression for the Subversion package that shows how -assertions can be used:. - -```nix -{ localServer ? false -, httpServer ? false -, sslSupport ? false -, pythonBindings ? false -, javaSwigBindings ? false -, javahlBindings ? false -, stdenv, fetchurl -, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null -}: - -assert localServer -> db4 != null; â‘ -assert httpServer -> httpd != null && httpd.expat == expat; â‘¡ -assert sslSupport -> openssl != null && (httpServer -> httpd.openssl == openssl); â‘¢ -assert pythonBindings -> swig != null && swig.pythonSupport; -assert javaSwigBindings -> swig != null && swig.javaSupport; -assert javahlBindings -> j2sdk != null; - -stdenv.mkDerivation { - name = "subversion-1.1.1"; - ... - openssl = if sslSupport then openssl else null; â‘£ - ... -} -``` - -The points of interest are: - -1. This assertion states that if Subversion is to have support for - local repositories, then Berkeley DB is needed. So if the Subversion - function is called with the `localServer` argument set to `true` but - the `db4` argument set to `null`, then the evaluation fails. - - Note that `->` is the [logical - implication](https://en.wikipedia.org/wiki/Truth_table#Logical_implication) - Boolean operation. - -2. This is a more subtle condition: if Subversion is built with Apache - (`httpServer`) support, then the Expat library (an XML library) used - by Subversion should be same as the one used by Apache. This is - because in this configuration Subversion code ends up being linked - with Apache code, and if the Expat libraries do not match, a build- - or runtime link error or incompatibility might occur. - -3. This assertion says that in order for Subversion to have SSL support - (so that it can access `https` URLs), an OpenSSL library must be - passed. Additionally, it says that *if* Apache support is enabled, - then Apache's OpenSSL should match Subversion's. (Note that if - Apache support is not enabled, we don't care about Apache's - OpenSSL.) - -4. The conditional here is not really related to assertions, but is - worth pointing out: it ensures that if SSL support is disabled, then - the Subversion derivation is not dependent on OpenSSL, even if a - non-`null` value was passed. This prevents an unnecessary rebuild of - Subversion if OpenSSL changes. - -## With-expressions - -A *with-expression*, - -```nix -with e1; e2 -``` - -introduces the set *e1* into the lexical scope of the expression *e2*. -For instance, - -```nix -let as = { x = "foo"; y = "bar"; }; -in with as; x + y -``` - -evaluates to `"foobar"` since the `with` adds the `x` and `y` attributes -of `as` to the lexical scope in the expression `x + y`. The most common -use of `with` is in conjunction with the `import` function. E.g., - -```nix -with (import ./definitions.nix); ... -``` - -makes all attributes defined in the file `definitions.nix` available as -if they were defined locally in a `let`-expression. - -The bindings introduced by `with` do not shadow bindings introduced by -other means, e.g. - -```nix -let a = 3; in with { a = 1; }; let a = 4; in with { a = 2; }; ... -``` - -establishes the same scope as - -```nix -let a = 1; in let a = 2; in let a = 3; in let a = 4; in ... -``` - -## Comments - -Comments can be single-line, started with a `#` character, or -inline/multi-line, enclosed within `/* ... */`. diff --git a/doc/manual/src/language/constructs/lookup-path.md b/doc/manual/src/language/constructs/lookup-path.md index e87d2922b..11b9fe88c 100644 --- a/doc/manual/src/language/constructs/lookup-path.md +++ b/doc/manual/src/language/constructs/lookup-path.md @@ -4,9 +4,9 @@ > > *lookup-path* = `<` *identifier* [ `/` *identifier* ]... `>` -A lookup path is an identifier with an optional path suffix that resolves to a [path value](@docroot@/language/values.md#type-path) if the identifier matches a search path entry. +A lookup path is an identifier with an optional path suffix that resolves to a [path value](@docroot@/language/types.md#type-path) if the identifier matches a search path entry. -The value of a lookup path is determined by [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath). +The value of a lookup path is determined by [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath). See [`builtins.findFile`](@docroot@/language/builtins.md#builtins-findFile) for details on lookup path resolution. diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index 75f824a34..8e3f0f791 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -12,12 +12,12 @@ It outputs an attribute set, and produces a [store derivation] as a side effect ### Required -- [`name`]{#attr-name} ([String](@docroot@/language/values.md#type-string)) +- [`name`]{#attr-name} ([String](@docroot@/language/types.md#type-string)) A symbolic name for the derivation. It is added to the [store path] of the corresponding [store derivation] as well as to its [output paths](@docroot@/glossary.md#gloss-output-path). - [store path]: @docroot@/glossary.md#gloss-store-path + [store path]: @docroot@/store/store-path.md > **Example** > @@ -31,7 +31,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect > The store derivation's path will be `/nix/store/-hello.drv`. > The [output](#attr-outputs) paths will be of the form `/nix/store/-hello[-]` -- [`system`]{#attr-system} ([String](@docroot@/language/values.md#type-string)) +- [`system`]{#attr-system} ([String](@docroot@/language/types.md#type-string)) The system type on which the [`builder`](#attr-builder) executable is meant to be run. @@ -64,9 +64,9 @@ It outputs an attribute set, and produces a [store derivation] as a side effect > } > ``` > - > [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) has the value of the [`system` configuration option], and defaults to the system type of the current Nix installation. + > [`builtins.currentSystem`](@docroot@/language/builtins.md#builtins-currentSystem) has the value of the [`system` configuration option], and defaults to the system type of the current Nix installation. -- [`builder`]{#attr-builder} ([Path](@docroot@/language/values.md#type-path) | [String](@docroot@/language/values.md#type-string)) +- [`builder`]{#attr-builder} ([Path](@docroot@/language/types.md#type-path) | [String](@docroot@/language/types.md#type-string)) Path to an executable that will perform the build. @@ -113,7 +113,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect ### Optional -- [`args`]{#attr-args} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) +- [`args`]{#attr-args} ([List](@docroot@/language/types.md#list) of [String](@docroot@/language/types.md#type-string)) Default: `[ ]` @@ -132,7 +132,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect > }; > ``` -- [`outputs`]{#attr-outputs} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) +- [`outputs`]{#attr-outputs} ([List](@docroot@/language/types.md#list) of [String](@docroot@/language/types.md#type-string)) Default: `[ "out" ]` @@ -141,7 +141,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect By default, a derivation produces a single output called `out`. However, derivations can produce multiple outputs. - This allows the associated [store objects](@docroot@/glossary.md#gloss-store-object) and their [closures](@docroot@/glossary.md#gloss-closure) to be copied or garbage-collected separately. + This allows the associated [store objects](@docroot@/store/store-object.md) and their [closures](@docroot@/glossary.md#gloss-closure) to be copied or garbage-collected separately. > **Example** > diff --git a/doc/manual/src/language/identifiers.md b/doc/manual/src/language/identifiers.md new file mode 100644 index 000000000..861ee3e20 --- /dev/null +++ b/doc/manual/src/language/identifiers.md @@ -0,0 +1,51 @@ +# Identifiers + +An *identifier* is an [ASCII](https://en.wikipedia.org/wiki/ASCII) character sequence that: +- Starts with a letter (`a-z`, `A-Z`) or underscore (`_`) +- Can contain any number of: + - Letters (`a-z`, `A-Z`) + - Digits (`0-9`) + - Underscores (`_`) + - Apostrophes (`'`) + - Hyphens (`-`) +- Is not one of the [keywords](#keywords) + +> **Syntax** +> +> *identifier* ~ `[A-Za-z_][A-Za-z0-9_'-]*` + +# Names + +A *name* can be written as an [identifier](#identifier) or a [string literal](./syntax.md#string-literal). + +> **Syntax** +> +> *name* → *identifier* | *string* + +Names are used in [attribute sets](./syntax.md#attrs-literal), [`let` bindings](./syntax.md#let-expressions), and [`inherit`](./syntax.md#inheriting-attributes). +Two names are the same if they represent the same sequence of characters, regardless of whether they are written as identifiers or strings. + +# Keywords + +These keywords are reserved and cannot be used as [identifiers](#identifiers): + +- [`assert`](./syntax.md#assertions) +- [`else`][if] +- [`if`][if] +- [`in`][let] +- [`inherit`](./syntax.md#inheriting-attributes) +- [`let`][let] +- [`or`](./operators.md#attribute-selection) (see note) +- [`rec`](./syntax.md#recursive-sets) +- [`then`][if] +- [`with`](./syntax.md#with-expressions) + +[if]: ./syntax.md#conditionals +[let]: ./syntax.md#let-expressions + +> **Note** +> +> The Nix language evaluator currently allows `or` to be used as a name in some contexts, for backwards compatibility reasons. +> Users are advised not to rely on this. +> +> There are long-standing issues with how `or` is parsed as a name, which can't be resolved without making a breaking change to the language. diff --git a/doc/manual/src/language/import-from-derivation.md b/doc/manual/src/language/import-from-derivation.md index fb12ba51a..e901f5bcf 100644 --- a/doc/manual/src/language/import-from-derivation.md +++ b/doc/manual/src/language/import-from-derivation.md @@ -2,9 +2,9 @@ The value of a Nix expression can depend on the contents of a [store object]. -[store object]: @docroot@/glossary.md#gloss-store-object +[store object]: @docroot@/store/store-object.md -Passing an expression `expr` that evaluates to a [store path](@docroot@/glossary.md#gloss-store-path) to any built-in function which reads from the filesystem constitutes Import From Derivation (IFD): +Passing an expression `expr` that evaluates to a [store path](@docroot@/store/store-path.md) to any built-in function which reads from the filesystem constitutes Import From Derivation (IFD): - [`import`](./builtins.md#builtins-import)` expr` - [`builtins.readFile`](./builtins.md#builtins-readFile)` expr` diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md index a26e43a05..2bfdbb8a0 100644 --- a/doc/manual/src/language/index.md +++ b/doc/manual/src/language/index.md @@ -1,7 +1,13 @@ # Nix Language The Nix language is designed for conveniently creating and composing *derivations* – precise descriptions of how contents of existing files are used to derive new files. -It is: + +> **Tip** +> +> These pages are written as a reference. +> If you are learning Nix, nix.dev has a good [introduction to the Nix language](https://nix.dev/tutorials/nix-language). + +The language is: - *domain-specific* @@ -47,7 +53,7 @@ This is an incomplete overview of language features, by example. - *Basic values* + *Basic values ([primitives](@docroot@/language/types.md#primitives))* @@ -65,7 +71,7 @@ This is an incomplete overview of language features, by example. - A string + A [string](@docroot@/language/types.md#type-string) @@ -88,6 +94,18 @@ This is an incomplete overview of language features, by example. + + + + `# Explanation` + + + + + A [comment](@docroot@/language/syntax.md#comments). + + + @@ -100,7 +118,7 @@ This is an incomplete overview of language features, by example. - String interpolation (expands to `"hello world"`, `"1 2 3"`, `"/nix/store/-bash-/bin/sh"`) + [String interpolation](@docroot@/language/string-interpolation.md) (expands to `"hello world"`, `"1 2 3"`, `"/nix/store/-bash-/bin/sh"`) @@ -112,7 +130,7 @@ This is an incomplete overview of language features, by example. - Booleans + [Booleans](@docroot@/language/types.md#type-boolean) @@ -124,7 +142,7 @@ This is an incomplete overview of language features, by example. - Null value + [Null](@docroot@/language/types.md#type-null) value @@ -136,7 +154,7 @@ This is an incomplete overview of language features, by example. - An integer + An [integer](@docroot@/language/types.md#type-int) @@ -148,7 +166,7 @@ This is an incomplete overview of language features, by example. - A floating point number + A [floating point number](@docroot@/language/types.md#type-float) @@ -160,7 +178,7 @@ This is an incomplete overview of language features, by example. - An absolute path + An absolute [path](@docroot@/language/types.md#type-path) @@ -172,7 +190,7 @@ This is an incomplete overview of language features, by example. - A path relative to the file containing this Nix expression + A [path](@docroot@/language/types.md#type-path) relative to the file containing this Nix expression @@ -184,7 +202,7 @@ This is an incomplete overview of language features, by example. - A home path. Evaluates to the `"/.config"`. + A home [path](@docroot@/language/types.md#type-path). Evaluates to the `"/.config"`. @@ -196,7 +214,7 @@ This is an incomplete overview of language features, by example. - Search path for Nix files. Value determined by [`$NIX_PATH` environment variable](../command-ref/env-common.md#env-NIX_PATH). + A [lookup path](@docroot@/language/constructs/lookup-path.md) for Nix files. Value determined by [`$NIX_PATH` environment variable](../command-ref/env-common.md#env-NIX_PATH). @@ -220,7 +238,7 @@ This is an incomplete overview of language features, by example. - A set with attributes named `x` and `y` + An [attribute set](@docroot@/language/types.md#attribute-set) with attributes named `x` and `y` @@ -244,7 +262,7 @@ This is an incomplete overview of language features, by example. - A recursive set, equivalent to `{ x = "foo"; y = "foobar"; }` + A [recursive set](@docroot@/language/syntax.md#recursive-sets), equivalent to `{ x = "foo"; y = "foobar"; }`. @@ -260,7 +278,7 @@ This is an incomplete overview of language features, by example. - Lists with three elements. + [Lists](@docroot@/language/types.md#list) with three elements. @@ -344,7 +362,7 @@ This is an incomplete overview of language features, by example. - Attribute selection (evaluates to `1`) + [Attribute selection](@docroot@/language/types.md#attribute-set) (evaluates to `1`) @@ -356,7 +374,7 @@ This is an incomplete overview of language features, by example. - Attribute selection with default (evaluates to `3`) + [Attribute selection](@docroot@/language/types.md#attribute-set) with default (evaluates to `3`) @@ -392,7 +410,7 @@ This is an incomplete overview of language features, by example. - Conditional expression + [Conditional expression](@docroot@/language/syntax.md#conditionals). @@ -404,7 +422,7 @@ This is an incomplete overview of language features, by example. - Assertion check (evaluates to `"yes!"`). + [Assertion](@docroot@/language/syntax.md#assertions) check (evaluates to `"yes!"`). @@ -416,7 +434,7 @@ This is an incomplete overview of language features, by example. - Variable definition + Variable definition. See [`let`-expressions](@docroot@/language/syntax.md#let-expressions). @@ -428,14 +446,44 @@ This is an incomplete overview of language features, by example. - Add all attributes from the given set to the scope (evaluates to `1`) + Add all attributes from the given set to the scope (evaluates to `1`). + + See [`with`-expressions](@docroot@/language/syntax.md#with-expressions) for details and shadowing caveats. - *Functions (lambdas)* + `inherit pkgs src;` + + + + + Adds the variables to the current scope (attribute set or `let` binding). + Desugars to `pkgs = pkgs; src = src;`. + See [Inheriting attributes](@docroot@/language/syntax.md#inheriting-attributes). + + + + + + + `inherit (pkgs) lib stdenv;` + + + + + Adds the attributes, from the attribute set in parentheses, to the current scope (attribute set or `let` binding). + Desugars to `lib = pkgs.lib; stdenv = pkgs.stdenv;`. + See [Inheriting attributes](@docroot@/language/syntax.md#inheriting-attributes). + + + + + + + *[Functions](@docroot@/language/syntax.md#functions) (lambdas)* @@ -452,7 +500,7 @@ This is an incomplete overview of language features, by example. - A function that expects an integer and returns it increased by 1 + A [function](@docroot@/language/syntax.md#functions) that expects an integer and returns it increased by 1. @@ -464,7 +512,7 @@ This is an incomplete overview of language features, by example. - Curried function, equivalent to `x: (y: x + y)`. Can be used like a function that takes two arguments and returns their sum. + Curried [function](@docroot@/language/syntax.md#functions), equivalent to `x: (y: x + y)`. Can be used like a function that takes two arguments and returns their sum. @@ -476,7 +524,7 @@ This is an incomplete overview of language features, by example. - A function call (evaluates to 101) + A [function](@docroot@/language/syntax.md#functions) call (evaluates to 101) @@ -488,7 +536,7 @@ This is an incomplete overview of language features, by example. - A function bound to a variable and subsequently called by name (evaluates to 103) + A [function](@docroot@/language/syntax.md#functions) bound to a variable and subsequently called by name (evaluates to 103) @@ -500,7 +548,7 @@ This is an incomplete overview of language features, by example. - A function that expects a set with required attributes `x` and `y` and concatenates them + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attributes `x` and `y` and concatenates them @@ -512,7 +560,7 @@ This is an incomplete overview of language features, by example. - A function that expects a set with required attribute `x` and optional `y`, using `"bar"` as default value for `y` + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attribute `x` and optional `y`, using `"bar"` as default value for `y` @@ -524,7 +572,7 @@ This is an incomplete overview of language features, by example. - A function that expects a set with required attributes `x` and `y` and ignores any other attributes + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attributes `x` and `y` and ignores any other attributes @@ -538,7 +586,7 @@ This is an incomplete overview of language features, by example. - A function that expects a set with required attributes `x` and `y`, and binds the whole set to `args` + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attributes `x` and `y`, and binds the whole set to `args` @@ -562,7 +610,8 @@ This is an incomplete overview of language features, by example. - Load and return Nix expression in given file + Load and return Nix expression in given file. + See [import](@docroot@/language/builtins.md#builtins-import). @@ -574,7 +623,8 @@ This is an incomplete overview of language features, by example. - Apply a function to every element of a list (evaluates to `[ 2 4 6 ]`) + Apply a function to every element of a list (evaluates to `[ 2 4 6 ]`). + See [`map`](@docroot@/language/builtins.md#builtins-map). diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index 6fd66864b..e1c020781 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -26,12 +26,16 @@ | Logical conjunction (`AND`) | *bool* `&&` *bool* | left | 12 | | Logical disjunction (`OR`) | *bool* \|\| *bool* | left | 13 | | [Logical implication] | *bool* `->` *bool* | right | 14 | +| [Pipe operator] (experimental) | *expr* `\|>` *func* | left | 15 | +| [Pipe operator] (experimental) | *func* `<\|` *expr* | right | 15 | -[string]: ./values.md#type-string -[path]: ./values.md#type-path -[number]: ./values.md#type-number -[list]: ./values.md#list -[attribute set]: ./values.md#attribute-set +[string]: ./types.md#type-string +[path]: ./types.md#type-path +[number]: ./types.md#type-float +[list]: ./types.md#list +[attribute set]: ./types.md#attribute-set + + ## Attribute selection @@ -42,12 +46,6 @@ Select the attribute denoted by attribute path *attrpath* from [attribute set] *attrset*. If the attribute doesn’t exist, return the *expr* after `or` if provided, otherwise abort evaluation. -An attribute path is a dot-separated list of [attribute names](./values.md#attribute-set). - -> **Syntax** -> -> *attrpath* = *name* [ `.` *name* ]... - [Attribute selection]: #attribute-selection ## Has attribute @@ -61,7 +59,7 @@ The result is a [Boolean] value. See also: [`builtins.hasAttr`](@docroot@/language/builtins.md#builtins-hasAttr) -[Boolean]: ./values.md#type-boolean +[Boolean]: ./types.md#type-boolean [Has attribute]: #has-attribute @@ -128,8 +126,8 @@ The result is a string. > The file or directory at *path* must exist and is copied to the [store]. > The path appears in the result as the corresponding [store path]. -[store path]: ../glossary.md#gloss-store-path -[store]: ../glossary.md#gloss-store +[store path]: @docroot@/store/store-path.md +[store]: @docroot@/glossary.md#gloss-store [String and path concatenation]: #string-and-path-concatenation @@ -141,7 +139,7 @@ The result is a string. Update [attribute set] *attrset1* with names and values from *attrset2*. -The returned attribute set will have of all the attributes in *attrset1* and *attrset2*. +The returned attribute set will have all of the attributes in *attrset1* and *attrset2*. If an attribute name is present in both, the attribute value from the latter is taken. [Update]: #update @@ -172,7 +170,7 @@ All comparison operators are implemented in terms of `<`, and the following equi - Numbers are type-compatible, see [arithmetic] operators. - Floating point numbers only differ up to a limited precision. -[function]: ./constructs.md#functions +[function]: ./syntax.md#functions [Equality]: #equality @@ -182,3 +180,34 @@ Equivalent to `!`*b1* `||` *b2*. [Logical implication]: #logical-implication +## Pipe operators + +- *a* `|>` *b* is equivalent to *b* *a* +- *a* `<|` *b* is equivalent to *a* *b* + +> **Example** +> +> ``` +> nix-repl> 1 |> builtins.add 2 |> builtins.mul 3 +> 9 +> +> nix-repl> builtins.add 1 <| builtins.mul 2 <| 3 +> 7 +> ``` + +> **Warning** +> +> This syntax is part of an +> [experimental feature](@docroot@/development/experimental-features.md) +> and may change in future releases. +> +> To use this syntax, make sure the +> [`pipe-operators` experimental feature](@docroot@/development/experimental-features.md#xp-feature-pipe-operators) +> is enabled. +> For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): +> +> ``` +> extra-experimental-features = pipe-operators +> ``` + +[Pipe operator]: #pipe-operators diff --git a/doc/manual/src/language/scope.md b/doc/manual/src/language/scope.md new file mode 100644 index 000000000..9373324e2 --- /dev/null +++ b/doc/manual/src/language/scope.md @@ -0,0 +1,28 @@ +# Scoping rules + +A *scope* in the Nix language is a dictionary keyed by [name](./identifiers.md#names), mapping each name to an expression and a *definition type*. +The definition type is either *explicit* or *implicit*. +Each entry in this dictionary is a *definition*. + +Explicit definitions are created by the following expressions: +- [let-expressions](syntax.md#let-expressions) +- [recursive attribute set literals](syntax.md#recursive-sets) (`rec`) +- [function literals](syntax.md#functions) + +Implicit definitions are only created by [with-expressions](./syntax.md#with-expressions). + +Every expression is *enclosed* by a scope. +The outermost expression is enclosed by the [built-in, global scope](./builtins.md), which contains only explicit definitions. +The expressions listed above *extend* their enclosing scope by adding new definitions, or replacing existing ones with the same name. +An explicit definition can replace a definition of any type; an implicit definition can only replace another implicit definition. + +Each of the above expressions defines which of its subexpressions are enclosed by the extended scope. +In all other cases, the same scope that encloses an expression is the enclosing scope for its subexpressions. + +The Nix language is [statically scoped](https://en.wikipedia.org/wiki/Scope_(computer_science)#Lexical_scope); +the value of a variable is determined only by the variable's enclosing scope, and not by the dynamic context in which the variable is evaluated. + +> **Note** +> +> Expressions entered into the [Nix REPL](@docroot@/command-ref/new-cli/nix3-repl.md) are enclosed by a scope that can be extended by command line arguments or previous REPL commands. +> These ways of extending scope are not, strictly speaking, part of the Nix language. diff --git a/doc/manual/src/language/string-context.md b/doc/manual/src/language/string-context.md new file mode 100644 index 000000000..6a3482cfd --- /dev/null +++ b/doc/manual/src/language/string-context.md @@ -0,0 +1,134 @@ +# String context + +> **Note** +> +> This is an advanced topic. +> The Nix language is designed to be used without the programmer consciously dealing with string contexts or even knowing what they are. + +A string in the Nix language is not just a sequence of characters like strings in other languages. +It is actually a pair of a sequence of characters and a *string context*. +The string context is an (unordered) set of *string context elements*. + +The purpose of string contexts is to collect non-string values attached to strings via +[string concatenation](./operators.md#string-concatenation), +[string interpolation](./string-interpolation.md), +and similar operations. +The idea is that a user can combine together values to create a build instructions for derivations without manually keeping track of where they come from. +Then the Nix language implicitly does that bookkeeping to efficiently obtain the closure of derivation inputs. + +> **Note** +> +> String contexts are *not* explicitly manipulated in idiomatic Nix language code. + +String context elements come in different forms: + +- [deriving path]{#string-context-element-derived-path} + + A string context element of this type is a [deriving path](@docroot@/glossary.md#gloss-deriving-path). + They can be either of type [constant](#string-context-constant) or [output](#string-context-output), which correspond to the types of deriving paths. + + - [Constant string context elements]{#string-context-constant} + + > **Example** + > + > [`builtins.storePath`] creates a string with a single constant string context element: + > + > ```nix + > builtins.getContext (builtins.storePath "/nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10") + > ``` + > evaluates to + > ```nix + > { + > "/nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10" = { + > path = true; + > }; + > } + > ``` + + [deriving path]: @docroot@/glossary.md#gloss-deriving-path + [store path]: @docroot@/glossary.md#gloss-store-path + [`builtins.storePath`]: ./builtins.md#builtins-storePath + + - [Output string context elements]{#string-context-output} + + > **Example** + > + > The behavior of string contexts are best demonstrated with a built-in function that is still experimental: [`builtins.outputOf`]. + > This example will *not* work with stable Nix! + > + > ```nix + > builtins.getContext + > (builtins.outputOf + > (builtins.storePath "/nix/store/fvchh9cvcr7kdla6n860hshchsba305w-hello-2.12.drv") + > "out") + > ``` + > evaluates to + > ```nix + > { + > "/nix/store/fvchh9cvcr7kdla6n860hshchsba305w-hello-2.12.drv" = { + > outputs = [ "out" ]; + > }; + > } + > ``` + + [`builtins.outputOf`]: ./builtins.md#builtins-outputOf + +- [*derivation deep*]{#string-context-element-derivation-deep} + + *derivation deep* is an advanced feature intended to be used with the + [`exportReferencesGraph` derivation attribute](./advanced-attributes.html#adv-attr-exportReferencesGraph). + A *derivation deep* string context element is a derivation path, and refers to both its outputs and the entire build closure of that derivation: + all its outputs, all the other derivations the given derivation depends on, and all the outputs of those. + + > **Example** + > + > The best way to illustrate *derivation deep* string contexts is with [`builtins.addDrvOutputDependencies`]. + > Take a regular constant string context element pointing to a derivation, and transform it into a "Derivation deep" string context element. + > + > ```nix + > builtins.getContext + > (builtins.addDrvOutputDependencies + > (builtins.storePath "/nix/store/fvchh9cvcr7kdla6n860hshchsba305w-hello-2.12.drv")) + > ``` + > evaluates to + > ```nix + > { + > "/nix/store/fvchh9cvcr7kdla6n860hshchsba305w-hello-2.12.drv" = { + > allOutputs = true; + > }; + > } + > ``` + + [`builtins.addDrvOutputDependencies`]: ./builtins.md#builtins-addDrvOutputDependencies + [`builtins.unsafeDiscardOutputDependency`]: ./builtins.md#builtins-unsafeDiscardOutputDependency + +## Inspecting string contexts + +Most basically, [`builtins.hasContext`] will tell whether a string has a non-empty context. + +When more granular information is needed, [`builtins.getContext`] can be used. +It creates an [attribute set] representing the string context, which can be inspected as usual. + +[`builtins.hasContext`]: ./builtins.md#builtins-hasContext +[`builtins.getContext`]: ./builtins.md#builtins-getContext +[attribute set]: ./types.md#attribute-set + +## Clearing string contexts + +[`buitins.unsafeDiscardStringContext`](./builtins.md#builtins-unsafeDiscardStringContext) will make a copy of a string, but with an empty string context. +The returned string can be used in more ways, e.g. by operators that require the string context to be empty. +The requirement to explicitly discard the string context in such use cases helps ensure that string context elements are not lost by mistake. +The "unsafe" marker is only there to remind that Nix normally guarantees that dependencies are tracked, whereas the returned string has lost them. + +## Constructing string contexts + +[`builtins.appendContext`] will create a copy of a string, but with additional string context elements. +The context is specified explicitly by an [attribute set] in the format that [`builtins.hasContext`] produces. +A string with arbitrary contexts can be made like this: + +1. Create a string with the desired string context elements. + (The contents of the string do not matter.) +2. Dump its context with [`builtins.getContext`]. +3. Combine it with a base string and repeated [`builtins.appendContext`] calls. + +[`builtins.appendContext`]: ./builtins.md#builtins-appendContext diff --git a/doc/manual/src/language/string-interpolation.md b/doc/manual/src/language/string-interpolation.md index 7d81c2020..1778bdfa0 100644 --- a/doc/manual/src/language/string-interpolation.md +++ b/doc/manual/src/language/string-interpolation.md @@ -4,9 +4,9 @@ String interpolation is a language feature where a [string], [path], or [attribu Such a construct is called *interpolated string*, and the expression inside is an [interpolated expression](#interpolated-expression). -[string]: ./values.md#type-string -[path]: ./values.md#type-path -[attribute set]: ./values.md#attribute-set +[string]: ./types.md#type-string +[path]: ./types.md#type-path +[attribute set]: ./types.md#attribute-set ## Examples @@ -20,7 +20,7 @@ Rather than writing (where `freetype` is a [derivation]), you can instead write -[derivation]: ../glossary.md#gloss-derivation +[derivation]: @docroot@/glossary.md#gloss-derivation ```nix "--with-freetype2-library=${freetype}/lib" @@ -43,6 +43,47 @@ configureFlags = " Note that Nix expressions and strings can be arbitrarily nested; in this case the outer string contains various interpolated expressions that themselves contain strings (e.g., `"-thread"`), some of which in turn contain interpolated expressions (e.g., `${mesa}`). +To write a literal `${` in an regular string, escape it with a backslash (`\`). + +> **Example** +> +> ```nix +> "echo \${PATH}" +> ``` +> +> "echo ${PATH}" + +To write a literal `${` in an indented string, escape it with two single quotes (`''`). + +> **Example** +> +> ```nix +> '' +> echo ''${PATH} +> '' +> ``` +> +> "echo ${PATH}\n" + +`$${` can be written literally in any string. + +> **Example** +> +> In Make, `$` in file names or recipes is represented as `$$`, see [GNU `make`: Basics of Variable Reference](https://www.gnu.org/software/make/manual/html_node/Reference.html#Basics-of-Variable-References). +> This can be expressed directly in the Nix language strings: +> +> ```nix +> '' +> MAKEVAR = Hello +> all: +> @export BASHVAR=world; echo $(MAKEVAR) $${BASHVAR} +> '' +> ``` +> +> "MAKEVAR = Hello\nall:\n\t@export BASHVAR=world; echo $(MAKEVAR) $\${BASHVAR}\n" + +See the [documentation on strings][string] for details. + ### Path Rather than writing @@ -107,9 +148,9 @@ An expression that is interpolated must evaluate to one of the following: A string interpolates to itself. -A path in an interpolated expression is first copied into the Nix store, and the resulting string is the [store path] of the newly created [store object](../glossary.md#gloss-store-object). +A path in an interpolated expression is first copied into the Nix store, and the resulting string is the [store path] of the newly created [store object](@docroot@/store/store-object.md). -[store path]: ../glossary.md#gloss-store-path +[store path]: @docroot@/store/store-path.md > **Example** > diff --git a/doc/manual/src/language/syntax.md b/doc/manual/src/language/syntax.md new file mode 100644 index 000000000..6108bacd6 --- /dev/null +++ b/doc/manual/src/language/syntax.md @@ -0,0 +1,866 @@ +# Language Constructs + +This section covers syntax and semantics of the Nix language. + +## Basic Literals + +### String {#string-literal} + + *Strings* can be written in three ways. + + The most common way is to enclose the string between double quotes, e.g., `"foo bar"`. + Strings can span multiple lines. + The results of other expressions can be included into a string by enclosing them in `${ }`, a feature known as [string interpolation]. + + [string interpolation]: ./string-interpolation.md + + The following must be escaped to represent them within a string, by prefixing with a backslash (`\`): + + - Double quote (`"`) + + > **Example** + > + > ```nix + > "\"" + > ``` + > + > "\"" + + - Backslash (`\`) + + > **Example** + > + > ```nix + > "\\" + > ``` + > + > "\\" + + - Dollar sign followed by an opening curly bracket (`${`) – "dollar-curly" + + > **Example** + > + > ```nix + > "\${" + > ``` + > + > "\${" + + The newline, carriage return, and tab characters can be written as `\n`, `\r` and `\t`, respectively. + + A "double-dollar-curly" (`$${`) can be written literally. + + > **Example** + > + > ```nix + > "$${" + > ``` + > + > "$\${" + + String values are output on the terminal with Nix-specific escaping. + Strings written to files will contain the characters encoded by the escaping. + + The second way to write string literals is as an *indented string*, which is enclosed between pairs of *double single-quotes* (`''`), like so: + + ```nix + '' + This is the first line. + This is the second line. + This is the third line. + '' + ``` + + This kind of string literal intelligently strips indentation from + the start of each line. To be precise, it strips from each line a + number of spaces equal to the minimal indentation of the string as a + whole (disregarding the indentation of empty lines). For instance, + the first and second line are indented two spaces, while the third + line is indented four spaces. Thus, two spaces are stripped from + each line, so the resulting string is + + ```nix + "This is the first line.\nThis is the second line.\n This is the third line.\n" + ``` + + > **Note** + > + > Whitespace and newline following the opening `''` is ignored if there is no non-whitespace text on the initial line. + + > **Warning** + > + > Prefixed tab characters are not stripped. + > + > > **Example** + > > + > > The following indented string is prefixed with tabs: + > > + > > '' + > > all: + > > @echo hello + > > '' + > > + > > "\tall:\n\t\t@echo hello\n" + + Indented strings support [string interpolation]. + + The following must be escaped to represent them in an indented string: + + - `$` is escaped by prefixing it with two single quotes (`''`) + + > **Example** + > + > ```nix + > '' + > ''$ + > '' + > ``` + > + > "$\n" + + - `''` is escaped by prefixing it with one single quote (`'`) + + > **Example** + > + > ```nix + > '' + > ''' + > '' + > ``` + > + > "''\n" + + These special characters are escaped as follows: + - Linefeed (`\n`): `''\n` + - Carriage return (`\r`): `''\r` + - Tab (`\t`): `''\t` + + `''\` escapes any other character. + + A "double-dollar-curly" (`$${`) can be written literally. + + > **Example** + > + > ```nix + > '' + > $${ + > '' + > ``` + > + > "$\${\n" + + Indented strings are primarily useful in that they allow multi-line + string literals to follow the indentation of the enclosing Nix + expression, and that less escaping is typically necessary for + strings representing languages such as shell scripts and + configuration files because `''` is much less common than `"`. + Example: + + ```nix + stdenv.mkDerivation { + ... + postInstall = + '' + mkdir $out/bin $out/etc + cp foo $out/bin + echo "Hello World" > $out/etc/foo.conf + ${if enableBar then "cp bar $out/bin" else ""} + ''; + ... + } + ``` + + Finally, as a convenience, *URIs* as defined in appendix B of + [RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as + is*, without quotes. For instance, the string + `"http://example.org/foo.tar.bz2"` can also be written as + `http://example.org/foo.tar.bz2`. + +### Number {#number-literal} + + + + Numbers, which can be *integers* (like `123`) or *floating point* + (like `123.43` or `.27e13`). + + See [arithmetic] and [comparison] operators for semantics. + + [arithmetic]: ./operators.md#arithmetic + [comparison]: ./operators.md#comparison + +### Path {#path-literal} + + *Paths* can be expressed by path literals such as `./builder.sh`. + + A path literal must contain at least one slash to be recognised as such. + For instance, `builder.sh` is not a path: + it's parsed as an expression that selects the attribute `sh` from the variable `builder`. + + Path literals are resolved relative to their [base directory](@docroot@/glossary.md#gloss-base-directory). + Path literals may also refer to absolute paths by starting with a slash. + + > **Note** + > + > Absolute paths make expressions less portable. + > In the case where a function translates a path literal into an absolute path string for a configuration file, it is recommended to write a string literal instead. + > This avoids some confusion about whether files at that location will be used during evaluation. + > It also avoids unintentional situations where some function might try to copy everything at the location into the store. + + If the first component of a path is a `~`, it is interpreted such that the rest of the path were relative to the user's home directory. + For example, `~/foo` would be equivalent to `/home/edolstra/foo` for a user whose home directory is `/home/edolstra`. + Path literals that start with `~` are not allowed in [pure](@docroot@/command-ref/conf-file.md#conf-pure-eval) evaluation. + + Path literals can also include [string interpolation], besides being [interpolated into other expressions]. + + [interpolated into other expressions]: ./string-interpolation.md#interpolated-expressions + + At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path. + + `a.${foo}/b.${bar}` is a syntactically valid number division operation. + `./a.${foo}/b.${bar}` is a path. + + [Lookup path](./constructs/lookup-path.md) literals such as `` also resolve to path values. + +## List {#list-literal} + +Lists are formed by enclosing a whitespace-separated list of values +between square brackets. For example, + +```nix +[ 123 ./foo.nix "abc" (f { x = y; }) ] +``` + +defines a list of four elements, the last being the result of a call to +the function `f`. Note that function calls have to be enclosed in +parentheses. If they had been omitted, e.g., + +```nix +[ 123 ./foo.nix "abc" f { x = y; } ] +``` + +the result would be a list of five elements, the fourth one being a +function and the fifth being a set. + +Note that lists are only lazy in values, and they are strict in length. + +Elements in a list can be accessed using [`builtins.elemAt`](./builtins.md#builtins-elemAt). + +## Attribute Set {#attrs-literal} + +An attribute set is a collection of name-value-pairs called *attributes*. + +Attribute sets are written enclosed in curly brackets (`{ }`). +Attribute names and attribute values are separated by an equal sign (`=`). +Each value can be an arbitrary expression, terminated by a semicolon (`;`) + +An attribute name is a string without context, and is denoted by a [name] (an [identifier](./identifiers.md#identifiers) or [string literal](#string-literal)). + +[name]: ./identifiers.md#names + +> **Syntax** +> +> *attrset* → `{` { *name* `=` *expr* `;` } `}` + +Attributes can appear in any order. +An attribute name may only occur once in each attribute set. + +> **Example** +> +> This defines an attribute set with attributes named: +> - `x` with the value `123`, an integer +> - `text` with the value `"Hello"`, a string +> - `y` where the value is the result of applying the function `f` to the attribute set `{ bla = 456; }` +> +> ```nix +> { +> x = 123; +> text = "Hello"; +> y = f { bla = 456; }; +> } +> ``` + +Attributes in nested attribute sets can be written using *attribute paths*. + +> **Syntax** +> +> *attrset* → `{` { *attrpath* `=` *expr* `;` } `}` + +An attribute path is a dot-separated list of [names][name]. + +> **Syntax** +> +> *attrpath* = *name* { `.` *name* } + + + +> **Example** +> +> ```nix +> { a.b.c = 1; a.b.d = 2; } +> ``` +> +> { +> a = { +> b = { +> c = 1; +> d = 2; +> }; +> }; +> } + +Attribute names can also be set implicitly by using the [`inherit` keyword](#inheriting-attributes). + +> **Example** +> +> ```nix +> { inherit (builtins) true; } +> ``` +> +> { true = true; } + +Attributes can be accessed with the [`.` operator](./operators.md#attribute-selection). + +Example: + +```nix +{ a = "Foo"; b = "Bar"; }.a +``` + +This evaluates to `"Foo"`. + +It is possible to provide a default value in an attribute selection using the `or` keyword. + +Example: + +```nix +{ a = "Foo"; b = "Bar"; }.c or "Xyzzy" +``` + +```nix +{ a = "Foo"; b = "Bar"; }.c.d.e.f.g or "Xyzzy" +``` + +will both evaluate to `"Xyzzy"` because there is no `c` attribute in the set. + +You can use arbitrary double-quoted strings as attribute names: + +```nix +{ "$!@#?" = 123; }."$!@#?" +``` + +```nix +let bar = "bar"; in +{ "foo ${bar}" = 123; }."foo ${bar}" +``` + +Both will evaluate to `123`. + +Attribute names support [string interpolation]: + +```nix +let bar = "foo"; in +{ foo = 123; }.${bar} +``` + +```nix +let bar = "foo"; in +{ ${bar} = 123; }.foo +``` + +Both will evaluate to `123`. + +In the special case where an attribute name inside of a set declaration +evaluates to `null` (which is normally an error, as `null` cannot be coerced to +a string), that attribute is simply not added to the set: + +```nix +{ ${if foo then "bar" else null} = true; } +``` + +This will evaluate to `{}` if `foo` evaluates to `false`. + +A set that has a `__functor` attribute whose value is callable (i.e. is +itself a function or a set with a `__functor` attribute whose value is +callable) can be applied as if it were a function, with the set itself +passed in first , e.g., + +```nix +let add = { __functor = self: x: x + self.x; }; + inc = add // { x = 1; }; +in inc 1 +``` + +evaluates to `2`. This can be used to attach metadata to a function +without the caller needing to treat it specially, or to implement a form +of object-oriented programming, for example. + +## Recursive sets + +Recursive sets are like normal [attribute sets](./types.md#attribute-set), but the attributes can refer to each other. + +> *rec-attrset* = `rec {` [ *name* `=` *expr* `;` `]`... `}` + +Example: + +```nix +rec { + x = y; + y = 123; +}.x +``` + +This evaluates to `123`. + +Note that without `rec` the binding `x = y;` would +refer to the variable `y` in the surrounding scope, if one exists, and +would be invalid if no such variable exists. That is, in a normal +(non-recursive) set, attributes are not added to the lexical scope; in a +recursive set, they are. + +Recursive sets of course introduce the danger of infinite recursion. For +example, the expression + +```nix +rec { + x = y; + y = x; +}.x +``` + +will crash with an `infinite recursion encountered` error message. + +## Let-expressions + +A let-expression allows you to define local variables for an expression. + +> *let-in* = `let` [ *identifier* = *expr* ]... `in` *expr* + +Example: + +```nix +let + x = "foo"; + y = "bar"; +in x + y +``` + +This evaluates to `"foobar"`. + +## Inheriting attributes + +When defining an [attribute set](./types.md#attribute-set) or in a [let-expression](#let-expressions) it is often convenient to copy variables from the surrounding lexical scope (e.g., when you want to propagate attributes). +This can be shortened using the `inherit` keyword. + +Example: + +```nix +let x = 123; in +{ + inherit x; + y = 456; +} +``` + +is equivalent to + +```nix +let x = 123; in +{ + x = x; + y = 456; +} +``` + +and both evaluate to `{ x = 123; y = 456; }`. + +> **Note** +> +> This works because `x` is added to the lexical scope by the `let` construct. + +It is also possible to inherit attributes from another attribute set. + +Example: + +In this fragment from `all-packages.nix`, + +```nix +graphviz = (import ../tools/graphics/graphviz) { + inherit fetchurl stdenv libpng libjpeg expat x11 yacc; + inherit (xorg) libXaw; +}; + +xorg = { + libX11 = ...; + libXaw = ...; + ... +} + +libpng = ...; +libjpg = ...; +... +``` + +the set used in the function call to the function defined in +`../tools/graphics/graphviz` inherits a number of variables from the +surrounding scope (`fetchurl` ... `yacc`), but also inherits `libXaw` +(the X Athena Widgets) from the `xorg` set. + +Summarizing the fragment + +```nix +... +inherit x y z; +inherit (src-set) a b c; +... +``` + +is equivalent to + +```nix +... +x = x; y = y; z = z; +a = src-set.a; b = src-set.b; c = src-set.c; +... +``` + +when used while defining local variables in a let-expression or while +defining a set. + +In a `let` expression, `inherit` can be used to selectively bring specific attributes of a set into scope. For example + + +```nix +let + x = { a = 1; b = 2; }; + inherit (builtins) attrNames; +in +{ + names = attrNames x; +} +``` + +is equivalent to + +```nix +let + x = { a = 1; b = 2; }; +in +{ + names = builtins.attrNames x; +} +``` + +both evaluate to `{ names = [ "a" "b" ]; }`. + +## Functions + +Functions have the following form: + +```nix +pattern: body +``` + +The pattern specifies what the argument of the function must look like, +and binds variables in the body to (parts of) the argument. There are +three kinds of patterns: + + - If a pattern is a single identifier, then the function matches any + argument. Example: + + ```nix + let negate = x: !x; + concat = x: y: x + y; + in if negate true then concat "foo" "bar" else "" + ``` + + Note that `concat` is a function that takes one argument and returns + a function that takes another argument. This allows partial + parameterisation (i.e., only filling some of the arguments of a + function); e.g., + + ```nix + map (concat "foo") [ "bar" "bla" "abc" ] + ``` + + evaluates to `[ "foobar" "foobla" "fooabc" ]`. + + - A *set pattern* of the form `{ name1, name2, …, nameN }` matches a + set containing the listed attributes, and binds the values of those + attributes to variables in the function body. For example, the + function + + ```nix + { x, y, z }: z + y + x + ``` + + can only be called with a set containing exactly the attributes `x`, + `y` and `z`. No other attributes are allowed. If you want to allow + additional arguments, you can use an ellipsis (`...`): + + ```nix + { x, y, z, ... }: z + y + x + ``` + + This works on any set that contains at least the three named + attributes. + + It is possible to provide *default values* for attributes, in + which case they are allowed to be missing. A default value is + specified by writing `name ? e`, where *e* is an arbitrary + expression. For example, + + ```nix + { x, y ? "foo", z ? "bar" }: z + y + x + ``` + + specifies a function that only requires an attribute named `x`, but + optionally accepts `y` and `z`. + + - An `@`-pattern provides a means of referring to the whole value + being matched: + + ```nix + args@{ x, y, z, ... }: z + y + x + args.a + ``` + + but can also be written as: + + ```nix + { x, y, z, ... } @ args: z + y + x + args.a + ``` + + Here `args` is bound to the argument *as passed*, which is further + matched against the pattern `{ x, y, z, ... }`. + The `@`-pattern makes mainly sense with an ellipsis(`...`) as + you can access attribute names as `a`, using `args.a`, which was + given as an additional attribute to the function. + + > **Warning** + > + > `args@` binds the name `args` to the attribute set that is passed to the function. + > In particular, `args` does *not* include any default values specified with `?` in the function's set pattern. + > + > For instance + > + > ```nix + > let + > f = args@{ a ? 23, ... }: [ a args ]; + > in + > f {} + > ``` + > + > is equivalent to + > + > ```nix + > let + > f = args @ { ... }: [ (args.a or 23) args ]; + > in + > f {} + > ``` + > + > and both expressions will evaluate to: + > + > ```nix + > [ 23 {} ] + > ``` + +Note that functions do not have names. If you want to give them a name, +you can bind them to an attribute, e.g., + +```nix +let concat = { x, y }: x + y; +in concat { x = "foo"; y = "bar"; } +``` + +## Conditionals + +Conditionals look like this: + +```nix +if e1 then e2 else e3 +``` + +where *e1* is an expression that should evaluate to a Boolean value +(`true` or `false`). + +## Assertions + +Assertions are generally used to check that certain requirements on or +between features and dependencies hold. They look like this: + +```nix +assert e1; e2 +``` + +where *e1* is an expression that should evaluate to a Boolean value. If +it evaluates to `true`, *e2* is returned; otherwise expression +evaluation is aborted and a backtrace is printed. + +Here is a Nix expression for the Subversion package that shows how +assertions can be used:. + +```nix +{ localServer ? false +, httpServer ? false +, sslSupport ? false +, pythonBindings ? false +, javaSwigBindings ? false +, javahlBindings ? false +, stdenv, fetchurl +, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null +}: + +assert localServer -> db4 != null; â‘ +assert httpServer -> httpd != null && httpd.expat == expat; â‘¡ +assert sslSupport -> openssl != null && (httpServer -> httpd.openssl == openssl); â‘¢ +assert pythonBindings -> swig != null && swig.pythonSupport; +assert javaSwigBindings -> swig != null && swig.javaSupport; +assert javahlBindings -> j2sdk != null; + +stdenv.mkDerivation { + name = "subversion-1.1.1"; + ... + openssl = if sslSupport then openssl else null; â‘£ + ... +} +``` + +The points of interest are: + +1. This assertion states that if Subversion is to have support for + local repositories, then Berkeley DB is needed. So if the Subversion + function is called with the `localServer` argument set to `true` but + the `db4` argument set to `null`, then the evaluation fails. + + Note that `->` is the [logical + implication](https://en.wikipedia.org/wiki/Truth_table#Logical_implication) + Boolean operation. + +2. This is a more subtle condition: if Subversion is built with Apache + (`httpServer`) support, then the Expat library (an XML library) used + by Subversion should be same as the one used by Apache. This is + because in this configuration Subversion code ends up being linked + with Apache code, and if the Expat libraries do not match, a build- + or runtime link error or incompatibility might occur. + +3. This assertion says that in order for Subversion to have SSL support + (so that it can access `https` URLs), an OpenSSL library must be + passed. Additionally, it says that *if* Apache support is enabled, + then Apache's OpenSSL should match Subversion's. (Note that if + Apache support is not enabled, we don't care about Apache's + OpenSSL.) + +4. The conditional here is not really related to assertions, but is + worth pointing out: it ensures that if SSL support is disabled, then + the Subversion derivation is not dependent on OpenSSL, even if a + non-`null` value was passed. This prevents an unnecessary rebuild of + Subversion if OpenSSL changes. + +## With-expressions + +A *with-expression*, + +```nix +with e1; e2 +``` + +introduces the set *e1* into the lexical scope of the expression *e2*. +For instance, + +```nix +let as = { x = "foo"; y = "bar"; }; +in with as; x + y +``` + +evaluates to `"foobar"` since the `with` adds the `x` and `y` attributes +of `as` to the lexical scope in the expression `x + y`. The most common +use of `with` is in conjunction with the `import` function. E.g., + +```nix +with (import ./definitions.nix); ... +``` + +makes all attributes defined in the file `definitions.nix` available as +if they were defined locally in a `let`-expression. + +The bindings introduced by `with` do not shadow bindings introduced by +other means, e.g. + +```nix +let a = 3; in with { a = 1; }; let a = 4; in with { a = 2; }; ... +``` + +establishes the same scope as + +```nix +let a = 1; in let a = 2; in let a = 3; in let a = 4; in ... +``` + +Variables coming from outer `with` expressions *are* shadowed: + +```nix +with { a = "outer"; }; +with { a = "inner"; }; +a +``` + +Does evaluate to `"inner"`. + +## Comments + +- Inline comments start with `#` and run until the end of the line. + + > **Example** + > + > ```nix + > # A number + > 2 # Equals 1 + 1 + > ``` + > + > ```console + > 2 + > ``` + +- Block comments start with `/*` and run until the next occurrence of `*/`. + + > **Example** + > + > ```nix + > /* + > Block comments + > can span multiple lines. + > */ "hello" + > ``` + > + > ```console + > "hello" + > ``` + + This means that block comments cannot be nested. + + > **Example** + > + > ```nix + > /* /* nope */ */ 1 + > ``` + > + > ```console + > error: syntax error, unexpected '*' + > + > at «string»:1:15: + > + > 1| /* /* nope */ * + > | ^ + > ``` + + Consider escaping nested comments and unescaping them in post-processing. + + > **Example** + > + > ```nix + > /* /* nested *\/ */ 1 + > ``` + > + > ```console + > 1 + > ``` diff --git a/doc/manual/src/language/types.md b/doc/manual/src/language/types.md new file mode 100644 index 000000000..229756e6b --- /dev/null +++ b/doc/manual/src/language/types.md @@ -0,0 +1,120 @@ +# Data Types + +Every value in the Nix language has one of the following types: + +* [Integer](#type-int) +* [Float](#type-float) +* [Boolean](#type-bool) +* [String](#type-string) +* [Path](#type-path) +* [Null](#type-null) +* [Attribute set](#type-attrs) +* [List](#type-list) +* [Function](#type-function) +* [External](#type-external) + +## Primitives + +### Integer {#type-int} + +An _integer_ in the Nix language is a signed 64-bit integer. + +Non-negative integers can be expressed as [integer literals](syntax.md#number-literal). +Negative integers are created with the [arithmetic negation operator](./operators.md#arithmetic). +The function [`builtins.isInt`](builtins.md#builtins-isInt) can be used to determine if a value is an integer. + +### Float {#type-float} + +A _float_ in the Nix language is a 64-bit [IEEE 754](https://en.wikipedia.org/wiki/IEEE_754) floating-point number. + +Most non-negative floats can be expressed as [float literals](syntax.md#number-literal). +Negative floats are created with the [arithmetic negation operator](./operators.md#arithmetic). +The function [`builtins.isFloat`](builtins.md#builtins-isFloat) can be used to determine if a value is a float. + +### Boolean {#type-bool} + +A _boolean_ in the Nix language is one of _true_ or _false_. + + + +These values are available as attributes of [`builtins`](builtins.md#builtins-builtins) as [`builtins.true`](builtins.md#builtins-true) and [`builtins.false`](builtins.md#builtins-false). +The function [`builtins.isBool`](builtins.md#builtins-isBool) can be used to determine if a value is a boolean. + +### String {#type-string} + +A _string_ in the Nix language is an immutable, finite-length sequence of bytes, along with a [string context](string-context.md). +Nix does not assume or support working natively with character encodings. + +String values without string context can be expressed as [string literals](syntax.md#string-literal). +The function [`builtins.isString`](builtins.md#builtins-isString) can be used to determine if a value is a string. + +### Path {#type-path} + +A _path_ in the Nix language is an immutable, finite-length sequence of bytes starting with `/`, representing a POSIX-style, canonical file system path. +Path values are distinct from string values, even if they contain the same sequence of bytes. +Operations that produce paths will simplify the result as the standard C function [`realpath`] would, except that there is no symbolic link resolution. + +[`realpath`]: https://pubs.opengroup.org/onlinepubs/9699919799/functions/realpath.html + +Paths are suitable for referring to local files, and are often preferable over strings. +- Path values do not contain trailing or duplicate slashes, `.`, or `..`. +- Relative path literals are automatically resolved relative to their [base directory]. +- Tooling can recognize path literals and provide additional features, such as autocompletion, refactoring automation and jump-to-file. + +[base directory]: @docroot@/glossary.md#gloss-base-directory + +A file is not required to exist at a given path in order for that path value to be valid, but a path that is converted to a string with [string interpolation] or [string-and-path concatenation] must resolve to a readable file or directory which will be copied into the Nix store. +For instance, evaluating `"${./foo.txt}"` will cause `foo.txt` from the same directory to be copied into the Nix store and result in the string `"/nix/store/-foo.txt"`. +Operations such as [`import`] can also expect a path to resolve to a readable file or directory. + +[string interpolation]: string-interpolation.md#interpolated-expression +[string-and-path concatenation]: operators.md#string-and-path-concatenation +[`import`]: builtins.md#builtins-import + +> **Note** +> +> The Nix language assumes that all input files will remain _unchanged_ while evaluating a Nix expression. +> For example, assume you used a file path in an interpolated string during a `nix repl` session. +> Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new [store path], since Nix might not re-read the file contents. +> Use `:r` to reset the repl as needed. + +[store path]: @docroot@/store/store-path.md + +Path values can be expressed as [path literals](syntax.md#path-literal). +The function [`builtins.isPath`](builtins.md#builtins-isPath) can be used to determine if a value is a path. + +### Null {#type-null} + +There is a single value of type _null_ in the Nix language. + + + +This value is available as an attribute on the [`builtins`](builtins.md#builtins-builtins) attribute set as [`builtins.null`](builtins.md#builtins-null). + +## Compound values + +### Attribute set {#type-attrs} + + + +An attribute set can be constructed with an [attribute set literal](syntax.md#attrs-literal). +The function [`builtins.isAttrs`](builtins.md#builtins-isAttrs) can be used to determine if a value is an attribute set. + +### List {#type-list} + + + +A list can be constructed with a [list literal](syntax.md#list-literal). +The function [`builtins.isList`](builtins.md#builtins-isList) can be used to determine if a value is a list. + +## Function {#type-function} + + + +A function can be constructed with a [function expression](syntax.md#functions). +The function [`builtins.isFunction`](builtins.md#builtins-isFunction) can be used to determine if a value is a function. + +## External {#type-external} + +An _external_ value is an opaque value created by a Nix [plugin](../command-ref/conf-file.md#conf-plugin-files). +Such a value can be substituted in Nix expressions but only created and used by plugin code. diff --git a/doc/manual/src/language/values.md b/doc/manual/src/language/values.md deleted file mode 100644 index 74ffc7070..000000000 --- a/doc/manual/src/language/values.md +++ /dev/null @@ -1,269 +0,0 @@ -# Data Types - -## Primitives - -- String - - *Strings* can be written in three ways. - - The most common way is to enclose the string between double quotes, - e.g., `"foo bar"`. Strings can span multiple lines. The special - characters `"` and `\` and the character sequence `${` must be - escaped by prefixing them with a backslash (`\`). Newlines, carriage - returns and tabs can be written as `\n`, `\r` and `\t`, - respectively. - - You can include the results of other expressions into a string by enclosing them in `${ }`, a feature known as [string interpolation]. - - [string interpolation]: ./string-interpolation.md - - The second way to write string literals is as an *indented string*, - which is enclosed between pairs of *double single-quotes*, like so: - - ```nix - '' - This is the first line. - This is the second line. - This is the third line. - '' - ``` - - This kind of string literal intelligently strips indentation from - the start of each line. To be precise, it strips from each line a - number of spaces equal to the minimal indentation of the string as a - whole (disregarding the indentation of empty lines). For instance, - the first and second line are indented two spaces, while the third - line is indented four spaces. Thus, two spaces are stripped from - each line, so the resulting string is - - ```nix - "This is the first line.\nThis is the second line.\n This is the third line.\n" - ``` - - Note that the whitespace and newline following the opening `''` is - ignored if there is no non-whitespace text on the initial line. - - Indented strings support [string interpolation]. - - Since `${` and `''` have special meaning in indented strings, you - need a way to quote them. `$` can be escaped by prefixing it with - `''` (that is, two single quotes), i.e., `''$`. `''` can be escaped - by prefixing it with `'`, i.e., `'''`. `$` removes any special - meaning from the following `$`. Linefeed, carriage-return and tab - characters can be written as `''\n`, `''\r`, `''\t`, and `''\` - escapes any other character. - - Indented strings are primarily useful in that they allow multi-line - string literals to follow the indentation of the enclosing Nix - expression, and that less escaping is typically necessary for - strings representing languages such as shell scripts and - configuration files because `''` is much less common than `"`. - Example: - - ```nix - stdenv.mkDerivation { - ... - postInstall = - '' - mkdir $out/bin $out/etc - cp foo $out/bin - echo "Hello World" > $out/etc/foo.conf - ${if enableBar then "cp bar $out/bin" else ""} - ''; - ... - } - ``` - - Finally, as a convenience, *URIs* as defined in appendix B of - [RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as - is*, without quotes. For instance, the string - `"http://example.org/foo.tar.bz2"` can also be written as - `http://example.org/foo.tar.bz2`. - -- Number - - Numbers, which can be *integers* (like `123`) or *floating point* - (like `123.43` or `.27e13`). - - See [arithmetic] and [comparison] operators for semantics. - - [arithmetic]: ./operators.md#arithmetic - [comparison]: ./operators.md#comparison - -- Path - - *Paths*, e.g., `/bin/sh` or `./builder.sh`. A path must contain at - least one slash to be recognised as such. For instance, `builder.sh` - is not a path: it's parsed as an expression that selects the - attribute `sh` from the variable `builder`. If the file name is - relative, i.e., if it does not begin with a slash, it is made - absolute at parse time relative to the directory of the Nix - expression that contained it. For instance, if a Nix expression in - `/foo/bar/bla.nix` refers to `../xyzzy/fnord.nix`, the absolute path - is `/foo/xyzzy/fnord.nix`. - - If the first component of a path is a `~`, it is interpreted as if - the rest of the path were relative to the user's home directory. - e.g. `~/foo` would be equivalent to `/home/edolstra/foo` for a user - whose home directory is `/home/edolstra`. - - For instance, evaluating `"${./foo.txt}"` will cause `foo.txt` in the current directory to be copied into the Nix store and result in the string `"/nix/store/-foo.txt"`. - - Note that the Nix language assumes that all input files will remain _unchanged_ while evaluating a Nix expression. - For example, assume you used a file path in an interpolated string during a `nix repl` session. - Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new [store path], since Nix might not re-read the file contents. - - [store path]: ../glossary.md#gloss-store-path - - Paths can include [string interpolation] and can themselves be [interpolated in other expressions]. - - [interpolated in other expressions]: ./string-interpolation.md#interpolated-expressions - - At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path. - - `a.${foo}/b.${bar}` is a syntactically valid division operation. - `./a.${foo}/b.${bar}` is a path. - - [Lookup paths](./constructs/lookup-path.md) such as `` resolve to path values. - -- Boolean - - *Booleans* with values `true` and `false`. - -- Null - - The null value, denoted as `null`. - -## List - -Lists are formed by enclosing a whitespace-separated list of values -between square brackets. For example, - -```nix -[ 123 ./foo.nix "abc" (f { x = y; }) ] -``` - -defines a list of four elements, the last being the result of a call to -the function `f`. Note that function calls have to be enclosed in -parentheses. If they had been omitted, e.g., - -```nix -[ 123 ./foo.nix "abc" f { x = y; } ] -``` - -the result would be a list of five elements, the fourth one being a -function and the fifth being a set. - -Note that lists are only lazy in values, and they are strict in length. - -Elements in a list can be accessed using [`builtins.elemAt`](./builtins.md#builtins-elemAt). - -## Attribute Set - -An attribute set is a collection of name-value-pairs (called *attributes*) enclosed in curly brackets (`{ }`). - -An attribute name can be an identifier or a [string](#string). -An identifier must start with a letter (`a-z`, `A-Z`) or underscore (`_`), and can otherwise contain letters (`a-z`, `A-Z`), numbers (`0-9`), underscores (`_`), apostrophes (`'`), or dashes (`-`). - -> **Syntax** -> -> *name* = *identifier* | *string* \ -> *identifier* ~ `[a-zA-Z_][a-zA-Z0-9_'-]*` - -Names and values are separated by an equal sign (`=`). -Each value is an arbitrary expression terminated by a semicolon (`;`). - -> **Syntax** -> -> *attrset* = `{` [ *name* `=` *expr* `;` ]... `}` - -Attributes can appear in any order. -An attribute name may only occur once. - -Example: - -```nix -{ - x = 123; - text = "Hello"; - y = f { bla = 456; }; -} -``` - -This defines a set with attributes named `x`, `text`, `y`. - -Attributes can be accessed with the [`.` operator](./operators.md#attribute-selection). - -Example: - -```nix -{ a = "Foo"; b = "Bar"; }.a -``` - -This evaluates to `"Foo"`. - -It is possible to provide a default value in an attribute selection using the `or` keyword. - -Example: - -```nix -{ a = "Foo"; b = "Bar"; }.c or "Xyzzy" -``` - -```nix -{ a = "Foo"; b = "Bar"; }.c.d.e.f.g or "Xyzzy" -``` - -will both evaluate to `"Xyzzy"` because there is no `c` attribute in the set. - -You can use arbitrary double-quoted strings as attribute names: - -```nix -{ "$!@#?" = 123; }."$!@#?" -``` - -```nix -let bar = "bar"; in -{ "foo ${bar}" = 123; }."foo ${bar}" -``` - -Both will evaluate to `123`. - -Attribute names support [string interpolation]: - -```nix -let bar = "foo"; in -{ foo = 123; }.${bar} -``` - -```nix -let bar = "foo"; in -{ ${bar} = 123; }.foo -``` - -Both will evaluate to `123`. - -In the special case where an attribute name inside of a set declaration -evaluates to `null` (which is normally an error, as `null` cannot be coerced to -a string), that attribute is simply not added to the set: - -```nix -{ ${if foo then "bar" else null} = true; } -``` - -This will evaluate to `{}` if `foo` evaluates to `false`. - -A set that has a `__functor` attribute whose value is callable (i.e. is -itself a function or a set with a `__functor` attribute whose value is -callable) can be applied as if it were a function, with the set itself -passed in first , e.g., - -```nix -let add = { __functor = self: x: x + self.x; }; - inc = add // { x = 1; }; -in inc 1 -``` - -evaluates to `2`. This can be used to attach metadata to a function -without the caller needing to treat it specially, or to implement a form -of object-oriented programming, for example. diff --git a/doc/manual/src/language/variables.md b/doc/manual/src/language/variables.md new file mode 100644 index 000000000..af6aff8a2 --- /dev/null +++ b/doc/manual/src/language/variables.md @@ -0,0 +1,10 @@ +# Variables + +A *variable* is an [identifier](identifiers.md) used as an expression. + +> **Syntax** +> +> *expression* → *identifier* + +A variable must have the same name as a definition in the [scope](./scope.md) that encloses it. +The value of a variable is the value of the corresponding expression in the enclosing scope. diff --git a/doc/manual/src/package-management/copy-closure.md b/doc/manual/src/package-management/copy-closure.md deleted file mode 100644 index 14326298b..000000000 --- a/doc/manual/src/package-management/copy-closure.md +++ /dev/null @@ -1,34 +0,0 @@ -# Copying Closures via SSH - -The command `nix-copy-closure` copies a Nix store path along with all -its dependencies to or from another machine via the SSH protocol. It -doesn’t copy store paths that are already present on the target machine. -For example, the following command copies Firefox with all its -dependencies: - - $ nix-copy-closure --to alice@itchy.example.org $(type -p firefox) - -See the [manpage for `nix-copy-closure`](../command-ref/nix-copy-closure.md) for details. - -With `nix-store ---export` and `nix-store --import` you can write the closure of a store -path (that is, the path and all its dependencies) to a file, and then -unpack that file into another Nix store. For example, - - $ nix-store --export $(nix-store --query --requisites $(type -p firefox)) > firefox.closure - -writes the closure of Firefox to a file. You can then copy this file to -another machine and install the closure: - - $ nix-store --import < firefox.closure - -Any store paths in the closure that are already present in the target -store are ignored. It is also possible to pipe the export into another -command, e.g. to copy and install a closure directly to/on another -machine: - - $ nix-store --export $(nix-store --query --requisites $(type -p firefox)) | bzip2 | \ - ssh alice@itchy.example.org "bunzip2 | nix-store --import" - -However, `nix-copy-closure` is generally more efficient because it only -copies paths that are not already present in the target Nix store. diff --git a/doc/manual/src/protocols/derivation-aterm.md b/doc/manual/src/protocols/derivation-aterm.md index e58b602a3..1ba757ae0 100644 --- a/doc/manual/src/protocols/derivation-aterm.md +++ b/doc/manual/src/protocols/derivation-aterm.md @@ -14,6 +14,6 @@ Derivations are serialised in one of the following formats: DrvWithVersion(, ...) ``` - The only `version-string`s that are in use today are for [experimental features](@docroot@/contributing/experimental-features.md): + The only `version-string`s that are in use today are for [experimental features](@docroot@/development/experimental-features.md): - - `"xp-dyn-drv"` for the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. + - `"xp-dyn-drv"` for the [`dynamic-derivations`](@docroot@/development/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. diff --git a/doc/manual/src/protocols/json/derivation.md b/doc/manual/src/protocols/json/derivation.md index 649d543cc..2f85340d6 100644 --- a/doc/manual/src/protocols/json/derivation.md +++ b/doc/manual/src/protocols/json/derivation.md @@ -3,7 +3,7 @@ > **Warning** > > This JSON format is currently -> [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) +> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) > and subject to change. The JSON serialization of a @@ -18,10 +18,30 @@ is a JSON object with the following fields: Information about the output paths of the derivation. This is a JSON object with one member per output, where the key is the output name and the value is a JSON object with these fields: - * `path`: The output path. + * `path`: + The output path, if it is known in advanced. + Otherwise, `null`. + + + * `method`: + For an output which will be [content addresed], a string representing the [method](@docroot@/store/store-object/content-address.md) of content addressing that is chosen. + Valid method strings are: + + - [`flat`](@docroot@/store/store-object/content-address.md#method-flat) + - [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive) + - [`text`](@docroot@/store/store-object/content-address.md#method-text) + - [`git`](@docroot@/store/store-object/content-address.md#method-git) + + Otherwise, `null`. * `hashAlgo`: - For fixed-output derivations, the hashing algorithm (e.g. `sha256`), optionally prefixed by `r:` if `hash` denotes a NAR hash rather than a flat file hash. + For an output which will be [content addresed], the name of the hash algorithm used. + Valid algorithm strings are: + + - `md5` + - `sha1` + - `sha256` + - `sha512` * `hash`: For fixed-output derivations, the expected content hash in base-16. @@ -32,7 +52,8 @@ is a JSON object with the following fields: > "outputs": { > "out": { > "path": "/nix/store/2543j7c6jn75blc3drf4g5vhb1rhdq29-source", - > "hashAlgo": "r:sha256", + > "method": "nar", + > "hashAlgo": "sha256", > "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62" > } > } diff --git a/doc/manual/src/protocols/json/store-object-info.md b/doc/manual/src/protocols/json/store-object-info.md index ba4ab098f..6b4f48437 100644 --- a/doc/manual/src/protocols/json/store-object-info.md +++ b/doc/manual/src/protocols/json/store-object-info.md @@ -3,7 +3,7 @@ > **Warning** > > This JSON format is currently -> [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) +> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) > and subject to change. Info about a [store object]. @@ -24,41 +24,45 @@ Info about a [store object]. An array of [store paths][store path], possibly including this one. -* `ca` (optional): +* `ca`: - Content address of this store object's file system object, used to compute its store path. + If the store object is [content-addressed], + this is the content address of this store object's file system object, used to compute its store path. + Otherwise (i.e. if it is [input-addressed]), this is `null`. -[store path]: @docroot@/glossary.md#gloss-store-path +[store path]: @docroot@/store/store-path.md [file system object]: @docroot@/store/file-system-object.md -[Nix Archive]: @docroot@/glossary.md#gloss-nar +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive ## Impure fields These are not intrinsic properties of the store object. In other words, the same store object residing in different store could have different values for these properties. -* `deriver` (optional): +* `deriver`: - The path to the [derivation] from which this store object is produced. + If known, the path to the [derivation] from which this store object was produced. + Otherwise `null`. [derivation]: @docroot@/glossary.md#gloss-store-derivation * `registrationTime` (optional): - When this derivation was added to the store. + If known, when this derivation was added to the store. + Otherwise `null`. -* `ultimate` (optional): +* `ultimate`: Whether this store object is trusted because we built it ourselves, rather than substituted a build product from elsewhere. -* `signatures` (optional): +* `signatures`: Signatures claiming that this store object is what it claims to be. Not relevant for [content-addressed] store objects, but useful for [input-addressed] store objects. - [content-addressed]: @docroot@/glossary.md#gloss-content-addressed-store-object - [input-addressed]: @docroot@/glossary.md#gloss-input-addressed-store-object +[content-addressed]: @docroot@/store/store-object/content-address.md +[input-addressed]: @docroot@/glossary.md#gloss-input-addressed-store-object ### `.narinfo` extra fields @@ -83,7 +87,7 @@ This information is not intrinsic to the store object, but about how it is store ## Computed closure fields -These fields are not stored at all, but computed by traverising the other other fields across all the store objects in a [closure]. +These fields are not stored at all, but computed by traversing the other fields across all the store objects in a [closure]. * `closureSize`: diff --git a/doc/manual/src/protocols/nix-archive.md b/doc/manual/src/protocols/nix-archive.md new file mode 100644 index 000000000..640b527f1 --- /dev/null +++ b/doc/manual/src/protocols/nix-archive.md @@ -0,0 +1,43 @@ +# Nix Archive (NAR) format + +This is the complete specification of the [Nix Archive] format. +The Nix Archive format closely follows the abstract specification of a [file system object] tree, +because it is designed to serialize exactly that data structure. + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#nix-archive +[file system object]: @docroot@/store/file-system-object.md + +The format of this specification is close to [Extended Backus–Naur form](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form), with the exception of the `str(..)` function / parameterized rule, which length-prefixes and pads strings. +This makes the resulting binary format easier to parse. + +Regular users do *not* need to know this information. +But for those interested in exactly how Nix works, e.g. if they are reimplementing it, this information can be useful. + +```ebnf +nar = str("nix-archive-1"), nar-obj; + +nar-obj = str("("), nar-obj-inner, str(")"); + +nar-obj-inner + = str("type"), str("regular") regular + | str("type"), str("symlink") symlink + | str("type"), str("directory") directory + ; + +regular = [ str("executable"), str("") ], str("contents"), str(contents); + +symlink = str("target"), str(target); + +(* side condition: directory entries must be ordered by their names *) +directory = { directory-entry }; + +directory-entry = str("entry"), str("("), str("name"), str(name), str("node"), nar-obj, str(")"); +``` + +The `str` function / parameterized rule is defined as follows: + +- `str(s)` = `int(|s|), pad(s);` + +- `int(n)` = the 64-bit little endian representation of the number `n` + +- `pad(s)` = the byte sequence `s`, padded with 0s to a multiple of 8 byte diff --git a/doc/manual/src/protocols/store-path.md b/doc/manual/src/protocols/store-path.md index 565c4fa75..52352d358 100644 --- a/doc/manual/src/protocols/store-path.md +++ b/doc/manual/src/protocols/store-path.md @@ -1,12 +1,14 @@ # Complete Store Path Calculation -This is the complete specification for how store paths are calculated. +This is the complete specification for how [store path]s are calculated. The format of this specification is close to [Extended Backus–Naur form](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form), but must deviate for a few things such as hash functions which we treat as bidirectional for specification purposes. Regular users do *not* need to know this information --- store paths can be treated as black boxes computed from the properties of the store objects they refer to. But for those interested in exactly how Nix works, e.g. if they are reimplementing it, this information can be useful. +[store path](@docroot@/store/store-path.md) + ## Store path proper ```ebnf @@ -34,18 +36,23 @@ where - `type` = one of: - ```ebnf - | "text" ( ":" store-path )* + | "text" { ":" store-path } ``` - for encoded derivations written to the store. + This is for the + ["Text"](@docroot@/store/store-object/content-address.md#method-text) + method of content addressing store objects. The optional trailing store paths are the references of the store object. - ```ebnf - | "source" ( ":" store-path )* + | "source" { ":" store-path } [ ":self" ] ``` - For paths copied to the store and hashed via a [Nix Archive (NAR)] and [SHA-256][sha-256]. - Just like in the text case, we can have the store objects referenced by their paths. + This is for the + ["Nix Archive"](@docroot@/store/store-object/content-address.md#method-nix-archive) + method of content addressing store objects, + if the hash algorithm is [SHA-256]. + Just like in the "Text" case, we can have the store objects referenced by their paths. Additionally, we can have an optional `:self` label to denote self reference. - ```ebnf @@ -53,8 +60,12 @@ where ``` For either the outputs built from derivations, - paths copied to the store hashed that area single file hashed directly, or the via a hash algorithm other than [SHA-256][sha-256]. - (in that case "source" is used; this is only necessary for compatibility). + or content-addressed store objects that are not using one of the two above cases. + To be explicit about the latter, that is currently these methods: + + - ["Flat"](@docroot@/store/store-object/content-address.md#method-flat) + - ["Git"](@docroot@/store/store-object/content-address.md#method-git) + - ["Nix Archive"](@docroot@/store/store-object/content-address.md#method-nix-archive) if the hash algorithm is not [SHA-256]. `id` is the name of the output (usually, "out"). For content-addressed store objects, `id`, is always "out". @@ -113,8 +124,8 @@ where Note that `id` = `"out"`, regardless of the name part of the store path. Also note that NAR + SHA-256 must not use this case, and instead must use the `type` = `"source:" ...` case. -[Nix Archive (NAR)]: @docroot@/glossary.md#gloss-NAR -[sha-256]: https://en.m.wikipedia.org/wiki/SHA-256 +[Nix Archive (NAR)]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive +[SHA-256]: https://en.m.wikipedia.org/wiki/SHA-256 ### Historical Note diff --git a/doc/manual/src/protocols/tarball-fetcher.md b/doc/manual/src/protocols/tarball-fetcher.md index 274fa6d63..5cff05d66 100644 --- a/doc/manual/src/protocols/tarball-fetcher.md +++ b/doc/manual/src/protocols/tarball-fetcher.md @@ -22,7 +22,7 @@ Link: ; rel="immutable" *flakeref* must be a tarball flakeref. It can contain the tarball flake attributes `narHash`, `rev`, `revCount` and `lastModified`. If `narHash` is included, its -value must be the NAR hash of the unpacked tarball (as computed via +value must be the [NAR hash][Nix Archive] of the unpacked tarball (as computed via `nix hash path`). Nix checks the contents of the returned tarball against the `narHash` attribute. The `rev` and `revCount` attributes are useful when the tarball flake is a mirror of a fetcher type that @@ -40,3 +40,31 @@ Link: ///archive/.tar.gz +``` + +> **Example** +> +> +> ```nix +> # flake.nix +> { +> inputs = { +> foo.url = "https://gitea.example.org/some-person/some-flake/archive/main.tar.gz"; +> bar.url = "https://gitea.example.org/some-other-person/other-flake/archive/442793d9ec0584f6a6e82fa253850c8085bb150a.tar.gz"; +> qux = { +> url = "https://forgejo.example.org/another-person/some-non-flake-repo/archive/development.tar.gz"; +> flake = false; +> }; +> }; +> outputs = { foo, bar, qux }: { /* ... */ }; +> } +``` + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive diff --git a/doc/manual/src/quick-start.md b/doc/manual/src/quick-start.md index 75853ced7..9eb7a3265 100644 --- a/doc/manual/src/quick-start.md +++ b/doc/manual/src/quick-start.md @@ -34,7 +34,7 @@ For more in-depth information you are kindly referred to subsequent chapters. lolcat: command not found ``` -1. Search for more packages on to try them out. +1. Search for more packages on [search.nixos.org](https://search.nixos.org/) to try them out. 1. Free up storage space: diff --git a/doc/manual/src/release-notes/index.md b/doc/manual/src/release-notes/index.md index cc805e631..d4e6292a6 100644 --- a/doc/manual/src/release-notes/index.md +++ b/doc/manual/src/release-notes/index.md @@ -1,12 +1,13 @@ # Nix Release Notes +The Nix release cycle is calendar-based as follows: + Nix has a release cycle of roughly 6 weeks. Notable changes and additions are announced in the release notes for each version. -Bugfixes can be backported on request to previous Nix releases. -We typically backport only as far back as the Nix version used in the latest NixOS release, which is announced in the [NixOS release notes](https://nixos.org/manual/nixos/stable/release-notes.html#ch-release-notes). - -Backports never skip releases. -If a feature is backported to version `x.y`, it must also be available in version `x.(y+1)`. -This ensures that upgrading from an older version with backports is still safe and no backported functionality will go missing. +The supported Nix versions are: +- The latest release +- The version used in the stable NixOS release, which is announced in the [NixOS release notes](https://nixos.org/manual/nixos/stable/release-notes.html#ch-release-notes). +Bugfixes and security issues are backported to every supported version. +Patch releases are published as needed. diff --git a/doc/manual/src/release-notes/rl-2.15.md b/doc/manual/src/release-notes/rl-2.15.md index 133121999..e7e52631b 100644 --- a/doc/manual/src/release-notes/rl-2.15.md +++ b/doc/manual/src/release-notes/rl-2.15.md @@ -11,7 +11,7 @@ As the choice of hash formats is no longer binary, the `--base16` flag is also added to explicitly specify the Base16 format, which is still the default. -* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](../glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents. +* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](@docroot@/glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents. The new `^` syntax for store paths introduced in Nix 2.13 allows explicitly referencing output paths of a derivation. Using this is better and more clear than relying on the now-removed `.drv` special handling. diff --git a/doc/manual/src/release-notes/rl-2.18.md b/doc/manual/src/release-notes/rl-2.18.md index 4bbc52b50..eb26fc9e7 100644 --- a/doc/manual/src/release-notes/rl-2.18.md +++ b/doc/manual/src/release-notes/rl-2.18.md @@ -13,7 +13,7 @@ - The `discard-references` feature has been stabilized. This means that the - [unsafeDiscardReferences](@docroot@/contributing/experimental-features.md#xp-feature-discard-references) + [unsafeDiscardReferences](@docroot@/development/experimental-features.md#xp-feature-discard-references) attribute is no longer guarded by an experimental flag and can be used freely. @@ -21,7 +21,7 @@ This only affects `nix-build --json` when "building" non-derivation things like fetched sources, which is a no-op. - A new builtin [`outputOf`](@docroot@/language/builtins.md#builtins-outputOf) has been added. - It is part of the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. + It is part of the [`dynamic-derivations`](@docroot@/development/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. - Flake follow paths at depths greater than 2 are now handled correctly, preventing "follows a non-existent input" errors. diff --git a/doc/manual/src/release-notes/rl-2.19.md b/doc/manual/src/release-notes/rl-2.19.md index ba6eb9c64..e2e2f85cc 100644 --- a/doc/manual/src/release-notes/rl-2.19.md +++ b/doc/manual/src/release-notes/rl-2.19.md @@ -17,8 +17,8 @@ - `nix-shell` shebang lines now support single-quoted arguments. -- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). - This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). +- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/development/experimental-features.md#xp-fetch-tree). + This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/development/experimental-features.md#xp-fetch-tree). - The interface for creating and updating lock files has been overhauled: @@ -33,7 +33,7 @@ - The flake-specific flags `--recreate-lock-file` and `--update-input` have been removed from all commands operating on installables. They are superceded by `nix flake update`. -- Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). +- Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches). - [`nix path-info --json`](@docroot@/command-ref/new-cli/nix3-path-info.md) (experimental) now returns a JSON map rather than JSON list. diff --git a/doc/manual/src/release-notes/rl-2.20.md b/doc/manual/src/release-notes/rl-2.20.md index 8ede168a4..eb724f600 100644 --- a/doc/manual/src/release-notes/rl-2.20.md +++ b/doc/manual/src/release-notes/rl-2.20.md @@ -200,3 +200,9 @@ while performing various operations (including `nix develop`, `nix flake update`, and so on). With several fixes to Nix's signal handlers, Nix commands will now exit quickly after Ctrl-C is pressed. + +- `nix copy` to a `ssh-ng` store now needs `--substitute-on-destination` (a.k.a. `-s`) + in order to substitute paths on the remote store instead of copying them. + The behavior is consistent with `nix copy` to a different kind of remote store. + Previously this behavior was controlled by the + `builders-use-substitutes` setting and `--substitute-on-destination` was ignored. diff --git a/doc/manual/src/release-notes/rl-2.21.md b/doc/manual/src/release-notes/rl-2.21.md new file mode 100644 index 000000000..75114f117 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.21.md @@ -0,0 +1,302 @@ +# Release 2.21.0 (2024-03-11) + +- Fix a fixed-output derivation sandbox escape (CVE-2024-27297) + + Cooperating Nix derivations could send file descriptors to files in the Nix + store to each other via Unix domain sockets in the abstract namespace. This + allowed one derivation to modify the output of the other derivation, after Nix + has registered the path as "valid" and immutable in the Nix database. + In particular, this allowed the output of fixed-output derivations to be + modified from their expected content. + + This isn't the case any more. + +- CLI options `--arg-from-file` and `--arg-from-stdin` [#10122](https://github.com/NixOS/nix/pull/10122) + + The new CLI option `--arg-from-file` *name* *path* passes the contents + of file *path* as a string value via the function argument *name* to a + Nix expression. Similarly, the new option `--arg-from-stdin` *name* + reads the contents of the string from standard input. + +- Concise error printing in `nix repl` [#9928](https://github.com/NixOS/nix/pull/9928) + + Previously, if an element of a list or attribute set threw an error while + evaluating, `nix repl` would print the entire error (including source location + information) inline. This output was clumsy and difficult to parse: + + ``` + nix-repl> { err = builtins.throw "uh oh!"; } + { err = «error: + … while calling the 'throw' builtin + at «string»:1:9: + 1| { err = builtins.throw "uh oh!"; } + | ^ + + error: uh oh!»; } + ``` + + Now, only the error message is displayed, making the output much more readable. + ``` + nix-repl> { err = builtins.throw "uh oh!"; } + { err = «error: uh oh!»; } + ``` + + However, if the whole expression being evaluated throws an error, source + locations and (if applicable) a stack trace are printed, just like you'd expect: + + ``` + nix-repl> builtins.throw "uh oh!" + error: + … while calling the 'throw' builtin + at «string»:1:1: + 1| builtins.throw "uh oh!" + | ^ + + error: uh oh! + ``` + +- `--debugger` can now access bindings from `let` expressions [#8827](https://github.com/NixOS/nix/issues/8827) [#9918](https://github.com/NixOS/nix/pull/9918) + + Breakpoints and errors in the bindings of a `let` expression can now access + those bindings in the debugger. Previously, only the body of `let` expressions + could access those bindings. + +- Enter the `--debugger` when `builtins.trace` is called if `debugger-on-trace` is set [#9914](https://github.com/NixOS/nix/pull/9914) + + If the `debugger-on-trace` option is set and `--debugger` is given, + `builtins.trace` calls will behave similarly to `builtins.break` and will enter + the debug REPL. This is useful for determining where warnings are being emitted + from. + +- Debugger prints source position information [#9913](https://github.com/NixOS/nix/pull/9913) + + The `--debugger` now prints source location information, instead of the + pointers of source location information. Before: + + ``` + nix-repl> :bt + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + 0x600001522598 + ``` + + After: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + /nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27 + + 131| + 132| bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs; + | ^ + 133| in + ``` + +- The `--debugger` will start more reliably in `let` expressions and function calls [#6649](https://github.com/NixOS/nix/issues/6649) [#9917](https://github.com/NixOS/nix/pull/9917) + + Previously, if you attempted to evaluate this file with the debugger: + + ```nix + let + a = builtins.trace "before inner break" ( + builtins.break "hello" + ); + b = builtins.trace "before outer break" ( + builtins.break a + ); + in + b + ``` + + Nix would correctly enter the debugger at `builtins.break a`, but if you asked + it to `:continue`, it would skip over the `builtins.break "hello"` expression + entirely. + + Now, Nix will correctly enter the debugger at both breakpoints. + +- Nested debuggers are no longer supported [#9920](https://github.com/NixOS/nix/pull/9920) + + Previously, evaluating an expression that throws an error in the debugger would + enter a second, nested debugger: + + ``` + nix-repl> builtins.throw "what" + error: what + + + Starting REPL to allow you to inspect the current state of the evaluator. + + Welcome to Nix 2.18.1. Type :? for help. + + nix-repl> + ``` + + Now, it just prints the error message like `nix repl`: + + ``` + nix-repl> builtins.throw "what" + error: + … while calling the 'throw' builtin + at «string»:1:1: + 1| builtins.throw "what" + | ^ + + error: what + ``` + +- Consistent order of function arguments in printed expressions [#9874](https://github.com/NixOS/nix/pull/9874) + + Function arguments are now printed in lexicographic order rather than the internal, creation-time based symbol order. + +- Fix duplicate attribute error positions for `inherit` [#9874](https://github.com/NixOS/nix/pull/9874) + + When an `inherit` caused a duplicate attribute error the position of the error was not reported correctly, placing the error with the inherit itself or at the start of the bindings block instead of the offending attribute name. + +- `inherit (x) ...` evaluates `x` only once [#9847](https://github.com/NixOS/nix/pull/9847) + + `inherit (x) a b ...` now evaluates the expression `x` only once for all inherited attributes rather than once for each inherited attribute. + This does not usually have a measurable impact, but side-effects (such as `builtins.trace`) would be duplicated and expensive expressions (such as derivations) could cause a measurable slowdown. + +- Store paths are allowed to start with `.` [#912](https://github.com/NixOS/nix/issues/912) [#9091](https://github.com/NixOS/nix/pull/9091) [#9095](https://github.com/NixOS/nix/pull/9095) [#9120](https://github.com/NixOS/nix/pull/9120) [#9121](https://github.com/NixOS/nix/pull/9121) [#9122](https://github.com/NixOS/nix/pull/9122) [#9130](https://github.com/NixOS/nix/pull/9130) [#9219](https://github.com/NixOS/nix/pull/9219) [#9224](https://github.com/NixOS/nix/pull/9224) [#9867](https://github.com/NixOS/nix/pull/9867) + + Leading periods were allowed by accident in Nix 2.4. The Nix team has considered this to be a bug, but this behavior has since been relied on by users, leading to unnecessary difficulties. + From now on, leading periods are supported. The names `.` and `..` are disallowed, as well as those starting with `.-` or `..-`. + + Nix versions that denied leading periods are documented [in the issue](https://github.com/NixOS/nix/issues/912#issuecomment-1919583286). + +- `nix repl` pretty-prints values [#9931](https://github.com/NixOS/nix/pull/9931) + + `nix repl` will now pretty-print values: + + ``` + { + attrs = { + a = { + b = { + c = { }; + }; + }; + }; + list = [ 1 ]; + list' = [ + 1 + 2 + 3 + ]; + } + ``` + +- Introduction of `--regex` and `--all` in `nix profile remove` and `nix profile upgrade` [#10166](https://github.com/NixOS/nix/pull/10166) + + Previously the command-line arguments for `nix profile remove` and `nix profile upgrade` matched the package entries using regular expression. + For instance: + + ``` + nix profile remove '.*vim.*' + ``` + + This would remove all packages that contain `vim` in their name. + + In most cases, only singular package names were used to remove and upgrade packages. Mixing this with regular expressions sometimes lead to unintended behavior. For instance, `python3.1` could match `python311`. + + To avoid unintended behavior, the arguments are now only matching exact names. + + Matching using regular expressions is still possible by using the new `--regex` flag: + + ``` + nix profile remove --regex '.*vim.*' + ``` + + One of the most useful cases for using regular expressions was to upgrade all packages. This was previously accomplished by: + + ``` + nix profile upgrade '.*' + ``` + + With the introduction of the `--all` flag, this now becomes more straightforward: + + ``` + nix profile upgrade --all + ``` + +- Visual clutter in `--debugger` is reduced [#9919](https://github.com/NixOS/nix/pull/9919) + + Before: + ``` + info: breakpoint reached + + + Starting REPL to allow you to inspect the current state of the evaluator. + + Welcome to Nix 2.20.0pre20231222_dirty. Type :? for help. + + nix-repl> :continue + error: uh oh + + + Starting REPL to allow you to inspect the current state of the evaluator. + + Welcome to Nix 2.20.0pre20231222_dirty. Type :? for help. + + nix-repl> + ``` + + After: + + ``` + info: breakpoint reached + + Nix 2.20.0pre20231222_dirty debugger + Type :? for help. + nix-repl> :continue + error: uh oh + + nix-repl> + ``` + +- Cycle detection in `nix repl` is simpler and more reliable [#8672](https://github.com/NixOS/nix/issues/8672) [#9926](https://github.com/NixOS/nix/pull/9926) + + The cycle detection in `nix repl`, `nix eval`, `builtins.trace`, and everywhere + else values are printed is now simpler and matches the cycle detection in + `nix-instantiate --eval` output. + + Before: + + ``` + nix eval --expr 'let self = { inherit self; }; in self' + { self = { self = «repeated»; }; } + ``` + + After: + + ``` + { self = «repeated»; } + ``` + +- In the debugger, `while evaluating the attribute` errors now include position information [#9915](https://github.com/NixOS/nix/pull/9915) + + Before: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + 0x600001522598 + ``` + + After: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + /nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27 + + 131| + 132| bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs; + | ^ + 133| in + ``` + +- Stack size is increased on macOS [#9860](https://github.com/NixOS/nix/pull/9860) + + Previously, Nix would set the stack size to 64MiB on Linux, but would leave the + stack size set to the default (approximately 8KiB) on macOS. Now, the stack + size is correctly set to 64MiB on macOS as well, which should reduce stack + overflow segfaults in deeply-recursive Nix expressions. + diff --git a/doc/manual/src/release-notes/rl-2.22.md b/doc/manual/src/release-notes/rl-2.22.md new file mode 100644 index 000000000..c78d3d692 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.22.md @@ -0,0 +1,21 @@ +# Release 2.22.0 (2024-04-23) + +### Significant changes + +- Remove experimental repl-flake [#10103](https://github.com/NixOS/nix/issues/10103) [#10299](https://github.com/NixOS/nix/pull/10299) + + The `repl-flake` experimental feature has been removed. The `nix repl` command now works like the rest of the new CLI in that `nix repl {path}` now tries to load a flake at `{path}` (or fails if the `flakes` experimental feature isn't enabled). + +### Other changes + +- `nix eval` prints derivations as `.drv` paths [#10200](https://github.com/NixOS/nix/pull/10200) + + `nix eval` will now print derivations as their `.drv` paths, rather than as + attribute sets. This makes commands like `nix eval nixpkgs#bash` terminate + instead of infinitely looping into recursive self-referential attributes: + + ```ShellSession + $ nix eval nixpkgs#bash + «derivation /nix/store/m32cbgbd598f4w299g0hwyv7gbw6rqcg-bash-5.2p26.drv» + ``` + diff --git a/doc/manual/src/release-notes/rl-2.23.md b/doc/manual/src/release-notes/rl-2.23.md new file mode 100644 index 000000000..ac842fdc0 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.23.md @@ -0,0 +1,102 @@ +# Release 2.23.0 (2024-06-03) + +- New builtin: `builtins.warn` [#306026](https://github.com/NixOS/nix/issues/306026) [#10592](https://github.com/NixOS/nix/pull/10592) + + `builtins.warn` behaves like `builtins.trace "warning: ${msg}"`, has an accurate log level, and is controlled by the options + [`debugger-on-trace`](@docroot@/command-ref/conf-file.md#conf-debugger-on-trace), + [`debugger-on-warn`](@docroot@/command-ref/conf-file.md#conf-debugger-on-warn) and + [`abort-on-warn`](@docroot@/command-ref/conf-file.md#conf-abort-on-warn). + +- Make `nix build --keep-going` consistent with `nix-build --keep-going` + + This means that if e.g. multiple fixed-output derivations fail to + build, all hash mismatches are displayed. + +- Modify `nix derivation {add,show}` JSON format [#9866](https://github.com/NixOS/nix/issues/9866) [#10722](https://github.com/NixOS/nix/pull/10722) + + The JSON format for derivations has been slightly revised to better conform to our [JSON guidelines](@docroot@/development/cli-guideline.md#returning-future-proof-json). + In particular, the hash algorithm and content addressing method of content-addresed derivation outputs are now separated into two fields `hashAlgo` and `method`, + rather than one field with an arcane `:`-separated format. + + This JSON format is only used by the experimental `nix derivation` family of commands, at this time. + Future revisions are expected as the JSON format is still not entirely in compliance even after these changes. + +- Warn on unknown settings anywhere in the command line [#10701](https://github.com/NixOS/nix/pull/10701) + + All `nix` commands will now properly warn when an unknown option is specified anywhere in the command line. + + Before: + + ```console + $ nix-instantiate --option foobar baz --expr '{}' + warning: unknown setting 'foobar' + $ nix-instantiate '{}' --option foobar baz --expr + $ nix eval --expr '{}' --option foobar baz + { } + ``` + + After: + + ```console + $ nix-instantiate --option foobar baz --expr '{}' + warning: unknown setting 'foobar' + $ nix-instantiate '{}' --option foobar baz --expr + warning: unknown setting 'foobar' + $ nix eval --expr '{}' --option foobar baz + warning: unknown setting 'foobar' + { } + ``` + +- `nix env shell` is the new `nix shell`, and `nix shell` remains an accepted alias [#10504](https://github.com/NixOS/nix/issues/10504) [#10807](https://github.com/NixOS/nix/pull/10807) + + This is part of an effort to bring more structure to the CLI subcommands. + + `nix env` will be about the process environment. + Future commands may include `nix env run` and `nix env print-env`. + + It is also somewhat analogous to the [planned](https://github.com/NixOS/nix/issues/10504) `nix dev shell` (currently `nix develop`), which is less about environment variables, and more about running a development shell, which is a more powerful command, but also requires more setup. + +- Flake operations that expect derivations now print the failing value and its type [#10778](https://github.com/NixOS/nix/pull/10778) + + In errors like `flake output attribute 'nixosConfigurations.yuki.config' is not a derivation or path`, the message now includes the failing value and type. + + Before: + + ``` + error: flake output attribute 'nixosConfigurations.yuki.config' is not a derivation or path + ```` + + After: + + ``` + error: expected flake output attribute 'nixosConfigurations.yuki.config' to be a derivation or path but found a set: { appstream = «thunk»; assertions = «thunk»; boot = { bcache = «thunk»; binfmt = «thunk»; binfmtMiscRegistrations = «thunk»; blacklistedKernelModules = «thunk»; bootMount = «thunk»; bootspec = «thunk»; cleanTmpDir = «thunk»; consoleLogLevel = «thunk»; «43 attributes elided» }; «48 attributes elided» } + ``` + +- `fetchTree` now fetches Git repositories shallowly by default [#10028](https://github.com/NixOS/nix/pull/10028) + + `builtins.fetchTree` now clones Git repositories shallowly by default, which reduces network traffic and disk usage significantly in many cases. + + Previously, the default behavior was to clone the full history of a specific tag or branch (e.g. `ref`) and only afterwards extract the files of one specific revision. + + From now on, the `ref` and `allRefs` arguments will be ignored, except if shallow cloning is disabled by setting `shallow = false`. + + The defaults for `builtins.fetchGit` remain unchanged. Here, shallow cloning has to be enabled manually by passing `shallow = true`. + +- Store object info JSON format now uses `null` rather than omitting fields [#9995](https://github.com/NixOS/nix/pull/9995) + + The [store object info JSON format](@docroot@/protocols/json/store-object-info.md), used for e.g. `nix path-info`, no longer omits fields to indicate absent information, but instead includes the fields with a `null` value. + For example, `"ca": null` is used to to indicate a store object that isn't content-addressed rather than omitting the `ca` field entirely. + This makes records of this sort more self-describing, and easier to consume programmatically. + + We will follow this design principle going forward; + the [JSON guidelines](@docroot@/development/json-guideline.md) in the contributing section have been updated accordingly. + +- Large path warnings [#10661](https://github.com/NixOS/nix/pull/10661) + + Nix can now warn when evaluation of a Nix expression causes a large + path to be copied to the Nix store. The threshold for this warning can + be configured using [the `warn-large-path-threshold` + setting](@docroot@/command-ref/conf-file.md#warn-large-path-threshold), + e.g. `--warn-large-path-threshold 100M` will warn about paths larger + than 100 MiB. + diff --git a/doc/manual/src/release-notes/rl-2.24.md b/doc/manual/src/release-notes/rl-2.24.md new file mode 100644 index 000000000..5bcc1d79c --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.24.md @@ -0,0 +1,324 @@ +# Release 2.24.0 (2024-07-31) + +### Significant changes + +- Harden user sandboxing + + The build directory has been hardened against interference with the outside world by nesting it inside another directory owned by (and only readable by) the daemon user. + + This is a low severity security fix, [CVE-2024-38531](https://www.cve.org/CVERecord?id=CVE-2024-38531). + + Credit: [**@alois31**](https://github.com/alois31), [**Linus Heckemann (@lheckemann)**](https://github.com/lheckemann) + Co-authors: [**@edolstra**](https://github.com/edolstra) + +- `nix-shell ` looks for `shell.nix` [#496](https://github.com/NixOS/nix/issues/496) [#2279](https://github.com/NixOS/nix/issues/2279) [#4529](https://github.com/NixOS/nix/issues/4529) [#5431](https://github.com/NixOS/nix/issues/5431) [#11053](https://github.com/NixOS/nix/issues/11053) [#11057](https://github.com/NixOS/nix/pull/11057) + + `nix-shell $x` now looks for `$x/shell.nix` when `$x` resolves to a directory. + + Although this might be seen as a breaking change, its primarily interactive usage makes it a minor issue. + This adjustment addresses a commonly reported problem. + + This also applies to `nix-shell` shebang scripts. Consider the following example: + + ```shell + #!/usr/bin/env nix-shell + #!nix-shell -i bash + ``` + + This will now load `shell.nix` from the script's directory, if it exists; `default.nix` otherwise. + + The old behavior can be opted into by setting the option [`nix-shell-always-looks-for-shell-nix`](@docroot@/command-ref/conf-file.md#conf-nix-shell-always-looks-for-shell-nix) to `false`. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + +- `nix-repl`'s `:doc` shows documentation comments [#3904](https://github.com/NixOS/nix/issues/3904) [#10771](https://github.com/NixOS/nix/issues/10771) [#1652](https://github.com/NixOS/nix/pull/1652) [#9054](https://github.com/NixOS/nix/pull/9054) [#11072](https://github.com/NixOS/nix/pull/11072) + + `nix repl` has a `:doc` command that previously only rendered documentation for internally defined functions. + This feature has been extended to also render function documentation comments, in accordance with [RFC 145]. + + Example: + + ``` + nix-repl> :doc lib.toFunction + Function toFunction + … defined at /home/user/h/nixpkgs/lib/trivial.nix:1072:5 + + Turns any non-callable values into constant functions. Returns + callable values as is. + + Inputs + + v + + : Any value + + Examples + + :::{.example} + + ## lib.trivial.toFunction usage example + + | nix-repl> lib.toFunction 1 2 + | 1 + | + | nix-repl> lib.toFunction (x: x + 1) 2 + | 3 + + ::: + ``` + + Known limitations: + - It does not render documentation for "formals", such as `{ /** the value to return */ x, ... }: x`. + - Some extensions to markdown are not yet supported, as you can see in the example above. + + We'd like to acknowledge [Yingchi Long (@inclyc)](https://github.com/inclyc) for proposing a proof of concept for this functionality in [#9054](https://github.com/NixOS/nix/pull/9054), as well as [@sternenseemann](https://github.com/sternenseemann) and [Johannes Kirschbauer (@hsjobeki)](https://github.com/hsjobeki) for their contributions, proposals, and their work on [RFC 145]. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + + [RFC 145]: https://github.com/NixOS/rfcs/pull/145 + +### Other changes + +- Solve `cached failure of attribute X` [#9165](https://github.com/NixOS/nix/issues/9165) [#10513](https://github.com/NixOS/nix/issues/10513) [#10564](https://github.com/NixOS/nix/pull/10564) + + This eliminates all "cached failure of attribute X" messages by forcing evaluation of the original value when needed to show the exception to the user. This enhancement improves error reporting by providing the underlying message and stack trace. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Run the flake regressions test suite [#10603](https://github.com/NixOS/nix/pull/10603) + + This update introduces a GitHub action to run a subset of the [flake regressions test suite](https://github.com/NixOS/flake-regressions), which includes 259 flakes with their expected evaluation results. Currently, the action runs the first 25 flakes due to the full test suite's extensive runtime. A manually triggered action may be implemented later to run the entire test suite. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Support unit prefixes in configuration settings [#10668](https://github.com/NixOS/nix/pull/10668) + + Configuration settings in Nix now support unit prefixes, allowing for more intuitive and readable configurations. For example, you can now specify [`--min-free 1G`](@docroot@/command-ref/opt-common.md#opt-min-free) to set the minimum free space to 1 gigabyte. + + This enhancement was extracted from [#7851](https://github.com/NixOS/nix/pull/7851) and is also useful for PR [#10661](https://github.com/NixOS/nix/pull/10661). + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- `nix build`: show all FOD errors with `--keep-going` [#10734](https://github.com/NixOS/nix/pull/10734) + + The [`nix build`](@docroot@/command-ref/new-cli/nix3-build.md) command has been updated to improve the behavior of the [`--keep-going`] flag. Now, when `--keep-going` is used, all hash-mismatch errors of failing fixed-output derivations (FODs) are displayed, similar to the behavior for other build failures. This enhancement ensures that all relevant build errors are shown, making it easier for users to update multiple derivations at once or to diagnose and fix issues. + + Author: [**Jörg Thalheim (@Mic92)**](https://github.com/Mic92), [**Maximilian Bosch (@Ma27)**](https://github.com/Ma27) + + [`--keep-going`](@docroot@/command-ref/opt-common.md#opt-keep-going) + +- Build with Meson [#2503](https://github.com/NixOS/nix/issues/2503) [#10378](https://github.com/NixOS/nix/pull/10378) [#10855](https://github.com/NixOS/nix/pull/10855) [#10904](https://github.com/NixOS/nix/pull/10904) [#10908](https://github.com/NixOS/nix/pull/10908) [#10914](https://github.com/NixOS/nix/pull/10914) [#10933](https://github.com/NixOS/nix/pull/10933) [#10936](https://github.com/NixOS/nix/pull/10936) [#10954](https://github.com/NixOS/nix/pull/10954) [#10955](https://github.com/NixOS/nix/pull/10955) [#10963](https://github.com/NixOS/nix/pull/10963) [#10967](https://github.com/NixOS/nix/pull/10967) [#10973](https://github.com/NixOS/nix/pull/10973) [#11034](https://github.com/NixOS/nix/pull/11034) [#11054](https://github.com/NixOS/nix/pull/11054) [#11055](https://github.com/NixOS/nix/pull/11055) [#11060](https://github.com/NixOS/nix/pull/11060) [#11064](https://github.com/NixOS/nix/pull/11064) [#11155](https://github.com/NixOS/nix/pull/11155) + + These changes aim to replace the use of autotools and `make` with Meson for building various components of Nix. Additionally, each library is built in its own derivation, leveraging Meson's "subprojects" feature to allow a single development shell for building all libraries while also supporting separate builds. This approach aims to improve productivity and build modularity, compared to both make and a monolithic Meson-based derivation. + + Special thanks to everyone who has contributed to the Meson port, particularly [**@p01arst0rm**](https://github.com/p01arst0rm) and [**@Qyriad**](https://github.com/Qyriad). + + Authors: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314), [**Tom Bereknyei**](https://github.com/tomberek), [**Théophane Hufschmitt (@thufschmitt)**](https://github.com/thufschmitt), [**Valentin Gagarin (@fricklerhandwerk)**](https://github.com/fricklerhandwerk), [**Robert Hensing (@roberth)**](https://github.com/roberth) + Co-authors: [**@p01arst0rm**](https://github.com/p01arst0rm), [**@Qyriad**](https://github.com/Qyriad) + +- Evaluation cache: fix cache regressions [#10570](https://github.com/NixOS/nix/issues/10570) [#11086](https://github.com/NixOS/nix/pull/11086) + + This update addresses two bugs in the evaluation cache system: + + 1. Regression in #10570: The evaluation cache was not being persisted in `nix develop`. + 2. Nix could sometimes try to commit the evaluation cache SQLite transaction without there being an active transaction, resulting in non-error errors being printed. + + Author: [**Lexi Mattick (@kognise)**](https://github.com/kognise) + +- Introduce `libnixflake` [#9063](https://github.com/NixOS/nix/pull/9063) + + A new library, `libnixflake`, has been introduced to better separate the Flakes layer within Nix. This change refactors the codebase to encapsulate Flakes-specific functionality within its own library. + + See the commits in the pull request for detailed changes, with the only significant code modifications happening in the initial commit. + + This change was alluded to in [RFC 134](https://github.com/nixos/rfcs/blob/master/rfcs/0134-nix-store-layer.md) and is a step towards a more modular and maintainable codebase. + + Author: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314) + +- CLI options `--arg-from-file` and `--arg-from-stdin` [#9913](https://github.com/NixOS/nix/pull/9913) + +- The `--debugger` now prints source location information, instead of the + pointers of source location information. Before: + + ``` + nix-repl> :bt + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + 0x600001522598 + ``` + + After: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + /nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27 + + 131| + 132| bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs; + | ^ + 133| in + ``` + +- Stop vendoring `toml11` + + We don't apply any patches to it, and vendoring it locks users into + bugs (it hasn't been updated since its introduction in late 2021). + + Author: [**Winter (@winterqt)**](https://github.com/winterqt) + +- Rename hash format `base32` to `nix32` [#8678](https://github.com/NixOS/nix/pull/8678) + + Hash format `base32` was renamed to `nix32` since it used a special nix-specific character set for + [Base32](https://en.wikipedia.org/wiki/Base32). + + **Deprecation**: Use `nix32` instead of `base32` as `toHashFormat` + + For the builtin `convertHash`, the `toHashFormat` parameter now accepts the same hash formats as the `--to`/`--from` + parameters of the `nix hash conert` command: `"base16"`, `"nix32"`, `"base64"`, and `"sri"`. The former `"base32"` value + remains as a deprecated alias for `"nix32"`. Please convert your code from: + + ```nix + builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base32";} + ``` + + to + + ```nix + builtins.convertHash { inherit hash hashAlgo; toHashFormat = "nix32";} + ``` + +- Add `pipe-operators` experimental feature [#11131](https://github.com/NixOS/nix/pull/11131) + + This is a draft implementation of [RFC 0148](https://github.com/NixOS/rfcs/pull/148). + + The `pipe-operators` experimental feature adds [`<|` and `|>` operators][pipe operators] to the Nix language. + *a* `|>` *b* is equivalent to the function application *b* *a*, and + *a* `<|` *b* is equivalent to the function application *a* *b*. + + For example: + + ``` + nix-repl> 1 |> builtins.add 2 |> builtins.mul 3 + 9 + + nix-repl> builtins.add 1 <| builtins.mul 2 <| 3 + 7 + ``` + + `<|` and `|>` are right and left associative, respectively, and have lower precedence than any other operator. + These properties may change in future releases. + + See [the RFC](https://github.com/NixOS/rfcs/pull/148) for more examples and rationale. + + [pipe operators]: @docroot@/language/operators.md#pipe-operators + +- `nix-shell` shebang uses relative path [#4232](https://github.com/NixOS/nix/issues/4232) [#5088](https://github.com/NixOS/nix/pull/5088) [#11058](https://github.com/NixOS/nix/pull/11058) + + + Relative [path](@docroot@/language/types.md#type-path) literals in `nix-shell` shebang scripts' options are now resolved relative to the [script's location](@docroot@/glossary.md?highlight=base%20directory#gloss-base-directory). + Previously they were resolved relative to the current working directory. + + For example, consider the following script in `~/myproject/say-hi`: + + ```shell + #!/usr/bin/env nix-shell + #!nix-shell --expr 'import ./shell.nix' + #!nix-shell --arg toolset './greeting-tools.nix' + #!nix-shell -i bash + hello + ``` + + Older versions of `nix-shell` would resolve `shell.nix` relative to the current working directory, such as the user's home directory in this example: + + ```console + [hostname:~]$ ./myproject/say-hi + error: + … while calling the 'import' builtin + at «string»:1:2: + 1| (import ./shell.nix) + | ^ + + error: path '/home/user/shell.nix' does not exist + ``` + + Since this release, `nix-shell` resolves `shell.nix` relative to the script's location, and `~/myproject/shell.nix` is used. + + ```console + $ ./myproject/say-hi + Hello, world! + ``` + + **Opt-out** + + This is technically a breaking change, so we have added an option so you can adapt independently of your Nix update. + The old behavior can be opted into by setting the option [`nix-shell-shebang-arguments-relative-to-script`](@docroot@/command-ref/conf-file.md#conf-nix-shell-shebang-arguments-relative-to-script) to `false`. + This option will be removed in a future release. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + +- Improve handling of tarballs that don't consist of a single top-level directory [#11195](https://github.com/NixOS/nix/pull/11195) + + In previous Nix releases, the tarball fetcher (used by `builtins.fetchTarball`) erroneously merged top-level directories into a single directory, and silently discarded top-level files that are not directories. This is no longer the case. The new behaviour is that *only* if the tarball consists of a single directory, the top-level path component of the files in the tarball is removed (similar to `tar`'s `--strip-components=1`). + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Improve handling of tarballs that don't consist of a single top-level directory [#11195](https://github.com/NixOS/nix/pull/11195) + + In previous Nix releases, the tarball fetcher (used by `builtins.fetchTarball`) erroneously merged top-level directories into a single directory, and silently discarded top-level files that are not directories. This is no longer the case. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Setting to warn about large paths [#10778](https://github.com/NixOS/nix/pull/10778) + + Nix can now warn when evaluation of a Nix expression causes a large + path to be copied to the Nix store. The threshold for this warning can + be configured using the `warn-large-path-threshold` setting, + e.g. `--warn-large-path-threshold 100M`. + + +# Contributors + +This release was made possible by the following 43 contributors: + +- Andreas Rammhold [**(@andir)**](https://github.com/andir) +- Andrew Marshall [**(@amarshall)**](https://github.com/amarshall) +- Brian McKenna [**(@puffnfresh)**](https://github.com/puffnfresh) +- Cameron [**(@SkamDart)**](https://github.com/SkamDart) +- Cole Helbling [**(@cole-h)**](https://github.com/cole-h) +- Corbin Simpson [**(@MostAwesomeDude)**](https://github.com/MostAwesomeDude) +- Eelco Dolstra [**(@edolstra)**](https://github.com/edolstra) +- Emily [**(@emilazy)**](https://github.com/emilazy) +- Enno Richter [**(@elohmeier)**](https://github.com/elohmeier) +- Farid Zakaria [**(@fzakaria)**](https://github.com/fzakaria) +- HaeNoe [**(@haenoe)**](https://github.com/haenoe) +- Hamir Mahal [**(@hamirmahal)**](https://github.com/hamirmahal) +- Harmen [**(@alicebob)**](https://github.com/alicebob) +- Ivan Trubach [**(@tie)**](https://github.com/tie) +- Jared Baur [**(@jmbaur)**](https://github.com/jmbaur) +- John Ericson [**(@Ericson2314)**](https://github.com/Ericson2314) +- Jonathan De Troye [**(@detroyejr)**](https://github.com/detroyejr) +- Jörg Thalheim [**(@Mic92)**](https://github.com/Mic92) +- Klemens Nanni [**(@klemensn)**](https://github.com/klemensn) +- Las Safin [**(@L-as)**](https://github.com/L-as) +- Lexi Mattick [**(@kognise)**](https://github.com/kognise) +- Matthew Bauer [**(@matthewbauer)**](https://github.com/matthewbauer) +- Max “Goldstein†Siling [**(@GoldsteinE)**](https://github.com/GoldsteinE) +- Mingye Wang [**(@Artoria2e5)**](https://github.com/Artoria2e5) +- Philip Taron [**(@philiptaron)**](https://github.com/philiptaron) +- Pierre Bourdon [**(@delroth)**](https://github.com/delroth) +- Pino Toscano [**(@pinotree)**](https://github.com/pinotree) +- RTUnreal [**(@RTUnreal)**](https://github.com/RTUnreal) +- Robert Hensing [**(@roberth)**](https://github.com/roberth) +- Romain Neil [**(@romain-neil)**](https://github.com/romain-neil) +- Ryan Hendrickson [**(@rhendric)**](https://github.com/rhendric) +- Sergei Trofimovich [**(@trofi)**](https://github.com/trofi) +- Shogo Takata [**(@pineapplehunter)**](https://github.com/pineapplehunter) +- Siddhant Kumar [**(@siddhantk232)**](https://github.com/siddhantk232) +- Silvan Mosberger [**(@infinisil)**](https://github.com/infinisil) +- Théophane Hufschmitt [**(@thufschmitt)**](https://github.com/thufschmitt) +- Valentin Gagarin [**(@fricklerhandwerk)**](https://github.com/fricklerhandwerk) +- Winter [**(@winterqt)**](https://github.com/winterqt) +- jade [**(@lf-)**](https://github.com/lf-) +- kirillrdy [**(@kirillrdy)**](https://github.com/kirillrdy) +- pennae [**(@pennae)**](https://github.com/pennae) +- poweredbypie [**(@poweredbypie)**](https://github.com/poweredbypie) +- tomberek [**(@tomberek)**](https://github.com/tomberek) diff --git a/doc/manual/src/release-notes/rl-2.4.md b/doc/manual/src/release-notes/rl-2.4.md index 8b566fc7b..1201e53b6 100644 --- a/doc/manual/src/release-notes/rl-2.4.md +++ b/doc/manual/src/release-notes/rl-2.4.md @@ -23,7 +23,7 @@ more than 2800 commits from 195 contributors since release 2.3. * The **`nix` command** has seen a lot of work and is now almost at feature parity with the old command-line interface (the `nix-*` commands). It aims to be [more modern, consistent and pleasant to - use](../contributing/cli-guideline.md) than the old CLI. It is still + use](../development/cli-guideline.md) than the old CLI. It is still marked as experimental but its interface should not change much anymore in future releases. diff --git a/doc/manual/src/store/file-system-object/content-address.md b/doc/manual/src/store/file-system-object/content-address.md new file mode 100644 index 000000000..410d7fb7c --- /dev/null +++ b/doc/manual/src/store/file-system-object/content-address.md @@ -0,0 +1,85 @@ +# Content-Addressing File System Objects + +For many operations, Nix needs to calculate [a content addresses](@docroot@/glossary.md#gloss-content-address) of [a file system object][file system object]. +Usually this is needed as part of +[content addressing store objects](../store-object/content-address.md), +since store objects always have a root file system object. +But some command-line utilities also just work on "raw" file system objects, not part of any store object. + +Every content addressing scheme Nix uses ultimately involves feeding data into a [hash function](https://en.wikipedia.org/wiki/Hash_function), and getting back an opaque fixed-size digest which is deemed a content address. +The various *methods* of content addressing thus differ in how abstract data (in this case, a file system object and its descendents) are fed into the hash function. + +## Serialising File System Objects { #serial } + +The simplest method is to serialise the entire file system object tree into a single binary string, and then hash that binary string, yielding the content address. +In this section we describe the currently-supported methods of serialising file system objects. + +### Flat { #serial-flat } + +A single file object can just be hashed by its contents. +This is not enough information to encode the fact that the file system object is a file, +but if we *already* know that the FSO is a single non-executable file by other means, it is sufficient. + +Because the hashed data is just the raw file, as is, this choice is good for compatibility with other systems. +For example, Unix commands like `sha256sum` or `sha1sum` will produce hashes for single files that match this. + +### Nix Archive (NAR) { #serial-nix-archive } + +For the other cases of [file system objects][file system object], especially directories with arbitrary descendents, we need a more complex serialisation format. +Examples of such serialisations are the ZIP and TAR file formats. +However, for our purposes these formats have two problems: + +- They do not have a canonical serialisation, meaning that given an FSO, there can +be many different serialisations. + For instance, TAR files can have variable amounts of padding between archive members; + and some archive formats leave the order of directory entries undefined. + This would be bad because we use serialisation to compute cryptographic hashes over file system objects, and for those hashes to be useful as a content address or for integrity checking, uniqueness is crucial. + Otherwise, correct hashes would report false mismatches, and the store would fail to find the content. + +- They store more information than we have in our notion of FSOs, such as time stamps. + This can cause FSOs that Nix should consider equal to hash to different values on different machines, just because the dates differ. + +- As a practical consideration, the TAR format is the only truly universal format in the Unix environment. + It has many problems, such as an inability to deal with long file names and files larger than 2^33 bytes. + Current implementations such as GNU Tar work around these limitations in various ways. + +For these reasons, Nix has its very own archive format—the Nix Archive (NAR) format, +which is carefully designed to avoid the problems described above. + +The exact specification of the Nix Archive format is in `protocols/nix-archive.md` + +## Content addressing File System Objects beyond a single serialisation pass + +Serialising the entire tree and then hashing that binary string is not the only option for content addressing, however. +Another technique is that of a [Merkle graph](https://en.wikipedia.org/wiki/Merkle_tree), where previously computed hashes are included in subsequent byte strings to be hashed. + +In particular, the Merkle graphs can match the original graph structure of file system objects: +we can first hash (serialised) child file system objects, and then hash parent objects using the hashes of their children in the serialisation (to be hashed) of the parent file system objects. + +Currently, there is one such Merkle DAG content addressing method supported. + +### Git ([experimental][xp-feature-git-hashing]) { #git } + +> **Warning** +> +> This method is part of the [`git-hashing`][xp-feature-git-hashing] experimental feature. + +Git's file system model is very close to Nix's, and so Git's content addressing method is a pretty good fit. +Just as with regular Git, files and symlinks are hashed as git "blobs", and directories are hashed as git "trees". + +However, one difference between Nix's and Git's file system model needs special treatment. +Plain files, executable files, and symlinks are not differentiated as distinctly addressable objects, but by their context: by the directory entry that refers to them. +That means so long as the root object is a directory, there is no problem: +every non-directory object is owned by a parent directory, and the entry that refers to it provides the missing information. +However, if the root object is not a directory, then we have no way of knowing which one of an executable file, non-executable file, or symlink it is supposed to be. + +In response to this, we have decided to treat a bare file as non-executable file. +This is similar to do what we do with [flat serialisation](#serial-flat), which also lacks this information. +To avoid an address collision, attempts to hash a bare executable file or symlink will result in an error (just as would happen for flat serialisation also). +Thus, Git can encode some, but not all of Nix's "File System Objects", and this sort of content-addressing is likewise partial. + +In the future, we may support a Git-like hash for such file system objects, or we may adopt another Merkle DAG format which is capable of representing all Nix file system objects. + +[file system object]: ../file-system-object.md +[store object]: ../store-object.md +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/doc/manual/src/store/store-object/content-address.md b/doc/manual/src/store/store-object/content-address.md new file mode 100644 index 000000000..02dce2836 --- /dev/null +++ b/doc/manual/src/store/store-object/content-address.md @@ -0,0 +1,95 @@ +# Content-Addressing Store Objects + +Just [like][fso-ca] [File System Objects][File System Object], +[Store Objects][Store Object] can also be [content-addressed](@docroot@/glossary.md#gloss-content-addressed), +unless they are [input-addressed](@docroot@/glossary.md#gloss-input-addressed-store-object). + +For store objects, the content address we produce will take the form of a [Store Path] rather than regular hash. +In particular, the content-addressing scheme will ensure that the digest of the store path is solely computed from the + +- file system object graph (the root one and its children, if it has any) +- references +- [store directory](../store-path.md#store-directory) +- name + +of the store object, and not any other information, which would not be an intrinsic property of that store object. + +For the full specification of the algorithms involved, see the [specification of store path digests][sp-spec]. + +[File System Object]: ../file-system-object.md +[Store Object]: ../store-object.md +[Store Path]: ../store-path.md + +## Content addressing each part of a store object + +### File System Objects + +With all currently supported store object content addressing methods, the file system object is always [content-addressed][fso-ca] first, and then that hash is incorporated into content address computation for the store object. + +### References + +With all currently supported store object content addressing methods, +other objects are referred to by their regular (string-encoded-) [store paths][Store Path]. + +Self-references however cannot be referred to by their path, because we are in the midst of describing how to compute that path! + +> The alternative would require finding as hash function fixed point, i.e. the solution to an equation in the form +> ``` +> digest = hash(..... || digest || ....) +> ``` +> which is computationally infeasible. +> As far as we know, this is equivalent to finding a hash collision. + +Instead we just have a "has self reference" boolean, which will end up affecting the digest. + +### Name and Store Directory + +These two items affect the digest in a way that is standard for store path digest computations and not specific to content-addressing. +Consult the [specification of store path digests][sp-spec] for further details. + +## Content addressing Methods + +For historical reasons, we don't support all features in all combinations. +Each currently supported method of content addressing chooses a single method of file system object hashing, and may offer some restrictions on references. +The names and store directories are unrestricted however. + +### Flat { #method-flat } + +This uses the corresponding [Flat](../file-system-object/content-address.md#serial-flat) method of file system object content addressing. + +References are not supported: store objects with flat hashing *and* references can not be created. + +### Text { #method-text } + +This also uses the corresponding [Flat](../file-system-object/content-address.md#serial-flat) method of file system object content addressing. + +References to other store objects are supported, but self references are not. + +This is the only store-object content-addressing method that is not named identically with a corresponding file system object method. +It is somewhat obscure, mainly used for "drv files" +(derivations serialized as store objects in their ["ATerm" file format](@docroot@/protocols/derivation-aterm.md)). +Prefer another method if possible. + +### Nix Archive { #method-nix-archive } + +This uses the corresponding [Nix Archive](../file-system-object/content-address.md#serial-nix-archive) method of file system object content addressing. + +References (to other store objects and self references alike) are supported so long as the hash algorithm is SHA-256, but not (neither kind) otherwise. + +### Git { #method-git } + +> **Warning** +> +> This method is part of the [`git-hashing`][xp-feature-git-hashing] experimental feature. + +This uses the corresponding [Git](../file-system-object/content-address.md#serial-git) method of file system object content addressing. + +References are not supported. + +Only SHA-1 is supported at this time. +If [SHA-256-based Git](https://git-scm.com/docs/hash-function-transition) +becomes more widespread, this restriction will be revisited. + +[fso-ca]: ../file-system-object/content-address.md +[sp-spec]: @docroot@/protocols/store-path.md +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/doc/manual/src/store/store-path.md b/doc/manual/src/store/store-path.md index b5ad0c654..beec2389b 100644 --- a/doc/manual/src/store/store-path.md +++ b/doc/manual/src/store/store-path.md @@ -1,5 +1,11 @@ # Store Path +> **Example** +> +> `/nix/store/a040m110amc4h71lds2jmr8qrkj2jhxd-git-2.38.1` +> +> A rendered store path + Nix implements references to [store objects](./index.md#store-object) as *store paths*. Think of a store path as an [opaque], [unique identifier]: @@ -37,6 +43,10 @@ A store path is rendered to a file system path as the concatenation of > store directory digest name > ``` +Exactly how the digest is calculated depends on the type of store path. +Store path digests are *supposed* to be opaque, and so for most operations, it is not necessary to know the details. +That said, the manual has a full [specification of store path digests](@docroot@/protocols/store-path.md). + ## Store Directory Every [Nix store](./index.md) has a store directory. @@ -46,7 +56,7 @@ But if the store has a file system representation, the store directory contains [file system objects]: ./file-system-object.md -This means a store path is not just derived from the referenced store object itself, but depends on the store the store object is in. +This means a store path is not just derived from the referenced store object itself, but depends on the store that the store object is in. > **Note** > diff --git a/flake.lock b/flake.lock index bb2e400c0..2ac413a69 100644 --- a/flake.lock +++ b/flake.lock @@ -3,11 +3,11 @@ "flake-compat": { "flake": false, "locked": { - "lastModified": 1673956053, - "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", "owner": "edolstra", "repo": "flake-compat", - "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", "type": "github" }, "original": { @@ -16,38 +16,100 @@ "type": "github" } }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1719994518, + "narHash": "sha256-pQMhCCHyQGRzdfAkdJ4cIWiw+JNuWsTX7f0ZYSyz0VY=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "9227223f6d922fee3c7b190b2cc238a99527bbb7", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "git-hooks-nix": { + "inputs": { + "flake-compat": [], + "gitignore": [], + "nixpkgs": [ + "nixpkgs" + ], + "nixpkgs-stable": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1721042469, + "narHash": "sha256-6FPUl7HVtvRHCCBQne7Ylp4p+dpP3P/OYuzjztZ4s70=", + "owner": "cachix", + "repo": "git-hooks.nix", + "rev": "f451c19376071a90d8c58ab1a953c6e9840527fd", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "git-hooks.nix", + "type": "github" + } + }, "libgit2": { "flake": false, "locked": { - "lastModified": 1697646580, - "narHash": "sha256-oX4Z3S9WtJlwvj0uH9HlYcWv+x1hqp8mhXl7HsLu2f0=", + "lastModified": 1715853528, + "narHash": "sha256-J2rCxTecyLbbDdsyBWn9w7r3pbKRMkI9E7RvRgAqBdY=", "owner": "libgit2", "repo": "libgit2", - "rev": "45fd9ed7ae1a9b74b957ef4f337bc3c8b3df01b5", + "rev": "36f7e21ad757a3dacc58cf7944329da6bc1d6e96", "type": "github" }, "original": { "owner": "libgit2", + "ref": "v1.8.1", "repo": "libgit2", "type": "github" } }, "nixpkgs": { "locked": { - "lastModified": 1709083642, - "narHash": "sha256-7kkJQd4rZ+vFrzWu8sTRtta5D1kBG0LSRYAfhtmMlSo=", + "lastModified": 1721548954, + "narHash": "sha256-7cCC8+Tdq1+3OPyc3+gVo9dzUNkNIQfwSDJ2HSi2u3o=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "b550fe4b4776908ac2a861124307045f8e717c8e", + "rev": "63d37ccd2d178d54e7fb691d7ec76000740ea24a", "type": "github" }, "original": { "owner": "NixOS", - "ref": "release-23.11", + "ref": "nixos-24.05", "repo": "nixpkgs", "type": "github" } }, + "nixpkgs-23-11": { + "locked": { + "lastModified": 1717159533, + "narHash": "sha256-oamiKNfr2MS6yH64rUn99mIZjc45nGJlj9eGth/3Xuw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446", + "type": "github" + } + }, "nixpkgs-regression": { "locked": { "lastModified": 1643052045, @@ -67,8 +129,11 @@ "root": { "inputs": { "flake-compat": "flake-compat", + "flake-parts": "flake-parts", + "git-hooks-nix": "git-hooks-nix", "libgit2": "libgit2", "nixpkgs": "nixpkgs", + "nixpkgs-23-11": "nixpkgs-23-11", "nixpkgs-regression": "nixpkgs-regression" } } diff --git a/flake.nix b/flake.nix index 42aaace67..9e8592e3a 100644 --- a/flake.nix +++ b/flake.nix @@ -1,18 +1,28 @@ { description = "The purely functional package manager"; - # TODO switch to nixos-23.11-small - # https://nixpk.gs/pr-tracker.html?pr=291954 - inputs.nixpkgs.url = "github:NixOS/nixpkgs/release-23.11"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; + inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446"; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; - inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; }; + inputs.libgit2 = { url = "github:libgit2/libgit2/v1.8.1"; flake = false; }; + + # dev tooling + inputs.flake-parts.url = "github:hercules-ci/flake-parts"; + inputs.git-hooks-nix.url = "github:cachix/git-hooks.nix"; + # work around https://github.com/NixOS/nix/issues/7730 + inputs.flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; + inputs.git-hooks-nix.inputs.nixpkgs.follows = "nixpkgs"; + inputs.git-hooks-nix.inputs.nixpkgs-stable.follows = "nixpkgs"; + # work around 7730 and https://github.com/NixOS/nix/issues/7807 + inputs.git-hooks-nix.inputs.flake-compat.follows = ""; + inputs.git-hooks-nix.inputs.gitignore.follows = ""; + + outputs = inputs@{ self, nixpkgs, nixpkgs-regression, libgit2, ... }: - outputs = { self, nixpkgs, nixpkgs-regression, libgit2, ... }: let inherit (nixpkgs) lib; - inherit (lib) fileset; officialRelease = false; @@ -31,14 +41,9 @@ crossSystems = [ "armv6l-unknown-linux-gnueabihf" "armv7l-unknown-linux-gnueabihf" - "x86_64-unknown-freebsd13" + "riscv64-unknown-linux-gnu" "x86_64-unknown-netbsd" - ]; - - # Nix doesn't yet build on this platform, so we put it in a - # separate list. We just use this for `devShells` and - # `nixpkgsFor`, which this depends on. - shellCrossSystems = crossSystems ++ [ + "x86_64-unknown-freebsd" "x86_64-w64-mingw32" ]; @@ -50,6 +55,18 @@ "stdenv" ]; + /** + `flatMapAttrs attrs f` applies `f` to each attribute in `attrs` and + merges the results into a single attribute set. + + This can be nested to form a build matrix where all the attributes + generated by the innermost `f` are returned as is. + (Provided that the names are unique.) + + See https://nixos.org/manual/nixpkgs/stable/index.html#function-library-lib.attrsets.concatMapAttrs + */ + flatMapAttrs = attrs: f: lib.concatMapAttrs f attrs; + forAllSystems = lib.genAttrs systems; forAllCrossSystems = lib.genAttrs crossSystems; @@ -63,6 +80,17 @@ }) stdenvs); + + # We don't apply flake-parts to the whole flake so that non-development attributes + # load without fetching any development inputs. + devFlake = inputs.flake-parts.lib.mkFlake { inherit inputs; } { + imports = [ ./maintainers/flake-module.nix ]; + systems = lib.subtractLists crossSystems systems; + perSystem = { system, ... }: { + _module.args.pkgs = nixpkgsFor.${system}.native; + }; + }; + # Memoize nixpkgs for different platforms for efficiency. nixpkgsFor = forAllSystems (system: let @@ -84,31 +112,9 @@ in { inherit stdenvs native; static = native.pkgsStatic; - cross = lib.genAttrs shellCrossSystems (crossSystem: make-pkgs crossSystem "stdenv"); + cross = forAllCrossSystems (crossSystem: make-pkgs crossSystem "stdenv"); }); - installScriptFor = tarballs: - nixpkgsFor.x86_64-linux.native.callPackage ./scripts/installer.nix { - inherit tarballs; - }; - - testNixVersions = pkgs: client: daemon: - pkgs.callPackage ./package.nix { - pname = - "nix-tests" - + lib.optionalString - (lib.versionAtLeast daemon.version "2.4pre20211005" && - lib.versionAtLeast client.version "2.4pre20211005") - "-${client.version}-against-${daemon.version}"; - - inherit fileset; - - test-client = client; - test-daemon = daemon; - - doBuild = false; - }; - binaryTarball = nix: pkgs: pkgs.callPackage ./scripts/binary-tarball.nix { inherit nix; }; @@ -120,244 +126,130 @@ { nixStable = prev.nix; - default-busybox-sandbox-shell = final.busybox.override { - useMusl = true; - enableStatic = true; - enableMinimal = true; - extraConfig = '' - CONFIG_FEATURE_FANCY_ECHO y - CONFIG_FEATURE_SH_MATH y - CONFIG_FEATURE_SH_MATH_64 y + # A new scope, so that we can use `callPackage` to inject our own interdependencies + # without "polluting" the top level "`pkgs`" attrset. + # This also has the benefit of providing us with a distinct set of packages + # we can iterate over. + nixComponents = lib.makeScope final.nixDependencies.newScope (import ./packaging/components.nix); - CONFIG_ASH y - CONFIG_ASH_OPTIMIZE_FOR_SIZE y - - CONFIG_ASH_ALIAS y - CONFIG_ASH_BASH_COMPAT y - CONFIG_ASH_CMDCMD y - CONFIG_ASH_ECHO y - CONFIG_ASH_GETOPTS y - CONFIG_ASH_INTERNAL_GLOB y - CONFIG_ASH_JOB_CONTROL y - CONFIG_ASH_PRINTF y - CONFIG_ASH_TEST y - ''; - }; - - libgit2-nix = final.libgit2.overrideAttrs (attrs: { - src = libgit2; - version = libgit2.lastModifiedDate; - cmakeFlags = attrs.cmakeFlags or [] - ++ [ "-DUSE_SSH=exec" ]; + # The dependencies are in their own scope, so that they don't have to be + # in Nixpkgs top level `pkgs` or `nixComponents`. + nixDependencies = lib.makeScope final.newScope (import ./packaging/dependencies.nix { + inherit inputs stdenv versionSuffix; + pkgs = final; }); - boehmgc-nix = (final.boehmgc.override { - enableLargeConfig = true; - }).overrideAttrs(o: { - patches = (o.patches or []) ++ [ - ./dep-patches/boehmgc-coroutine-sp-fallback.diff + nix = final.nixComponents.nix; - # https://github.com/ivmai/bdwgc/pull/586 - ./dep-patches/boehmgc-traceable_allocator-public.diff - ]; - }); - - changelog-d-nix = final.buildPackages.callPackage ./misc/changelog-d.nix { }; - - nix = - let - officialRelease = false; - versionSuffix = - if officialRelease - then "" - else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}"; - - in final.callPackage ./package.nix { - inherit - fileset - stdenv - versionSuffix - ; - officialRelease = false; - boehmgc = final.boehmgc-nix; - libgit2 = final.libgit2-nix; - busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell; - } // { - # this is a proper separate downstream package, but put - # here also for back compat reasons. - perl-bindings = final.nix-perl-bindings; - }; - - nix-perl-bindings = final.callPackage ./perl { - inherit fileset stdenv; + nix_noTests = final.nix.override { + doInstallCheck = false; + doCheck = false; }; + # See https://github.com/NixOS/nixpkgs/pull/214409 + # Remove when fixed in this flake's nixpkgs + pre-commit = + if prev.stdenv.hostPlatform.system == "i686-linux" + then (prev.pre-commit.override (o: { dotnet-sdk = ""; })).overridePythonAttrs (o: { doCheck = false; }) + else prev.pre-commit; + }; in { # A Nixpkgs overlay that overrides the 'nix' and - # 'nix.perl-bindings' packages. + # 'nix-perl-bindings' packages. overlays.default = overlayFor (p: p.stdenv); - hydraJobs = { - - # Binary package for various platforms. - build = forAllSystems (system: self.packages.${system}.nix); - - shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation); - - buildStatic = lib.genAttrs linux64BitSystems (system: self.packages.${system}.nix-static); - - buildCross = forAllCrossSystems (crossSystem: - lib.genAttrs ["x86_64-linux"] (system: self.packages.${system}."nix-${crossSystem}")); - - buildNoGc = forAllSystems (system: - self.packages.${system}.nix.override { enableGC = false; } - ); - - buildNoTests = forAllSystems (system: - self.packages.${system}.nix.override { - doCheck = false; - doInstallCheck = false; - installUnitTests = false; - } - ); - - # Toggles some settings for better coverage. Windows needs these - # library combinations, and Debian build Nix with GNU readline too. - buildReadlineNoMarkdown = forAllSystems (system: - self.packages.${system}.nix.override { - enableMarkdown = false; - readlineFlavor = "readline"; - } - ); - - # Perl bindings for various platforms. - perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nix.perl-bindings); - - # Binary tarball for various platforms, containing a Nix store - # with the closure of 'nix' package, and the second half of - # the installation script. - binaryTarball = forAllSystems (system: binaryTarball nixpkgsFor.${system}.native.nix nixpkgsFor.${system}.native); - - binaryTarballCross = lib.genAttrs ["x86_64-linux"] (system: - forAllCrossSystems (crossSystem: - binaryTarball - self.packages.${system}."nix-${crossSystem}" - nixpkgsFor.${system}.cross.${crossSystem})); - - # The first half of the installation script. This is uploaded - # to https://nixos.org/nix/install. It downloads the binary - # tarball for the user's system and calls the second half of the - # installation script. - installerScript = installScriptFor [ - # Native - self.hydraJobs.binaryTarball."x86_64-linux" - self.hydraJobs.binaryTarball."i686-linux" - self.hydraJobs.binaryTarball."aarch64-linux" - self.hydraJobs.binaryTarball."x86_64-darwin" - self.hydraJobs.binaryTarball."aarch64-darwin" - # Cross - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" - ]; - installerScriptForGHA = installScriptFor [ - # Native - self.hydraJobs.binaryTarball."x86_64-linux" - self.hydraJobs.binaryTarball."x86_64-darwin" - # Cross - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" - ]; - - # docker image with Nix inside - dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); - - # Line coverage analysis. - coverage = nixpkgsFor.x86_64-linux.native.nix.override { - pname = "nix-coverage"; - withCoverageChecks = true; - }; - - # API docs for Nix's unstable internal C++ interfaces. - internal-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ./package.nix { - inherit fileset; - doBuild = false; - enableInternalAPIDocs = true; - }; - - # System tests. - tests = import ./tests/nixos { inherit lib nixpkgs nixpkgsFor; } // { - - # Make sure that nix-env still produces the exact same result - # on a particular version of Nixpkgs. - evalNixpkgs = - let - inherit (nixpkgsFor.x86_64-linux.native) runCommand nix; - in - runCommand "eval-nixos" { buildInputs = [ nix ]; } - '' - type -p nix-env - # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593. - time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages - [[ $(sha1sum < packages | cut -c1-40) = ff451c521e61e4fe72bdbe2d0ca5d1809affa733 ]] - mkdir $out - ''; - - nixpkgsLibTests = - forAllSystems (system: - import (nixpkgs + "/lib/tests/release.nix") - { pkgs = nixpkgsFor.${system}.native; - nixVersions = [ self.packages.${system}.nix ]; - } - ); - }; - - metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" { - pkgs = nixpkgsFor.x86_64-linux.native; - nixpkgs = nixpkgs-regression; - }; - - installTests = forAllSystems (system: - let pkgs = nixpkgsFor.${system}.native; in - pkgs.runCommand "install-tests" { - againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix; - againstCurrentUnstable = - # FIXME: temporarily disable this on macOS because of #3605. - if system == "x86_64-linux" - then testNixVersions pkgs pkgs.nix pkgs.nixUnstable - else null; - # Disabled because the latest stable version doesn't handle - # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work - # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable; - } "touch $out"); - - installerTests = import ./tests/installer { - binaryTarballs = self.hydraJobs.binaryTarball; - inherit nixpkgsFor; - }; - + hydraJobs = import ./packaging/hydra.nix { + inherit + inputs + binaryTarball + forAllCrossSystems + forAllSystems + lib + linux64BitSystems + nixpkgsFor + self + ; }; checks = forAllSystems (system: { binaryTarball = self.hydraJobs.binaryTarball.${system}; - perlBindings = self.hydraJobs.perlBindings.${system}; installTests = self.hydraJobs.installTests.${system}; nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system}; rl-next = let pkgs = nixpkgsFor.${system}.native; in pkgs.buildPackages.runCommand "test-rl-next-release-notes" { } '' - LANG=C.UTF-8 ${pkgs.changelog-d-nix}/bin/changelog-d ${./doc/manual/rl-next} >$out + LANG=C.UTF-8 ${pkgs.changelog-d}/bin/changelog-d ${./doc/manual/rl-next} >$out ''; + repl-completion = nixpkgsFor.${system}.native.callPackage ./tests/repl-completion.nix { }; } // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) { dockerImage = self.hydraJobs.dockerImage.${system}; - }); + } // (lib.optionalAttrs (!(builtins.elem system linux32BitSystems))) { + # Some perl dependencies are broken on i686-linux. + # Since the support is only best-effort there, disable the perl + # bindings - packages = forAllSystems (system: rec { - inherit (nixpkgsFor.${system}.native) nix changelog-d-nix; - default = nix; - } // (lib.optionalAttrs (builtins.elem system linux64BitSystems) { - nix-static = nixpkgsFor.${system}.static.nix; + # Temporarily disabled because GitHub Actions OOM issues. Once + # the old build system is gone and we are back to one build + # system, we should reenable this. + #perlBindings = self.hydraJobs.perlBindings.${system}; + } + # Add "passthru" tests + // flatMapAttrs ({ + "" = nixpkgsFor.${system}.native; + } // lib.optionalAttrs (! nixpkgsFor.${system}.native.stdenv.hostPlatform.isDarwin) { + # TODO: enable static builds for darwin, blocked on: + # https://github.com/NixOS/nixpkgs/issues/320448 + "static-" = nixpkgsFor.${system}.static; + }) + (nixpkgsPrefix: nixpkgs: + flatMapAttrs nixpkgs.nixComponents + (pkgName: pkg: + flatMapAttrs pkg.tests or {} + (testName: test: { + "${nixpkgsPrefix}${pkgName}-${testName}" = test; + }) + ) + ) + // devFlake.checks.${system} or {} + ); + + packages = forAllSystems (system: + { # Here we put attributes that map 1:1 into packages., ie + # for which we don't apply the full build matrix such as cross or static. + inherit (nixpkgsFor.${system}.native) + changelog-d; + default = self.packages.${system}.nix; + nix-internal-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-internal-api-docs; + nix-external-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-external-api-docs; + } + # We need to flatten recursive attribute sets of derivations to pass `flake check`. + // flatMapAttrs + { # Components we'll iterate over in the upcoming lambda + "nix" = { }; + # Temporarily disabled because GitHub Actions OOM issues. Once + # the old build system is gone and we are back to one build + # system, we should reenable these. + #"nix-util" = { }; + #"nix-store" = { }; + #"nix-fetchers" = { }; + } + (pkgName: {}: { + # These attributes go right into `packages.`. + "${pkgName}" = nixpkgsFor.${system}.native.nixComponents.${pkgName}; + "${pkgName}-static" = nixpkgsFor.${system}.static.nixComponents.${pkgName}; + } + // flatMapAttrs (lib.genAttrs crossSystems (_: { })) (crossSystem: {}: { + # These attributes go right into `packages.`. + "${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName}; + }) + // flatMapAttrs (lib.genAttrs stdenvs (_: { })) (stdenvName: {}: { + # These attributes go right into `packages.`. + "${pkgName}-${stdenvName}" = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nixComponents.${pkgName}; + }) + ) + // lib.optionalAttrs (builtins.elem system linux64BitSystems) { dockerImage = let pkgs = nixpkgsFor.${system}.native; @@ -372,21 +264,27 @@ ln -s ${image} $image echo "file binary-dist $image" >> $out/nix-support/hydra-build-products ''; - } // builtins.listToAttrs (map - (crossSystem: { - name = "nix-${crossSystem}"; - value = nixpkgsFor.${system}.cross.${crossSystem}.nix; - }) - crossSystems) - // builtins.listToAttrs (map - (stdenvName: { - name = "nix-${stdenvName}"; - value = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nix; - }) - stdenvs))); + }); devShells = let - makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; forDevShell = true; }).overrideAttrs (attrs: { + makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; forDevShell = true; }).overrideAttrs (attrs: + let + modular = devFlake.getSystem stdenv.buildPlatform.system; + transformFlag = prefix: flag: + assert builtins.isString flag; + let + rest = builtins.substring 2 (builtins.stringLength flag) flag; + in + "-D${prefix}:${rest}"; + havePerl = stdenv.buildPlatform == stdenv.hostPlatform && stdenv.hostPlatform.isUnix; + ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags; + in { + pname = "shell-for-" + attrs.pname; + + # Remove the version suffix to avoid unnecessary attempts to substitute in nix develop + version = lib.fileContents ./.version; + name = attrs.pname; + installFlags = "sysconfdir=$(out)/etc"; shellHook = '' PATH=$prefix/bin:$PATH @@ -397,11 +295,62 @@ XDG_DATA_DIRS+=:$out/share ''; + # We use this shell with the local checkout, not unpackPhase. + src = null; + + env = { + # Needed for Meson to find Boost. + # https://github.com/NixOS/nixpkgs/issues/86131. + BOOST_INCLUDEDIR = "${lib.getDev pkgs.nixDependencies.boost}/include"; + BOOST_LIBRARYDIR = "${lib.getLib pkgs.nixDependencies.boost}/lib"; + # For `make format`, to work without installing pre-commit + _NIX_PRE_COMMIT_HOOKS_CONFIG = + "${(pkgs.formats.yaml { }).generate "pre-commit-config.yaml" modular.pre-commit.settings.rawConfig}"; + }; + + mesonFlags = + map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents.nix-util.mesonFlags) + ++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents.nix-store.mesonFlags) + ++ map (transformFlag "libfetchers") (ignoreCrossFile pkgs.nixComponents.nix-fetchers.mesonFlags) + ++ lib.optionals havePerl (map (transformFlag "perl") (ignoreCrossFile pkgs.nixComponents.nix-perl-bindings.mesonFlags)) + ++ map (transformFlag "libexpr") (ignoreCrossFile pkgs.nixComponents.nix-expr.mesonFlags) + ++ map (transformFlag "libcmd") (ignoreCrossFile pkgs.nixComponents.nix-cmd.mesonFlags) + ; + nativeBuildInputs = attrs.nativeBuildInputs or [] + ++ pkgs.nixComponents.nix-util.nativeBuildInputs + ++ pkgs.nixComponents.nix-store.nativeBuildInputs + ++ pkgs.nixComponents.nix-fetchers.nativeBuildInputs + ++ lib.optionals havePerl pkgs.nixComponents.nix-perl-bindings.nativeBuildInputs + ++ pkgs.nixComponents.nix-internal-api-docs.nativeBuildInputs + ++ pkgs.nixComponents.nix-external-api-docs.nativeBuildInputs + ++ lib.optional + (!stdenv.buildPlatform.canExecute stdenv.hostPlatform + # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479 + && !(stdenv.hostPlatform.isWindows && stdenv.buildPlatform.isDarwin) + && stdenv.hostPlatform.emulatorAvailable pkgs.buildPackages + && lib.meta.availableOn stdenv.buildPlatform (stdenv.hostPlatform.emulator pkgs.buildPackages)) + pkgs.buildPackages.mesonEmulatorHook + ++ [ + pkgs.buildPackages.cmake + pkgs.buildPackages.shellcheck + pkgs.buildPackages.changelog-d + modular.pre-commit.settings.package + (pkgs.writeScriptBin "pre-commit-hooks-install" + modular.pre-commit.settings.installationScript) + ] # TODO: Remove the darwin check once # https://github.com/NixOS/nixpkgs/pull/291814 is available ++ lib.optional (stdenv.cc.isClang && !stdenv.buildPlatform.isDarwin) pkgs.buildPackages.bear ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) pkgs.buildPackages.clang-tools; + + buildInputs = attrs.buildInputs or [] + ++ [ + pkgs.gtest + pkgs.rapidcheck + ] + ++ lib.optional havePerl pkgs.perl + ; }); in forAllSystems (system: @@ -413,8 +362,8 @@ in (makeShells "native" nixpkgsFor.${system}.native) // (lib.optionalAttrs (!nixpkgsFor.${system}.native.stdenv.isDarwin) - (makeShells "static" nixpkgsFor.${system}.static)) // - (lib.genAttrs shellCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv)) // + (makeShells "static" nixpkgsFor.${system}.static) // + (forAllCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv))) // { default = self.devShells.${system}.native-stdenvPackages; } diff --git a/local.mk b/local.mk index 3f3abb9f0..b27c7031e 100644 --- a/local.mk +++ b/local.mk @@ -2,9 +2,14 @@ GLOBAL_CXXFLAGS += -Wno-deprecated-declarations -Werror=switch # Allow switch-enum to be overridden for files that do not support it, usually because of dependency headers. ERROR_SWITCH_ENUM = -Werror=switch-enum -$(foreach i, config.h $(wildcard src/lib*/*.hh), \ +$(foreach i, config.h $(wildcard src/lib*/*.hh) $(filter-out %_internal.h, $(wildcard src/lib*c/*.h)), \ $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) +ifdef HOST_UNIX + $(foreach i, $(wildcard src/lib*/unix/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) +endif + $(GCH): src/libutil/util.hh config.h -GCH_CXXFLAGS = -I src/libutil +GCH_CXXFLAGS = $(INCLUDE_libutil) diff --git a/m4/gcc_bug_80431.m4 b/m4/gcc_bug_80431.m4 index e42f01956..cdc4ddb40 100644 --- a/m4/gcc_bug_80431.m4 +++ b/m4/gcc_bug_80431.m4 @@ -46,11 +46,13 @@ AC_DEFUN([ENSURE_NO_GCC_BUG_80431], ]])], [status_80431=0], [status_80431=$?], - [ - # Assume we're bug-free when cross-compiling - ]) + [status_80431='']) AC_LANG_POP(C++) AS_CASE([$status_80431], + [''],[ + AC_MSG_RESULT(cannot check because cross compiling) + AC_MSG_NOTICE(assume we are bug free) + ], [0],[ AC_MSG_RESULT(yes) ], diff --git a/maintainers/README.md b/maintainers/README.md index fa321c7c0..b92833497 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -30,17 +30,18 @@ We aim to achieve this by improving the contributor experience and attracting mo ## Members - Eelco Dolstra (@edolstra) – Team lead -- Théophane Hufschmitt (@thufschmitt) - Valentin Gagarin (@fricklerhandwerk) - Thomas Bereknyei (@tomberek) - Robert Hensing (@roberth) - John Ericson (@Ericson2314) +The team is on Github as [@NixOS/nix-team](https://github.com/orgs/NixOS/teams/nix-team). + ## Meeting protocol -The team meets twice a week: +The team meets twice a week (times are denoted in the [Europe/Amsterdam](https://en.m.wikipedia.org/wiki/Time_in_the_Netherlands) time zone): -- Discussion meeting: [Fridays 13:00-14:00 CET](https://calendar.google.com/calendar/event?eid=MHNtOGVuNWtrZXNpZHR2bW1sM3QyN2ZjaGNfMjAyMjExMjVUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) +- Discussion meeting: [Wednesday 21:00-22:00 Europe/Amsterdam](https://www.google.com/calendar/event?eid=ZG5rZzNyajRjajducGV2NGY5aGkzYWIwdnJfMjAyNDA1MDhUMTkwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) 1. Triage issues and pull requests from the [No Status](#no-status) column (30 min) 2. Discuss issues and pull requests from the [To discuss](#to-discuss) column (30 min). @@ -49,15 +50,19 @@ The team meets twice a week: - mark it as draft if it is blocked on the contributor - escalate it back to the team by moving it to To discuss, and leaving a comment as to why the issue needs to be discussed again. -- Work meeting: [Mondays 13:00-15:00 CET](https://calendar.google.com/calendar/event?eid=NTM1MG1wNGJnOGpmOTZhYms3bTB1bnY5cWxfMjAyMjExMjFUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) +- Work meeting: [Mondays 14:00-16:00 Europe/Amsterdam](https://www.google.com/calendar/event?eid=Ym52NDdzYnRic2NzcDcybjZiNDhpNzhpa3NfMjAyNDA1MTNUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) 1. Code review on pull requests from [In review](#in-review). 2. Other chores and tasks. Meeting notes are collected on a [collaborative scratchpad](https://pad.lassul.us/Cv7FpYx-Ri-4VjUykQOLAw). -Notes on issues and pull requests are posted as comments and linked from the meeting notes, so they are easy to find from both places. +Notes on issues and pull requests are posted as comments and linked from the meeting notes, so they are can be found from both places. [All meeting notes](https://discourse.nixos.org/search?expanded=true&q=Nix%20team%20meeting%20minutes%20%23%20%23dev%3Anix%20in%3Atitle%20order%3Alatest_topic) are published on Discourse under the [Nix category](https://discourse.nixos.org/c/dev/nix/50). +Team meetings are generally open to anyone interested. +We can make exceptions to discuss sensitive issues, such as security incidents or people matters. +Contact any team member to get a calendar invite for reminders and updates. + ## Project board protocol The team uses a [GitHub project board](https://github.com/orgs/NixOS/projects/19/views/1) for tracking its work. diff --git a/maintainers/data/release-credits-email-to-handle.json b/maintainers/data/release-credits-email-to-handle.json new file mode 100644 index 000000000..cddc1a6e7 --- /dev/null +++ b/maintainers/data/release-credits-email-to-handle.json @@ -0,0 +1,52 @@ +{ + "bogus": "bogus", + "edolstra@gmail.com": "edolstra", + "roberth@users.noreply.github.com": "roberth", + "toscano.pino@tiscali.it": "pinotree", + "valentin@gagarin.work": "fricklerhandwerk", + "mr.trubach@icloud.com": "tie", + "robert@roberthensing.nl": "roberth", + "lix@jade.fyi": "lf-", + "cole.e.helbling@outlook.com": "cole-h", + "joerg@thalheim.io": "Mic92", + "John.Ericson@Obsidian.Systems": "Ericson2314", + "ryan.hendrickson@alum.mit.edu": "rhendric", + "67135060+poweredbypie@users.noreply.github.com": "poweredbypie", + "detroyejr@outlook.com": "detroyejr", + "silvan.mosberger@tweag.io": "infinisil", + "vcs@emily.moe": "emilazy", + "farid.m.zakaria@gmail.com": "fzakaria", + "22859658+RTUnreal@users.noreply.github.com": "RTUnreal", + "me@las.rs": "L-as", + "philip.taron@gmail.com": "philiptaron", + "root@goldstein.rs": "GoldsteinE", + "tomberek@users.noreply.github.com": "tomberek", + "lexi.mattick@neuralink.com": "kognise", + "andrew@johnandrewmarshall.com": "amarshall", + "contact@romain-neil.fr": "romain-neil", + "Mic92@users.noreply.github.com": "Mic92", + "valentin.gagarin@tweag.io": "fricklerhandwerk", + "siddhantk232@gmail.com": "siddhantk232", + "kn@openbsd.org": "klemensn", + "slyich@gmail.com": "trofi", + "theophane.hufschmitt@tweag.io": "thufschmitt", + "alicebob@lijzij.de": "alicebob", + "winter@winter.cafe": "winterqt", + "brian@brianmckenna.org": "puffnfresh", + "git@haenoe.party": "haenoe", + "peshogo@gmail.com": "pineapplehunter", + "poweredbypie@users.noreply.github.com": "poweredbypie", + "arthur200126@gmail.com": "Artoria2e5", + "tomberek@gmail.com": "tomberek", + "jaredbaur@fastmail.com": "jmbaur", + "andreas@rammhold.de": "andir", + "hamirmahal@gmail.com": "hamirmahal", + "git@JohnEricson.me": "Ericson2314", + "8763518+SkamDart@users.noreply.github.com": "SkamDart", + "kirillrdy@gmail.com": "kirillrdy", + "pennae@lix.systems": "pennae", + "delroth@gmail.com": "delroth", + "enno@nerdworks.de": "elohmeier", + "mjbauer95@gmail.com": "matthewbauer", + "MostAwesomeDude@gmail.com": "MostAwesomeDude" +} \ No newline at end of file diff --git a/maintainers/data/release-credits-handle-to-name.json b/maintainers/data/release-credits-handle-to-name.json new file mode 100644 index 000000000..abf9ed05b --- /dev/null +++ b/maintainers/data/release-credits-handle-to-name.json @@ -0,0 +1,45 @@ +{ + "fzakaria": "Farid Zakaria", + "kognise": "Lexi Mattick", + "L-as": "Las Safin", + "haenoe": "HaeNoe", + "andir": "Andreas Rammhold", + "matthewbauer": "Matthew Bauer", + "emilazy": "Emily", + "pineapplehunter": "Shogo Takata", + "RTUnreal": null, + "jmbaur": "Jared Baur", + "Ericson2314": "John Ericson", + "pinotree": "Pino Toscano", + "tie": "Ivan Trubach", + "poweredbypie": null, + "fricklerhandwerk": "Valentin Gagarin", + "Mic92": "J\u00f6rg Thalheim", + "alicebob": "Harmen", + "elohmeier": "Enno Richter", + "delroth": "Pierre Bourdon", + "kirillrdy": null, + "thufschmitt": "Th\u00e9ophane Hufschmitt", + "detroyejr": "Jonathan De Troye", + "klemensn": "Klemens Nanni", + "tomberek": null, + "rhendric": "Ryan Hendrickson", + "philiptaron": "Philip Taron", + "puffnfresh": "Brian McKenna", + "lf-": "jade", + "romain-neil": "Romain Neil", + "hamirmahal": "Hamir Mahal", + "edolstra": "Eelco Dolstra", + "Artoria2e5": "Mingye Wang", + "SkamDart": "Cameron", + "roberth": "Robert Hensing", + "amarshall": "Andrew Marshall", + "trofi": "Sergei Trofimovich", + "cole-h": "Cole Helbling", + "infinisil": "Silvan Mosberger", + "siddhantk232": "Siddhant Kumar", + "winterqt": "Winter", + "GoldsteinE": "Max \u201cGoldstein\u201d Siling", + "pennae": null, + "MostAwesomeDude": "Corbin Simpson" +} \ No newline at end of file diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix new file mode 100644 index 000000000..be91df536 --- /dev/null +++ b/maintainers/flake-module.nix @@ -0,0 +1,677 @@ +{ lib, getSystem, inputs, ... }: + +{ + imports = [ + inputs.git-hooks-nix.flakeModule + ]; + + perSystem = { config, pkgs, ... }: { + + # https://flake.parts/options/pre-commit-hooks-nix.html#options + pre-commit.settings = { + hooks = { + clang-format = { + enable = true; + excludes = [ + # We don't want to format test data + # ''tests/(?!nixos/).*\.nix'' + ''^tests/unit/[^/]*/data/.*$'' + + # Don't format vendored code + ''^doc/manual/redirects\.js$'' + ''^doc/manual/theme/highlight\.js$'' + + # We haven't applied formatting to these files yet + ''^doc/manual/redirects\.js$'' + ''^doc/manual/theme/highlight\.js$'' + ''^precompiled-headers\.h$'' + ''^src/build-remote/build-remote\.cc$'' + ''^src/libcmd/built-path\.cc$'' + ''^src/libcmd/built-path\.hh$'' + ''^src/libcmd/command\.cc$'' + ''^src/libcmd/command\.hh$'' + ''^src/libcmd/common-eval-args\.cc$'' + ''^src/libcmd/common-eval-args\.hh$'' + ''^src/libcmd/editor-for\.cc$'' + ''^src/libcmd/installable-attr-path\.cc$'' + ''^src/libcmd/installable-attr-path\.hh$'' + ''^src/libcmd/installable-derived-path\.cc$'' + ''^src/libcmd/installable-derived-path\.hh$'' + ''^src/libcmd/installable-flake\.cc$'' + ''^src/libcmd/installable-flake\.hh$'' + ''^src/libcmd/installable-value\.cc$'' + ''^src/libcmd/installable-value\.hh$'' + ''^src/libcmd/installables\.cc$'' + ''^src/libcmd/installables\.hh$'' + ''^src/libcmd/legacy\.hh$'' + ''^src/libcmd/markdown\.cc$'' + ''^src/libcmd/misc-store-flags\.cc$'' + ''^src/libcmd/repl-interacter\.cc$'' + ''^src/libcmd/repl-interacter\.hh$'' + ''^src/libcmd/repl\.cc$'' + ''^src/libcmd/repl\.hh$'' + ''^src/libexpr-c/nix_api_expr\.cc$'' + ''^src/libexpr-c/nix_api_external\.cc$'' + ''^src/libexpr/attr-path\.cc$'' + ''^src/libexpr/attr-path\.hh$'' + ''^src/libexpr/attr-set\.cc$'' + ''^src/libexpr/attr-set\.hh$'' + ''^src/libexpr/eval-cache\.cc$'' + ''^src/libexpr/eval-cache\.hh$'' + ''^src/libexpr/eval-error\.cc$'' + ''^src/libexpr/eval-inline\.hh$'' + ''^src/libexpr/eval-settings\.cc$'' + ''^src/libexpr/eval-settings\.hh$'' + ''^src/libexpr/eval\.cc$'' + ''^src/libexpr/eval\.hh$'' + ''^src/libexpr/function-trace\.cc$'' + ''^src/libexpr/gc-small-vector\.hh$'' + ''^src/libexpr/get-drvs\.cc$'' + ''^src/libexpr/get-drvs\.hh$'' + ''^src/libexpr/json-to-value\.cc$'' + ''^src/libexpr/nixexpr\.cc$'' + ''^src/libexpr/nixexpr\.hh$'' + ''^src/libexpr/parser-state\.hh$'' + ''^src/libexpr/pos-table\.hh$'' + ''^src/libexpr/primops\.cc$'' + ''^src/libexpr/primops\.hh$'' + ''^src/libexpr/primops/context\.cc$'' + ''^src/libexpr/primops/fetchClosure\.cc$'' + ''^src/libexpr/primops/fetchMercurial\.cc$'' + ''^src/libexpr/primops/fetchTree\.cc$'' + ''^src/libexpr/primops/fromTOML\.cc$'' + ''^src/libexpr/print-ambiguous\.cc$'' + ''^src/libexpr/print-ambiguous\.hh$'' + ''^src/libexpr/print-options\.hh$'' + ''^src/libexpr/print\.cc$'' + ''^src/libexpr/print\.hh$'' + ''^src/libexpr/search-path\.cc$'' + ''^src/libexpr/symbol-table\.hh$'' + ''^src/libexpr/value-to-json\.cc$'' + ''^src/libexpr/value-to-json\.hh$'' + ''^src/libexpr/value-to-xml\.cc$'' + ''^src/libexpr/value-to-xml\.hh$'' + ''^src/libexpr/value\.hh$'' + ''^src/libexpr/value/context\.cc$'' + ''^src/libexpr/value/context\.hh$'' + ''^src/libfetchers/attrs\.cc$'' + ''^src/libfetchers/cache\.cc$'' + ''^src/libfetchers/cache\.hh$'' + ''^src/libfetchers/fetch-settings\.cc$'' + ''^src/libfetchers/fetch-settings\.hh$'' + ''^src/libfetchers/fetch-to-store\.cc$'' + ''^src/libfetchers/fetchers\.cc$'' + ''^src/libfetchers/fetchers\.hh$'' + ''^src/libfetchers/filtering-source-accessor\.cc$'' + ''^src/libfetchers/filtering-source-accessor\.hh$'' + ''^src/libfetchers/fs-source-accessor\.cc$'' + ''^src/libfetchers/fs-source-accessor\.hh$'' + ''^src/libfetchers/git-utils\.cc$'' + ''^src/libfetchers/git-utils\.hh$'' + ''^src/libfetchers/github\.cc$'' + ''^src/libfetchers/indirect\.cc$'' + ''^src/libfetchers/memory-source-accessor\.cc$'' + ''^src/libfetchers/path\.cc$'' + ''^src/libfetchers/registry\.cc$'' + ''^src/libfetchers/registry\.hh$'' + ''^src/libfetchers/tarball\.cc$'' + ''^src/libfetchers/tarball\.hh$'' + ''^src/libfetchers/git\.cc$'' + ''^src/libfetchers/mercurial\.cc$'' + ''^src/libflake/flake/config\.cc$'' + ''^src/libflake/flake/flake\.cc$'' + ''^src/libflake/flake/flake\.hh$'' + ''^src/libflake/flake/flakeref\.cc$'' + ''^src/libflake/flake/flakeref\.hh$'' + ''^src/libflake/flake/lockfile\.cc$'' + ''^src/libflake/flake/lockfile\.hh$'' + ''^src/libflake/flake/url-name\.cc$'' + ''^src/libmain/common-args\.cc$'' + ''^src/libmain/common-args\.hh$'' + ''^src/libmain/loggers\.cc$'' + ''^src/libmain/loggers\.hh$'' + ''^src/libmain/progress-bar\.cc$'' + ''^src/libmain/shared\.cc$'' + ''^src/libmain/shared\.hh$'' + ''^src/libmain/unix/stack\.cc$'' + ''^src/libstore/binary-cache-store\.cc$'' + ''^src/libstore/binary-cache-store\.hh$'' + ''^src/libstore/build-result\.hh$'' + ''^src/libstore/builtins\.hh$'' + ''^src/libstore/builtins/buildenv\.cc$'' + ''^src/libstore/builtins/buildenv\.hh$'' + ''^src/libstore/common-protocol-impl\.hh$'' + ''^src/libstore/common-protocol\.cc$'' + ''^src/libstore/common-protocol\.hh$'' + ''^src/libstore/common-ssh-store-config\.hh$'' + ''^src/libstore/content-address\.cc$'' + ''^src/libstore/content-address\.hh$'' + ''^src/libstore/daemon\.cc$'' + ''^src/libstore/daemon\.hh$'' + ''^src/libstore/derivations\.cc$'' + ''^src/libstore/derivations\.hh$'' + ''^src/libstore/derived-path-map\.cc$'' + ''^src/libstore/derived-path-map\.hh$'' + ''^src/libstore/derived-path\.cc$'' + ''^src/libstore/derived-path\.hh$'' + ''^src/libstore/downstream-placeholder\.cc$'' + ''^src/libstore/downstream-placeholder\.hh$'' + ''^src/libstore/dummy-store\.cc$'' + ''^src/libstore/export-import\.cc$'' + ''^src/libstore/filetransfer\.cc$'' + ''^src/libstore/filetransfer\.hh$'' + ''^src/libstore/gc-store\.hh$'' + ''^src/libstore/globals\.cc$'' + ''^src/libstore/globals\.hh$'' + ''^src/libstore/http-binary-cache-store\.cc$'' + ''^src/libstore/legacy-ssh-store\.cc$'' + ''^src/libstore/legacy-ssh-store\.hh$'' + ''^src/libstore/length-prefixed-protocol-helper\.hh$'' + ''^src/libstore/linux/personality\.cc$'' + ''^src/libstore/linux/personality\.hh$'' + ''^src/libstore/local-binary-cache-store\.cc$'' + ''^src/libstore/local-fs-store\.cc$'' + ''^src/libstore/local-fs-store\.hh$'' + ''^src/libstore/log-store\.cc$'' + ''^src/libstore/log-store\.hh$'' + ''^src/libstore/machines\.cc$'' + ''^src/libstore/machines\.hh$'' + ''^src/libstore/make-content-addressed\.cc$'' + ''^src/libstore/make-content-addressed\.hh$'' + ''^src/libstore/misc\.cc$'' + ''^src/libstore/names\.cc$'' + ''^src/libstore/names\.hh$'' + ''^src/libstore/nar-accessor\.cc$'' + ''^src/libstore/nar-accessor\.hh$'' + ''^src/libstore/nar-info-disk-cache\.cc$'' + ''^src/libstore/nar-info-disk-cache\.hh$'' + ''^src/libstore/nar-info\.cc$'' + ''^src/libstore/nar-info\.hh$'' + ''^src/libstore/outputs-spec\.cc$'' + ''^src/libstore/outputs-spec\.hh$'' + ''^src/libstore/parsed-derivations\.cc$'' + ''^src/libstore/path-info\.cc$'' + ''^src/libstore/path-info\.hh$'' + ''^src/libstore/path-references\.cc$'' + ''^src/libstore/path-regex\.hh$'' + ''^src/libstore/path-with-outputs\.cc$'' + ''^src/libstore/path\.cc$'' + ''^src/libstore/path\.hh$'' + ''^src/libstore/pathlocks\.cc$'' + ''^src/libstore/pathlocks\.hh$'' + ''^src/libstore/profiles\.cc$'' + ''^src/libstore/profiles\.hh$'' + ''^src/libstore/realisation\.cc$'' + ''^src/libstore/realisation\.hh$'' + ''^src/libstore/remote-fs-accessor\.cc$'' + ''^src/libstore/remote-fs-accessor\.hh$'' + ''^src/libstore/remote-store-connection\.hh$'' + ''^src/libstore/remote-store\.cc$'' + ''^src/libstore/remote-store\.hh$'' + ''^src/libstore/s3-binary-cache-store\.cc$'' + ''^src/libstore/s3\.hh$'' + ''^src/libstore/serve-protocol-impl\.cc$'' + ''^src/libstore/serve-protocol-impl\.hh$'' + ''^src/libstore/serve-protocol\.cc$'' + ''^src/libstore/serve-protocol\.hh$'' + ''^src/libstore/sqlite\.cc$'' + ''^src/libstore/sqlite\.hh$'' + ''^src/libstore/ssh-store\.cc$'' + ''^src/libstore/ssh\.cc$'' + ''^src/libstore/ssh\.hh$'' + ''^src/libstore/store-api\.cc$'' + ''^src/libstore/store-api\.hh$'' + ''^src/libstore/store-dir-config\.hh$'' + ''^src/libstore/build/derivation-goal\.cc$'' + ''^src/libstore/build/derivation-goal\.hh$'' + ''^src/libstore/build/drv-output-substitution-goal\.cc$'' + ''^src/libstore/build/drv-output-substitution-goal\.hh$'' + ''^src/libstore/build/entry-points\.cc$'' + ''^src/libstore/build/goal\.cc$'' + ''^src/libstore/build/goal\.hh$'' + ''^src/libstore/unix/build/hook-instance\.cc$'' + ''^src/libstore/unix/build/local-derivation-goal\.cc$'' + ''^src/libstore/unix/build/local-derivation-goal\.hh$'' + ''^src/libstore/build/substitution-goal\.cc$'' + ''^src/libstore/build/substitution-goal\.hh$'' + ''^src/libstore/build/worker\.cc$'' + ''^src/libstore/build/worker\.hh$'' + ''^src/libstore/builtins/fetchurl\.cc$'' + ''^src/libstore/builtins/unpack-channel\.cc$'' + ''^src/libstore/gc\.cc$'' + ''^src/libstore/local-overlay-store\.cc$'' + ''^src/libstore/local-overlay-store\.hh$'' + ''^src/libstore/local-store\.cc$'' + ''^src/libstore/local-store\.hh$'' + ''^src/libstore/unix/user-lock\.cc$'' + ''^src/libstore/unix/user-lock\.hh$'' + ''^src/libstore/optimise-store\.cc$'' + ''^src/libstore/unix/pathlocks\.cc$'' + ''^src/libstore/posix-fs-canonicalise\.cc$'' + ''^src/libstore/posix-fs-canonicalise\.hh$'' + ''^src/libstore/uds-remote-store\.cc$'' + ''^src/libstore/uds-remote-store\.hh$'' + ''^src/libstore/windows/build\.cc$'' + ''^src/libstore/worker-protocol-impl\.hh$'' + ''^src/libstore/worker-protocol\.cc$'' + ''^src/libstore/worker-protocol\.hh$'' + ''^src/libutil-c/nix_api_util_internal\.h$'' + ''^src/libutil/archive\.cc$'' + ''^src/libutil/archive\.hh$'' + ''^src/libutil/args\.cc$'' + ''^src/libutil/args\.hh$'' + ''^src/libutil/args/root\.hh$'' + ''^src/libutil/callback\.hh$'' + ''^src/libutil/canon-path\.cc$'' + ''^src/libutil/canon-path\.hh$'' + ''^src/libutil/chunked-vector\.hh$'' + ''^src/libutil/closure\.hh$'' + ''^src/libutil/comparator\.hh$'' + ''^src/libutil/compute-levels\.cc$'' + ''^src/libutil/config-impl\.hh$'' + ''^src/libutil/config\.cc$'' + ''^src/libutil/config\.hh$'' + ''^src/libutil/current-process\.cc$'' + ''^src/libutil/current-process\.hh$'' + ''^src/libutil/english\.cc$'' + ''^src/libutil/english\.hh$'' + ''^src/libutil/environment-variables\.cc$'' + ''^src/libutil/error\.cc$'' + ''^src/libutil/error\.hh$'' + ''^src/libutil/exit\.hh$'' + ''^src/libutil/experimental-features\.cc$'' + ''^src/libutil/experimental-features\.hh$'' + ''^src/libutil/file-content-address\.cc$'' + ''^src/libutil/file-content-address\.hh$'' + ''^src/libutil/file-descriptor\.cc$'' + ''^src/libutil/file-descriptor\.hh$'' + ''^src/libutil/file-path-impl\.hh$'' + ''^src/libutil/file-path\.hh$'' + ''^src/libutil/file-system\.cc$'' + ''^src/libutil/file-system\.hh$'' + ''^src/libutil/finally\.hh$'' + ''^src/libutil/fmt\.hh$'' + ''^src/libutil/fs-sink\.cc$'' + ''^src/libutil/fs-sink\.hh$'' + ''^src/libutil/git\.cc$'' + ''^src/libutil/git\.hh$'' + ''^src/libutil/hash\.cc$'' + ''^src/libutil/hash\.hh$'' + ''^src/libutil/hilite\.cc$'' + ''^src/libutil/hilite\.hh$'' + ''^src/libutil/source-accessor\.hh$'' + ''^src/libutil/json-impls\.hh$'' + ''^src/libutil/json-utils\.cc$'' + ''^src/libutil/json-utils\.hh$'' + ''^src/libutil/linux/cgroup\.cc$'' + ''^src/libutil/linux/namespaces\.cc$'' + ''^src/libutil/logging\.cc$'' + ''^src/libutil/logging\.hh$'' + ''^src/libutil/lru-cache\.hh$'' + ''^src/libutil/memory-source-accessor\.cc$'' + ''^src/libutil/memory-source-accessor\.hh$'' + ''^src/libutil/pool\.hh$'' + ''^src/libutil/position\.cc$'' + ''^src/libutil/position\.hh$'' + ''^src/libutil/posix-source-accessor\.cc$'' + ''^src/libutil/posix-source-accessor\.hh$'' + ''^src/libutil/processes\.hh$'' + ''^src/libutil/ref\.hh$'' + ''^src/libutil/references\.cc$'' + ''^src/libutil/references\.hh$'' + ''^src/libutil/regex-combinators\.hh$'' + ''^src/libutil/serialise\.cc$'' + ''^src/libutil/serialise\.hh$'' + ''^src/libutil/signals\.hh$'' + ''^src/libutil/signature/local-keys\.cc$'' + ''^src/libutil/signature/local-keys\.hh$'' + ''^src/libutil/signature/signer\.cc$'' + ''^src/libutil/signature/signer\.hh$'' + ''^src/libutil/source-accessor\.cc$'' + ''^src/libutil/source-accessor\.hh$'' + ''^src/libutil/source-path\.cc$'' + ''^src/libutil/source-path\.hh$'' + ''^src/libutil/split\.hh$'' + ''^src/libutil/suggestions\.cc$'' + ''^src/libutil/suggestions\.hh$'' + ''^src/libutil/sync\.hh$'' + ''^src/libutil/terminal\.cc$'' + ''^src/libutil/terminal\.hh$'' + ''^src/libutil/thread-pool\.cc$'' + ''^src/libutil/thread-pool\.hh$'' + ''^src/libutil/topo-sort\.hh$'' + ''^src/libutil/types\.hh$'' + ''^src/libutil/unix/file-descriptor\.cc$'' + ''^src/libutil/unix/file-path\.cc$'' + ''^src/libutil/unix/monitor-fd\.hh$'' + ''^src/libutil/unix/processes\.cc$'' + ''^src/libutil/unix/signals-impl\.hh$'' + ''^src/libutil/unix/signals\.cc$'' + ''^src/libutil/unix-domain-socket\.cc$'' + ''^src/libutil/unix/users\.cc$'' + ''^src/libutil/url-parts\.hh$'' + ''^src/libutil/url\.cc$'' + ''^src/libutil/url\.hh$'' + ''^src/libutil/users\.cc$'' + ''^src/libutil/users\.hh$'' + ''^src/libutil/util\.cc$'' + ''^src/libutil/util\.hh$'' + ''^src/libutil/variant-wrapper\.hh$'' + ''^src/libutil/windows/environment-variables\.cc$'' + ''^src/libutil/windows/file-descriptor\.cc$'' + ''^src/libutil/windows/file-path\.cc$'' + ''^src/libutil/windows/processes\.cc$'' + ''^src/libutil/windows/users\.cc$'' + ''^src/libutil/windows/windows-error\.cc$'' + ''^src/libutil/windows/windows-error\.hh$'' + ''^src/libutil/xml-writer\.cc$'' + ''^src/libutil/xml-writer\.hh$'' + ''^src/nix-build/nix-build\.cc$'' + ''^src/nix-channel/nix-channel\.cc$'' + ''^src/nix-collect-garbage/nix-collect-garbage\.cc$'' + ''^src/nix-env/buildenv.nix$'' + ''^src/nix-env/nix-env\.cc$'' + ''^src/nix-env/user-env\.cc$'' + ''^src/nix-env/user-env\.hh$'' + ''^src/nix-instantiate/nix-instantiate\.cc$'' + ''^src/nix-store/dotgraph\.cc$'' + ''^src/nix-store/graphml\.cc$'' + ''^src/nix-store/nix-store\.cc$'' + ''^src/nix/add-to-store\.cc$'' + ''^src/nix/app\.cc$'' + ''^src/nix/build\.cc$'' + ''^src/nix/bundle\.cc$'' + ''^src/nix/cat\.cc$'' + ''^src/nix/config-check\.cc$'' + ''^src/nix/config\.cc$'' + ''^src/nix/copy\.cc$'' + ''^src/nix/derivation-add\.cc$'' + ''^src/nix/derivation-show\.cc$'' + ''^src/nix/derivation\.cc$'' + ''^src/nix/develop\.cc$'' + ''^src/nix/diff-closures\.cc$'' + ''^src/nix/dump-path\.cc$'' + ''^src/nix/edit\.cc$'' + ''^src/nix/eval\.cc$'' + ''^src/nix/flake\.cc$'' + ''^src/nix/fmt\.cc$'' + ''^src/nix/hash\.cc$'' + ''^src/nix/log\.cc$'' + ''^src/nix/ls\.cc$'' + ''^src/nix/main\.cc$'' + ''^src/nix/make-content-addressed\.cc$'' + ''^src/nix/nar\.cc$'' + ''^src/nix/optimise-store\.cc$'' + ''^src/nix/path-from-hash-part\.cc$'' + ''^src/nix/path-info\.cc$'' + ''^src/nix/prefetch\.cc$'' + ''^src/nix/profile\.cc$'' + ''^src/nix/realisation\.cc$'' + ''^src/nix/registry\.cc$'' + ''^src/nix/repl\.cc$'' + ''^src/nix/run\.cc$'' + ''^src/nix/run\.hh$'' + ''^src/nix/search\.cc$'' + ''^src/nix/sigs\.cc$'' + ''^src/nix/store-copy-log\.cc$'' + ''^src/nix/store-delete\.cc$'' + ''^src/nix/store-gc\.cc$'' + ''^src/nix/store-info\.cc$'' + ''^src/nix/store-repair\.cc$'' + ''^src/nix/store\.cc$'' + ''^src/nix/unix/daemon\.cc$'' + ''^src/nix/upgrade-nix\.cc$'' + ''^src/nix/verify\.cc$'' + ''^src/nix/why-depends\.cc$'' + + ''^tests/functional/plugins/plugintest\.cc'' + ''^tests/functional/test-libstoreconsumer/main\.cc'' + ''^tests/nixos/ca-fd-leak/sender\.c'' + ''^tests/nixos/ca-fd-leak/smuggler\.c'' + ''^tests/nixos/user-sandboxing/attacker\.c'' + ''^tests/unit/libexpr-support/tests/libexpr\.hh'' + ''^tests/unit/libexpr-support/tests/value/context\.cc'' + ''^tests/unit/libexpr-support/tests/value/context\.hh'' + ''^tests/unit/libexpr/derived-path\.cc'' + ''^tests/unit/libexpr/error_traces\.cc'' + ''^tests/unit/libexpr/eval\.cc'' + ''^tests/unit/libexpr/json\.cc'' + ''^tests/unit/libexpr/main\.cc'' + ''^tests/unit/libexpr/primops\.cc'' + ''^tests/unit/libexpr/search-path\.cc'' + ''^tests/unit/libexpr/trivial\.cc'' + ''^tests/unit/libexpr/value/context\.cc'' + ''^tests/unit/libexpr/value/print\.cc'' + ''^tests/unit/libfetchers/public-key\.cc'' + ''^tests/unit/libflake/flakeref\.cc'' + ''^tests/unit/libflake/url-name\.cc'' + ''^tests/unit/libstore-support/tests/derived-path\.cc'' + ''^tests/unit/libstore-support/tests/derived-path\.hh'' + ''^tests/unit/libstore-support/tests/nix_api_store\.hh'' + ''^tests/unit/libstore-support/tests/outputs-spec\.cc'' + ''^tests/unit/libstore-support/tests/outputs-spec\.hh'' + ''^tests/unit/libstore-support/tests/path\.cc'' + ''^tests/unit/libstore-support/tests/path\.hh'' + ''^tests/unit/libstore-support/tests/protocol\.hh'' + ''^tests/unit/libstore/common-protocol\.cc'' + ''^tests/unit/libstore/content-address\.cc'' + ''^tests/unit/libstore/derivation\.cc'' + ''^tests/unit/libstore/derived-path\.cc'' + ''^tests/unit/libstore/downstream-placeholder\.cc'' + ''^tests/unit/libstore/machines\.cc'' + ''^tests/unit/libstore/nar-info-disk-cache\.cc'' + ''^tests/unit/libstore/nar-info\.cc'' + ''^tests/unit/libstore/outputs-spec\.cc'' + ''^tests/unit/libstore/path-info\.cc'' + ''^tests/unit/libstore/path\.cc'' + ''^tests/unit/libstore/serve-protocol\.cc'' + ''^tests/unit/libstore/worker-protocol\.cc'' + ''^tests/unit/libutil-support/tests/characterization\.hh'' + ''^tests/unit/libutil-support/tests/hash\.cc'' + ''^tests/unit/libutil-support/tests/hash\.hh'' + ''^tests/unit/libutil/args\.cc'' + ''^tests/unit/libutil/canon-path\.cc'' + ''^tests/unit/libutil/chunked-vector\.cc'' + ''^tests/unit/libutil/closure\.cc'' + ''^tests/unit/libutil/compression\.cc'' + ''^tests/unit/libutil/config\.cc'' + ''^tests/unit/libutil/file-content-address\.cc'' + ''^tests/unit/libutil/git\.cc'' + ''^tests/unit/libutil/hash\.cc'' + ''^tests/unit/libutil/hilite\.cc'' + ''^tests/unit/libutil/json-utils\.cc'' + ''^tests/unit/libutil/logging\.cc'' + ''^tests/unit/libutil/lru-cache\.cc'' + ''^tests/unit/libutil/pool\.cc'' + ''^tests/unit/libutil/references\.cc'' + ''^tests/unit/libutil/suggestions\.cc'' + ''^tests/unit/libutil/tests\.cc'' + ''^tests/unit/libutil/url\.cc'' + ''^tests/unit/libutil/xml-writer\.cc'' + ]; + }; + shellcheck = { + enable = true; + excludes = [ + # We haven't linted these files yet + ''^config/install-sh$'' + ''^misc/bash/completion\.sh$'' + ''^misc/fish/completion\.fish$'' + ''^misc/zsh/completion\.zsh$'' + ''^scripts/create-darwin-volume\.sh$'' + ''^scripts/install-darwin-multi-user\.sh$'' + ''^scripts/install-multi-user\.sh$'' + ''^scripts/install-nix-from-closure\.sh$'' + ''^scripts/install-systemd-multi-user\.sh$'' + ''^src/nix/get-env\.sh$'' + ''^tests/functional/build\.sh$'' + ''^tests/functional/ca/build-dry\.sh$'' + ''^tests/functional/ca/build-with-garbage-path\.sh$'' + ''^tests/functional/ca/common\.sh$'' + ''^tests/functional/ca/concurrent-builds\.sh$'' + ''^tests/functional/ca/eval-store\.sh$'' + ''^tests/functional/ca/gc\.sh$'' + ''^tests/functional/ca/import-derivation\.sh$'' + ''^tests/functional/ca/new-build-cmd\.sh$'' + ''^tests/functional/ca/nix-shell\.sh$'' + ''^tests/functional/ca/post-hook\.sh$'' + ''^tests/functional/ca/recursive\.sh$'' + ''^tests/functional/ca/repl\.sh$'' + ''^tests/functional/ca/selfref-gc\.sh$'' + ''^tests/functional/ca/why-depends\.sh$'' + ''^tests/functional/characterisation-test-infra\.sh$'' + ''^tests/functional/check\.sh$'' + ''^tests/functional/common/vars-and-functions\.sh$'' + ''^tests/functional/completions\.sh$'' + ''^tests/functional/compute-levels\.sh$'' + ''^tests/functional/config\.sh$'' + ''^tests/functional/db-migration\.sh$'' + ''^tests/functional/debugger\.sh$'' + ''^tests/functional/dependencies\.builder0\.sh$'' + ''^tests/functional/dependencies\.sh$'' + ''^tests/functional/dump-db\.sh$'' + ''^tests/functional/dyn-drv/build-built-drv\.sh$'' + ''^tests/functional/dyn-drv/common\.sh$'' + ''^tests/functional/dyn-drv/dep-built-drv\.sh$'' + ''^tests/functional/dyn-drv/eval-outputOf\.sh$'' + ''^tests/functional/dyn-drv/old-daemon-error-hack\.sh$'' + ''^tests/functional/dyn-drv/recursive-mod-json\.sh$'' + ''^tests/functional/eval-store\.sh$'' + ''^tests/functional/eval\.sh$'' + ''^tests/functional/export-graph\.sh$'' + ''^tests/functional/export\.sh$'' + ''^tests/functional/extra-sandbox-profile\.sh$'' + ''^tests/functional/fetchClosure\.sh$'' + ''^tests/functional/fetchGit\.sh$'' + ''^tests/functional/fetchGitRefs\.sh$'' + ''^tests/functional/fetchGitSubmodules\.sh$'' + ''^tests/functional/fetchGitVerification\.sh$'' + ''^tests/functional/fetchMercurial\.sh$'' + ''^tests/functional/fetchurl\.sh$'' + ''^tests/functional/fixed\.builder1\.sh$'' + ''^tests/functional/fixed\.builder2\.sh$'' + ''^tests/functional/fixed\.sh$'' + ''^tests/functional/flakes/absolute-paths\.sh$'' + ''^tests/functional/flakes/check\.sh$'' + ''^tests/functional/flakes/common\.sh$'' + ''^tests/functional/flakes/config\.sh$'' + ''^tests/functional/flakes/develop\.sh$'' + ''^tests/functional/flakes/flakes\.sh$'' + ''^tests/functional/flakes/follow-paths\.sh$'' + ''^tests/functional/flakes/prefetch\.sh$'' + ''^tests/functional/flakes/run\.sh$'' + ''^tests/functional/flakes/show\.sh$'' + ''^tests/functional/fmt\.sh$'' + ''^tests/functional/fmt\.simple\.sh$'' + ''^tests/functional/gc-auto\.sh$'' + ''^tests/functional/gc-concurrent\.builder\.sh$'' + ''^tests/functional/gc-concurrent\.sh$'' + ''^tests/functional/gc-concurrent2\.builder\.sh$'' + ''^tests/functional/gc-non-blocking\.sh$'' + ''^tests/functional/gc\.sh$'' + ''^tests/functional/git-hashing/common\.sh$'' + ''^tests/functional/git-hashing/simple\.sh$'' + ''^tests/functional/hash-convert\.sh$'' + ''^tests/functional/help\.sh$'' + ''^tests/functional/impure-derivations\.sh$'' + ''^tests/functional/impure-env\.sh$'' + ''^tests/functional/impure-eval\.sh$'' + ''^tests/functional/install-darwin\.sh$'' + ''^tests/functional/lang\.sh$'' + ''^tests/functional/legacy-ssh-store\.sh$'' + ''^tests/functional/linux-sandbox\.sh$'' + ''^tests/functional/local-overlay-store/add-lower-inner\.sh$'' + ''^tests/functional/local-overlay-store/add-lower\.sh$'' + ''^tests/functional/local-overlay-store/bad-uris\.sh$'' + ''^tests/functional/local-overlay-store/build-inner\.sh$'' + ''^tests/functional/local-overlay-store/build\.sh$'' + ''^tests/functional/local-overlay-store/check-post-init-inner\.sh$'' + ''^tests/functional/local-overlay-store/check-post-init\.sh$'' + ''^tests/functional/local-overlay-store/common\.sh$'' + ''^tests/functional/local-overlay-store/delete-duplicate-inner\.sh$'' + ''^tests/functional/local-overlay-store/delete-duplicate\.sh$'' + ''^tests/functional/local-overlay-store/delete-refs-inner\.sh$'' + ''^tests/functional/local-overlay-store/delete-refs\.sh$'' + ''^tests/functional/local-overlay-store/gc-inner\.sh$'' + ''^tests/functional/local-overlay-store/gc\.sh$'' + ''^tests/functional/local-overlay-store/optimise-inner\.sh$'' + ''^tests/functional/local-overlay-store/optimise\.sh$'' + ''^tests/functional/local-overlay-store/redundant-add-inner\.sh$'' + ''^tests/functional/local-overlay-store/redundant-add\.sh$'' + ''^tests/functional/local-overlay-store/remount\.sh$'' + ''^tests/functional/local-overlay-store/stale-file-handle-inner\.sh$'' + ''^tests/functional/local-overlay-store/stale-file-handle\.sh$'' + ''^tests/functional/local-overlay-store/verify-inner\.sh$'' + ''^tests/functional/local-overlay-store/verify\.sh$'' + ''^tests/functional/logging\.sh$'' + ''^tests/functional/misc\.sh$'' + ''^tests/functional/multiple-outputs\.sh$'' + ''^tests/functional/nar-access\.sh$'' + ''^tests/functional/nested-sandboxing\.sh$'' + ''^tests/functional/nested-sandboxing/command\.sh$'' + ''^tests/functional/nix-build\.sh$'' + ''^tests/functional/nix-channel\.sh$'' + ''^tests/functional/nix-collect-garbage-d\.sh$'' + ''^tests/functional/nix-copy-ssh-common\.sh$'' + ''^tests/functional/nix-copy-ssh-ng\.sh$'' + ''^tests/functional/nix-copy-ssh\.sh$'' + ''^tests/functional/nix-daemon-untrusting\.sh$'' + ''^tests/functional/nix-profile\.sh$'' + ''^tests/functional/nix-shell\.sh$'' + ''^tests/functional/nix_path\.sh$'' + ''^tests/functional/optimise-store\.sh$'' + ''^tests/functional/output-normalization\.sh$'' + ''^tests/functional/parallel\.builder\.sh$'' + ''^tests/functional/parallel\.sh$'' + ''^tests/functional/pass-as-file\.sh$'' + ''^tests/functional/path-from-hash-part\.sh$'' + ''^tests/functional/path-info\.sh$'' + ''^tests/functional/placeholders\.sh$'' + ''^tests/functional/plugins\.sh$'' + ''^tests/functional/post-hook\.sh$'' + ''^tests/functional/pure-eval\.sh$'' + ''^tests/functional/push-to-store-old\.sh$'' + ''^tests/functional/push-to-store\.sh$'' + ''^tests/functional/read-only-store\.sh$'' + ''^tests/functional/readfile-context\.sh$'' + ''^tests/functional/recursive\.sh$'' + ''^tests/functional/referrers\.sh$'' + ''^tests/functional/remote-store\.sh$'' + ''^tests/functional/repair\.sh$'' + ''^tests/functional/restricted\.sh$'' + ''^tests/functional/search\.sh$'' + ''^tests/functional/secure-drv-outputs\.sh$'' + ''^tests/functional/selfref-gc\.sh$'' + ''^tests/functional/shell\.sh$'' + ''^tests/functional/shell\.shebang\.sh$'' + ''^tests/functional/signing\.sh$'' + ''^tests/functional/simple\.builder\.sh$'' + ''^tests/functional/simple\.sh$'' + ''^tests/functional/ssh-relay\.sh$'' + ''^tests/functional/store-info\.sh$'' + ''^tests/functional/structured-attrs\.sh$'' + ''^tests/functional/substitute-with-invalid-ca\.sh$'' + ''^tests/functional/suggestions\.sh$'' + ''^tests/functional/supplementary-groups\.sh$'' + ''^tests/functional/tarball\.sh$'' + ''^tests/functional/test-infra\.sh$'' + ''^tests/functional/test-libstoreconsumer\.sh$'' + ''^tests/functional/timeout\.sh$'' + ''^tests/functional/toString-path\.sh$'' + ''^tests/functional/user-envs-migration\.sh$'' + ''^tests/functional/user-envs-test-case\.sh$'' + ''^tests/functional/user-envs\.builder\.sh$'' + ''^tests/functional/user-envs\.sh$'' + ''^tests/functional/why-depends\.sh$'' + ''^tests/functional/zstd\.sh$'' + ''^tests/unit/libutil/data/git/check-data\.sh$'' + ]; + }; + # TODO: nixfmt, https://github.com/NixOS/nixfmt/issues/153 + }; + }; + }; + + # We'll be pulling from this in the main flake + flake.getSystem = getSystem; +} diff --git a/maintainers/local.mk b/maintainers/local.mk new file mode 100644 index 000000000..88d594d67 --- /dev/null +++ b/maintainers/local.mk @@ -0,0 +1,15 @@ + +.PHONY: format +print-top-help += echo ' format: Format source code' + +# This uses the cached .pre-commit-hooks.yaml file +format: + @if ! type -p pre-commit &>/dev/null; then \ + echo "make format: pre-commit not found. Please use \`nix develop\`."; \ + exit 1; \ + fi; \ + if test -z "$$_NIX_PRE_COMMIT_HOOKS_CONFIG"; then \ + echo "make format: _NIX_PRE_COMMIT_HOOKS_CONFIG not set. Please use \`nix develop\`."; \ + exit 1; \ + fi; \ + pre-commit run --config $$_NIX_PRE_COMMIT_HOOKS_CONFIG --all-files diff --git a/maintainers/release-credits b/maintainers/release-credits new file mode 100755 index 000000000..7a5c87d7d --- /dev/null +++ b/maintainers/release-credits @@ -0,0 +1,184 @@ +#!/usr/bin/env nix +# vim: set filetype=python: +#!nix develop --impure --expr +#!nix `` +#!nix let flake = builtins.getFlake ("git+file://" + toString ../.); +#!nix pkgs = flake.inputs.nixpkgs.legacyPackages.${builtins.currentSystem}; +#!nix in pkgs.mkShell { nativeBuildInputs = [ +#!nix (pkgs.python3.withPackages (ps: with ps; [ requests ])) +#!nix ]; } +#!nix `` --command python3 + +# This script lists out the contributors for a given release. +# It must be run from the root of the Nix repository. + +import os +import sys +import json +import requests + +github_token = os.environ.get("GITHUB_TOKEN") +if not github_token: + print("GITHUB_TOKEN is not set. If you hit the rate limit, set it", file=sys.stderr) + # Might be ok, as we have a cache. + # raise ValueError("GITHUB_TOKEN must be set") + +# 1. Read the current version in .version +version = os.environ.get("VERSION") +if not version: + version = open(".version").read().strip() + +print(f"Generating release credits for Nix {version}", file=sys.stderr) + +# 2. Compute previous version +vcomponents = version.split(".") +if len(vcomponents) >= 2: + prev_version = f"{vcomponents[0]}.{int(vcomponents[1])-1}.0" +else: + raise ValueError(".version must have at least two components") + +# For unreleased versions +endref = "HEAD" +# For older releases +# endref = version + +# 2. Find the merge base between the current version and the previous version +mergeBase = os.popen(f"git merge-base {prev_version} {endref}").read().strip() +print(f"Merge base between {prev_version} and {endref} is {mergeBase}", file=sys.stderr) + +# 3. Find the date of the merge base +mergeBaseDate = os.popen(f"git show -s --format=%ci {mergeBase}").read().strip()[0:10] +print(f"Merge base date is {mergeBaseDate}", file=sys.stderr) + +# 4. Get the commits between the merge base and the current version + +def get_commits(): + raw = os.popen(f"git log --pretty=format:'%H\t%an\t%ae' {mergeBase}..{endref}").read().strip() + lines = raw.split("\n") + return [ { "hash": items[0], "author": items[1], "email": items[2] } + for line in lines + for items in (line.split("\t"),) + ] + +def commits_to_first_commit_by_email(commits): + by_email = dict() + for commit in commits: + email = commit["email"] + if email not in by_email: + by_email[email] = commit + return by_email + + +samples = commits_to_first_commit_by_email(get_commits()) + +# For quick testing, only pick two samples from the dict +# samples = dict(list(samples.items())[:2]) + +# Query the GitHub API to get handle +def get_github_commit(commit): + url = f"https://api.github.com/repos/NixOS/nix/commits/{commit['hash']}" + headers = {'Authorization': f'token {github_token}'} + response = requests.get(url, headers=headers) + response.raise_for_status() + return response.json() + +class Cache: + def __init__(self, filename, require = True): + self.filename = filename + try: + with open(filename, "r") as f: + self.values = json.load(f) + except FileNotFoundError: + if require: + raise + self.values = dict() + def save(self): + with open(self.filename, "w") as f: + json.dump(self.values, f, indent=4) + print(f"Saved cache to {self.filename}", file=sys.stderr) + +# The email to handle cache maps email addresses to either +# - a handle (string) +# - None (if no handle was found) +email_to_handle_cache = Cache("maintainers/data/release-credits-email-to-handle.json") + +handles = set() +emails = dict() + +for sample in samples: + s = samples[sample] + email = s["email"] + if not email in email_to_handle_cache.values: + print(f"Querying GitHub API for {s['hash']}, to get handle for {s['email']}") + ghc = get_github_commit(samples[sample]) + gha = ghc["author"] + if gha and gha["login"]: + handle = gha["login"] + print(f"Handle: {handle}") + email_to_handle_cache.values[email] = handle + else: + print(f"Found no handle for {s['email']}") + email_to_handle_cache.values[email] = None + handle = email_to_handle_cache.values[email] + if handle is not None: + handles.add(handle) + else: + emails[email] = s["author"] + +# print(email_to_handle_cache.values) + +email_to_handle_cache.save() + +handle_to_name_cache = Cache("maintainers/data/release-credits-handle-to-name.json") + +print(f"Found {len(handles)} handles", file=sys.stderr) + +for handle in handles: + if not handle in handle_to_name_cache.values: + print(f"Querying GitHub API for {handle}, to get name", file=sys.stderr) + url = f"https://api.github.com/users/{handle}" + headers = {'Authorization': f'token {github_token}'} + response = requests.get(url, headers=headers) + response.raise_for_status() + user = response.json() + name = user["name"] + print(f"Name: {name}", file=sys.stderr) + handle_to_name_cache.values[handle] = name + +handle_to_name_cache.save() + +entries = list() + +def is_bot(handle): + return handle.endswith("[bot]") + +for handle in handles: + name = handle_to_name_cache.values[handle] + if name is None: + # This way it looks more regular + name = handle + + # Note to bots: I'm only doing this because it would distract humans, and + # therefore delay your creation. I'm sorry; it's for your benefit. + if not is_bot(handle): + entries += [ f"- {name} [**(@{handle})**](https://github.com/{handle})" ] + +def shuffle(entries): + salt = os.urandom(16) + return sorted(entries, key=lambda x: hash((x, salt))) + +# Fair ordering is undecidable +entries = shuffle(entries) + +# For a sanity check, we could sort the entries by handle instead. +# entries = sorted(entries) + +print("") +print(f"This release was made possible by the following {len(entries)} contributors:") +print("") + +for entry in entries: + print(entry) + +for email in emails: + print(f"- {emails[email]}") diff --git a/maintainers/release-notes b/maintainers/release-notes index 2d84485c1..c0c4ee734 100755 --- a/maintainers/release-notes +++ b/maintainers/release-notes @@ -1,5 +1,6 @@ #!/usr/bin/env nix -#!nix shell .#changelog-d-nix --command bash +# vim: set filetype=bash: +#!nix shell .#changelog-d --command bash # --- CONFIGURATION --- @@ -151,6 +152,13 @@ section_title="Release $version_full ($DATE)" echo "# $section_title" echo changelog-d doc/manual/rl-next | sed -e 's/ *$//' + + if ! $IS_PATCH; then + echo + echo "# Contributors" + echo + VERSION=$version_full ./maintainers/release-credits + fi ) | tee -a $file log "Wrote $file" diff --git a/maintainers/release-process.md b/maintainers/release-process.md index da6886ea9..7a2b3c0a7 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -39,6 +39,10 @@ release: * Proof-read / edit / rearrange the release notes if needed. Breaking changes and highlights should go to the top. +* Run `maintainers/release-credits` to make sure the credits script works + and produces a sensible output. Some emails might not automatically map to + a GitHub handle. + * Push. ```console diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 4e2c379f0..731988568 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -11,6 +11,8 @@ use JSON::PP; use LWP::UserAgent; use Net::Amazon::S3; +delete $ENV{'shell'}; # shut up a LWP::UserAgent.pm warning + my $evalId = $ARGV[0] or die "Usage: $0 EVAL-ID\n"; my $releasesBucketName = "nix-releases"; @@ -36,11 +38,11 @@ sub fetch { my $evalUrl = "https://hydra.nixos.org/eval/$evalId"; my $evalInfo = decode_json(fetch($evalUrl, 'application/json')); #print Dumper($evalInfo); -my $flakeUrl = $evalInfo->{flake} or die; -my $flakeInfo = decode_json(`nix flake metadata --json "$flakeUrl"` or die); -my $nixRev = $flakeInfo->{revision} or die; +my $flakeUrl = $evalInfo->{flake}; +my $flakeInfo = decode_json(`nix flake metadata --json "$flakeUrl"` or die) if $flakeUrl; +my $nixRev = ($flakeInfo ? $flakeInfo->{revision} : $evalInfo->{jobsetevalinputs}->{nix}->{revision}) or die; -my $buildInfo = decode_json(fetch("$evalUrl/job/build.x86_64-linux", 'application/json')); +my $buildInfo = decode_json(fetch("$evalUrl/job/build.nix.x86_64-linux", 'application/json')); #print Dumper($buildInfo); my $releaseName = $buildInfo->{nixname}; @@ -83,12 +85,19 @@ my $channelsBucket = $s3_us->bucket($channelsBucketName) or die; sub getStorePath { my ($jobName, $output) = @_; my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json')); - return $buildInfo->{buildoutputs}->{$output or "out"}->{path} or die "cannot get store path for '$jobName'"; + return $buildInfo->{buildoutputs}->{$output or "out"}->{path} // die "cannot get store path for '$jobName'"; } sub copyManual { - my $manual = getStorePath("build.x86_64-linux", "doc"); - print "$manual\n"; + my $manual; + eval { + $manual = getStorePath("build.nix.x86_64-linux", "doc"); + }; + if ($@) { + warn "$@"; + return; + } + print "Manual: $manual\n"; my $manualNar = "$tmpDir/$releaseName-manual.nar.xz"; print "$manualNar\n"; @@ -154,19 +163,34 @@ downloadFile("binaryTarball.x86_64-linux", "1"); downloadFile("binaryTarball.aarch64-linux", "1"); downloadFile("binaryTarball.x86_64-darwin", "1"); downloadFile("binaryTarball.aarch64-darwin", "1"); -downloadFile("binaryTarballCross.x86_64-linux.armv6l-unknown-linux-gnueabihf", "1"); -downloadFile("binaryTarballCross.x86_64-linux.armv7l-unknown-linux-gnueabihf", "1"); +eval { + downloadFile("binaryTarballCross.x86_64-linux.armv6l-unknown-linux-gnueabihf", "1"); +}; +warn "$@" if $@; +eval { + downloadFile("binaryTarballCross.x86_64-linux.armv7l-unknown-linux-gnueabihf", "1"); +}; +warn "$@" if $@; +eval { + downloadFile("binaryTarballCross.x86_64-linux.riscv64-unknown-linux-gnu", "1"); +}; +warn "$@" if $@; downloadFile("installerScript", "1"); # Upload docker images to dockerhub. my $dockerManifest = ""; my $dockerManifestLatest = ""; +my $haveDocker = 0; for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) { my $system = $platforms->[0]; my $dockerPlatform = $platforms->[1]; my $fn = "nix-$version-docker-image-$dockerPlatform.tar.gz"; - downloadFile("dockerImage.$system", "1", $fn); + eval { + downloadFile("dockerImage.$system", "1", $fn); + }; + die "$@" if $@; + $haveDocker = 1; print STDERR "loading docker image for $dockerPlatform...\n"; system("docker load -i $tmpDir/$fn") == 0 or die; @@ -194,31 +218,34 @@ for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) { $dockerManifestLatest .= " --amend $latestTag" } -print STDERR "creating multi-platform docker manifest...\n"; -system("docker manifest rm nixos/nix:$version"); -system("docker manifest create nixos/nix:$version $dockerManifest") == 0 or die; -if ($isLatest) { - print STDERR "creating latest multi-platform docker manifest...\n"; - system("docker manifest rm nixos/nix:latest"); - system("docker manifest create nixos/nix:latest $dockerManifestLatest") == 0 or die; -} +if ($haveDocker) { + print STDERR "creating multi-platform docker manifest...\n"; + system("docker manifest rm nixos/nix:$version"); + system("docker manifest create nixos/nix:$version $dockerManifest") == 0 or die; + if ($isLatest) { + print STDERR "creating latest multi-platform docker manifest...\n"; + system("docker manifest rm nixos/nix:latest"); + system("docker manifest create nixos/nix:latest $dockerManifestLatest") == 0 or die; + } -print STDERR "pushing multi-platform docker manifest...\n"; -system("docker manifest push nixos/nix:$version") == 0 or die; + print STDERR "pushing multi-platform docker manifest...\n"; + system("docker manifest push nixos/nix:$version") == 0 or die; -if ($isLatest) { - print STDERR "pushing latest multi-platform docker manifest...\n"; - system("docker manifest push nixos/nix:latest") == 0 or die; + if ($isLatest) { + print STDERR "pushing latest multi-platform docker manifest...\n"; + system("docker manifest push nixos/nix:latest") == 0 or die; + } } # Upload nix-fallback-paths.nix. write_file("$tmpDir/fallback-paths.nix", "{\n" . - " x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" . - " i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" . - " aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" . - " x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" . - " aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" . + " x86_64-linux = \"" . getStorePath("build.nix.x86_64-linux") . "\";\n" . + " i686-linux = \"" . getStorePath("build.nix.i686-linux") . "\";\n" . + " aarch64-linux = \"" . getStorePath("build.nix.aarch64-linux") . "\";\n" . + " riscv64-linux = \"" . getStorePath("buildCross.nix.riscv64-unknown-linux-gnu.x86_64-linux") . "\";\n" . + " x86_64-darwin = \"" . getStorePath("build.nix.x86_64-darwin") . "\";\n" . + " aarch64-darwin = \"" . getStorePath("build.nix.aarch64-darwin") . "\";\n" . "}\n"); # Upload release files to S3. diff --git a/meson.build b/meson.build new file mode 100644 index 000000000..1554244ab --- /dev/null +++ b/meson.build @@ -0,0 +1,44 @@ +# This is just a stub project to include all the others as subprojects +# for development shell purposes + +project('nix-dev-shell', 'cpp', + version : files('.version'), + subproject_dir : 'src', +) + +# Internal Libraries +subproject('libutil') +subproject('libstore') +subproject('libfetchers') +subproject('libexpr') +subproject('libflake') +subproject('libmain') +subproject('libcmd') + +# Executables +subproject('nix') + +# Docs +subproject('internal-api-docs') +subproject('external-api-docs') + +# External C wrapper libraries +subproject('libutil-c') +subproject('libstore-c') +subproject('libexpr-c') +subproject('libmain-c') + +# Language Bindings +if not meson.is_cross_build() + subproject('perl') +endif + +# Testing +subproject('nix-util-test-support') +subproject('nix-util-tests') +subproject('nix-store-test-support') +subproject('nix-store-tests') +subproject('nix-fetchers-tests') +subproject('nix-expr-test-support') +subproject('nix-expr-tests') +subproject('nix-flake-tests') diff --git a/misc/changelog-d.cabal.nix b/misc/changelog-d.cabal.nix deleted file mode 100644 index 76f9353cd..000000000 --- a/misc/changelog-d.cabal.nix +++ /dev/null @@ -1,31 +0,0 @@ -{ mkDerivation, aeson, base, bytestring, cabal-install-parsers -, Cabal-syntax, containers, directory, filepath, frontmatter -, generic-lens-lite, lib, mtl, optparse-applicative, parsec, pretty -, regex-applicative, text, pkgs -}: -let rev = "f30f6969e9cd8b56242309639d58acea21c99d06"; -in -mkDerivation { - pname = "changelog-d"; - version = "0.1"; - src = pkgs.fetchurl { - name = "changelog-d-${rev}.tar.gz"; - url = "https://codeberg.org/roberth/changelog-d/archive/${rev}.tar.gz"; - hash = "sha256-8a2+i5u7YoszAgd5OIEW0eYUcP8yfhtoOIhLJkylYJ4="; - } // { inherit rev; }; - isLibrary = false; - isExecutable = true; - libraryHaskellDepends = [ - aeson base bytestring cabal-install-parsers Cabal-syntax containers - directory filepath frontmatter generic-lens-lite mtl parsec pretty - regex-applicative text - ]; - executableHaskellDepends = [ - base bytestring Cabal-syntax directory filepath - optparse-applicative - ]; - doHaddock = false; - description = "Concatenate changelog entries into a single one"; - license = lib.licenses.gpl3Plus; - mainProgram = "changelog-d"; -} diff --git a/misc/changelog-d.nix b/misc/changelog-d.nix deleted file mode 100644 index 1b20f4596..000000000 --- a/misc/changelog-d.nix +++ /dev/null @@ -1,31 +0,0 @@ -# Taken temporarily from -{ - callPackage, - lib, - haskell, - haskellPackages, -}: - -let - hsPkg = haskellPackages.callPackage ./changelog-d.cabal.nix { }; - - addCompletions = haskellPackages.generateOptparseApplicativeCompletions ["changelog-d"]; - - haskellModifications = - lib.flip lib.pipe [ - addCompletions - haskell.lib.justStaticExecutables - ]; - - mkDerivationOverrides = finalAttrs: oldAttrs: { - - version = oldAttrs.version + "-git-${lib.strings.substring 0 7 oldAttrs.src.rev}"; - - meta = oldAttrs.meta // { - homepage = "https://codeberg.org/roberth/changelog-d"; - maintainers = [ lib.maintainers.roberth ]; - }; - - }; -in - (haskellModifications hsPkg).overrideAttrs mkDerivationOverrides diff --git a/misc/systemv/nix-daemon b/misc/systemv/nix-daemon index fea537167..e8326f947 100755 --- a/misc/systemv/nix-daemon +++ b/misc/systemv/nix-daemon @@ -34,6 +34,7 @@ else fi # Source function library. +# shellcheck source=/dev/null . /etc/init.d/functions LOCKFILE=/var/lock/subsys/nix-daemon @@ -41,14 +42,20 @@ RUNDIR=/var/run/nix PIDFILE=${RUNDIR}/nix-daemon.pid RETVAL=0 -base=${0##*/} +# https://www.shellcheck.net/wiki/SC3004 +# Check if gettext exists +if ! type gettext > /dev/null 2>&1 +then + # If not, create a dummy function that returns the input verbatim + gettext() { printf '%s' "$1"; } +fi start() { mkdir -p ${RUNDIR} chown ${NIX_DAEMON_USER}:${NIX_DAEMON_USER} ${RUNDIR} - echo -n $"Starting nix daemon... " + printf '%s' "$(gettext 'Starting nix daemon... ')" daemonize -u $NIX_DAEMON_USER -p ${PIDFILE} $NIX_DAEMON_BIN $NIX_DAEMON_OPTS RETVAL=$? @@ -58,7 +65,7 @@ start() { } stop() { - echo -n $"Shutting down nix daemon: " + printf '%s' "$(gettext 'Shutting down nix daemon: ')" killproc -p ${PIDFILE} $NIX_DAEMON_BIN RETVAL=$? [ $RETVAL -eq 0 ] && rm -f ${LOCKFILE} ${PIDFILE} @@ -67,7 +74,7 @@ stop() { } reload() { - echo -n $"Reloading nix daemon... " + printf '%s' "$(gettext 'Reloading nix daemon... ')" killproc -p ${PIDFILE} $NIX_DAEMON_BIN -HUP RETVAL=$? echo @@ -105,7 +112,7 @@ case "$1" in fi ;; *) - echo $"Usage: $0 {start|stop|status|restart|condrestart}" + printf '%s' "$(gettext "Usage: $0 {start|stop|status|restart|condrestart}")" exit 2 ;; esac diff --git a/mk/common-test.sh b/mk/common-test.sh index 2783d293b..817422c40 100644 --- a/mk/common-test.sh +++ b/mk/common-test.sh @@ -1,27 +1,23 @@ +# shellcheck shell=bash + # Remove overall test dir (at most one of the two should match) and # remove file extension. -test_name=$(echo -n "$test" | sed \ - -e "s|^tests/unit/[^/]*/data/||" \ + +test_name=$(echo -n "${test?must be defined by caller (test runner)}" | sed \ + -e "s|^src/[^/]*-test/data/||" \ -e "s|^tests/functional/||" \ -e "s|\.sh$||" \ ) +# shellcheck disable=SC2016 TESTS_ENVIRONMENT=( "TEST_NAME=$test_name" 'NIX_REMOTE=' 'PS4=+(${BASH_SOURCE[0]-$0}:$LINENO) ' ) -: ${BASH:=/usr/bin/env bash} +read -r -a bash <<< "${BASH:-/usr/bin/env bash}" run () { - cd "$(dirname $1)" && env "${TESTS_ENVIRONMENT[@]}" $BASH -x -e -u -o pipefail $(basename $1) -} - -init_test () { - run "$init" 2>/dev/null > /dev/null -} - -run_test_proper () { - run "$test" + cd "$(dirname "$1")" && env "${TESTS_ENVIRONMENT[@]}" "${bash[@]}" -x -e -u -o pipefail "$(basename "$1")" } diff --git a/mk/compilation-database.mk b/mk/compilation-database.mk new file mode 100644 index 000000000..f69dc0de0 --- /dev/null +++ b/mk/compilation-database.mk @@ -0,0 +1,11 @@ +compile-commands-json-files := + +define write-compile-commands + _srcs := $$(sort $$(foreach src, $$($(1)_SOURCES), $$(src))) + + $(1)_COMPILE_COMMANDS_JSON := $$(addprefix $(buildprefix), $$(addsuffix .compile_commands.json, $$(basename $$(_srcs)))) + + compile-commands-json-files += $$($(1)_COMPILE_COMMANDS_JSON) + + clean-files += $$($(1)_COMPILE_COMMANDS_JSON) +endef diff --git a/mk/cxx-big-literal.mk b/mk/cxx-big-literal.mk index 85365df8e..d64a171c8 100644 --- a/mk/cxx-big-literal.mk +++ b/mk/cxx-big-literal.mk @@ -1,5 +1,5 @@ %.gen.hh: % - @echo 'R"foo(' >> $@.tmp + @echo 'R"__NIX_STR(' >> $@.tmp $(trace-gen) cat $< >> $@.tmp - @echo ')foo"' >> $@.tmp + @echo ')__NIX_STR"' >> $@.tmp @mv $@.tmp $@ diff --git a/mk/debug-test.sh b/mk/debug-test.sh index 52482c01e..0dd4406c3 100755 --- a/mk/debug-test.sh +++ b/mk/debug-test.sh @@ -3,12 +3,8 @@ set -eu -o pipefail test=$1 -init=${2-} dir="$(dirname "${BASH_SOURCE[0]}")" source "$dir/common-test.sh" -if [ -n "$init" ]; then - (init_test) -fi -run_test_proper +run "$test" diff --git a/mk/lib.mk b/mk/lib.mk index fe0add1c9..1e7af6ad5 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -68,6 +68,7 @@ include mk/patterns.mk include mk/templates.mk include mk/cxx-big-literal.mk include mk/tests.mk +include mk/compilation-database.mk # Include all sub-Makefiles. @@ -86,17 +87,23 @@ $(foreach script, $(bin-scripts), $(eval $(call install-program-in,$(script),$(b $(foreach script, $(bin-scripts), $(eval programs-list += $(script))) $(foreach script, $(noinst-scripts), $(eval programs-list += $(script))) $(foreach template, $(template-files), $(eval $(call instantiate-template,$(template)))) -install_test_init=tests/functional/init.sh $(foreach test, $(install-tests), \ - $(eval $(call run-test,$(test),$(install_test_init))) \ + $(eval $(call run-test,$(test))) \ $(eval installcheck: $(test).test)) $(foreach test-group, $(install-tests-groups), \ - $(eval $(call run-test-group,$(test-group),$(install_test_init))) \ + $(eval $(call run-test-group,$(test-group))) \ $(eval installcheck: $(test-group).test-group) \ $(foreach test, $($(test-group)-tests), \ - $(eval $(call run-test,$(test),$(install_test_init))) \ + $(eval $(call run-test,$(test))) \ $(eval $(test-group).test-group: $(test).test))) +# Compilation database. +$(foreach lib, $(libraries), $(eval $(call write-compile-commands,$(lib)))) +$(foreach prog, $(programs), $(eval $(call write-compile-commands,$(prog)))) + +compile_commands.json: $(compile-commands-json-files) + @jq --slurp '.' $^ >$@ + # Include makefiles requiring built programs. $(foreach mf, $(makefiles-late), $(eval $(call include-sub-makefile,$(mf)))) diff --git a/mk/patterns.mk b/mk/patterns.mk index c81150260..4caa2039e 100644 --- a/mk/patterns.mk +++ b/mk/patterns.mk @@ -1,11 +1,41 @@ + +# These are the complete command lines we use to compile C and C++ files. +# - $< is the source file. +# - $1 is the object file to create. +CC_CMD=$(CC) -o $1 -c $< $(CPPFLAGS) $(GLOBAL_CFLAGS) $(CFLAGS) $($1_CFLAGS) -MMD -MF $(call filename-to-dep,$1) -MP +CXX_CMD=$(CXX) -o $1 -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($1_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep,$1) -MP + +# We use COMPILE_COMMANDS_JSON_CMD to turn a compilation command (like CC_CMD +# or CXX_CMD above) into a comple_commands.json file. We rely on bash native +# word splitting to define the positional arguments. +# - $< is the source file being compiled. +COMPILE_COMMANDS_JSON_CMD=jq --null-input '{ directory: $$ENV.PWD, file: "$<", arguments: $$ARGS.positional }' --args -- + + $(buildprefix)%.o: %.cc @mkdir -p "$(dir $@)" - $(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cxx) $(call CXX_CMD,$@) $(buildprefix)%.o: %.cpp @mkdir -p "$(dir $@)" - $(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cxx) $(call CXX_CMD,$@) $(buildprefix)%.o: %.c @mkdir -p "$(dir $@)" - $(trace-cc) $(CC) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CFLAGS) $(CFLAGS) $($@_CFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cc) $(call CC_CMD,$@) + +# In the following we need to replace the .compile_commands.json extension in $@ with .o +# to make the object file. This is needed because CC_CMD and CXX_CMD do further expansions +# based on the object file name (i.e. *_CXXFLAGS and filename-to-dep). + +$(buildprefix)%.compile_commands.json: %.cc + @mkdir -p "$(dir $@)" + $(trace-jq) $(COMPILE_COMMANDS_JSON_CMD) $(call CXX_CMD,$(@:.compile_commands.json=.o)) > $@ + +$(buildprefix)%.compile_commands.json: %.cpp + @mkdir -p "$(dir $@)" + $(trace-jq) $(COMPILE_COMMANDS_JSON_CMD) $(call CXX_CMD,$(@:.compile_commands.json=.o)) > $@ + +$(buildprefix)%.compile_commands.json: %.c + @mkdir -p "$(dir $@)" + $(trace-jq) $(COMPILE_COMMANDS_JSON_CMD) $(call CC_CMD,$(@:.compile_commands.json=.o)) > $@ diff --git a/mk/platform.mk b/mk/platform.mk index fe960dedf..22c114a20 100644 --- a/mk/platform.mk +++ b/mk/platform.mk @@ -29,4 +29,8 @@ ifdef HOST_OS HOST_SOLARIS = 1 HOST_UNIX = 1 endif + ifeq ($(HOST_KERNEL), gnu) + HOST_HURD = 1 + HOST_UNIX = 1 + endif endif diff --git a/mk/precompiled-headers.mk b/mk/precompiled-headers.mk index cdd3daecd..f2803eb79 100644 --- a/mk/precompiled-headers.mk +++ b/mk/precompiled-headers.mk @@ -8,7 +8,7 @@ GCH = $(buildprefix)precompiled-headers.h.gch $(GCH): precompiled-headers.h @rm -f $@ @mkdir -p "$(dir $@)" - $(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS) + $(trace-gen) $(CXX) -c -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS) clean-files += $(GCH) diff --git a/mk/run-test.sh b/mk/run-test.sh index da9c5a473..7f9f1d5f8 100755 --- a/mk/run-test.sh +++ b/mk/run-test.sh @@ -8,7 +8,6 @@ yellow="" normal="" test=$1 -init=${2-} dir="$(dirname "${BASH_SOURCE[0]}")" source "$dir/common-test.sh" @@ -22,20 +21,18 @@ if [ -t 1 ]; then fi run_test () { - if [ -n "$init" ]; then - (init_test 2>/dev/null > /dev/null) - fi - log="$(run_test_proper 2>&1)" && status=0 || status=$? + log="$(run "$test" 2>&1)" && status=0 || status=$? } run_test -if [ $status -eq 0 ]; then +if [[ "$status" = 0 ]]; then echo "$post_run_msg [${green}PASS$normal]" -elif [ $status -eq 99 ]; then +elif [[ "$status" = 77 ]]; then echo "$post_run_msg [${yellow}SKIP$normal]" else echo "$post_run_msg [${red}FAIL$normal]" + # shellcheck disable=SC2001 echo "$log" | sed 's/^/ /' exit "$status" fi diff --git a/mk/tests.mk b/mk/tests.mk index bac9b704a..0a10f6d3b 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -12,8 +12,8 @@ endef define run-test - $(eval $(call run-bash,$1.test,$1 $(test-deps),mk/run-test.sh $1 $2)) - $(eval $(call run-bash,$1.test-debug,$1 $(test-deps),mk/debug-test.sh $1 $2)) + $(eval $(call run-bash,$1.test,$1 $(test-deps),mk/run-test.sh $1)) + $(eval $(call run-bash,$1.test-debug,$1 $(test-deps),mk/debug-test.sh $1)) endef diff --git a/mk/tracing.mk b/mk/tracing.mk index 1fc5573d7..09db1e617 100644 --- a/mk/tracing.mk +++ b/mk/tracing.mk @@ -10,6 +10,8 @@ ifeq ($(V), 0) trace-install = @echo " INST " $@; trace-mkdir = @echo " MKDIR " $@; trace-test = @echo " TEST " $@; + trace-sh = @echo " SH " $@; + trace-jq = @echo " JQ " $@; suppress = @ diff --git a/package.nix b/package.nix index 20796a386..a7c8923e8 100644 --- a/package.nix +++ b/package.nix @@ -13,17 +13,16 @@ , curl , editline , readline -, fileset , flex , git , gtest , jq -, doxygen , libarchive , libcpuid , libgit2 , libseccomp , libsodium +, man , lowdown , mdbook , mdbook-linkcheck @@ -33,7 +32,8 @@ , pkg-config , rapidcheck , sqlite -, util-linux +, toml11 +, unixtools , xz , busybox-sandbox-shell ? null @@ -48,10 +48,8 @@ , pname ? "nix" , versionSuffix ? "" -, officialRelease ? false -# Whether to build Nix. Useful to skip for tasks like (a) just -# generating API docs or (b) testing existing pre-built versions of Nix +# Whether to build Nix. Useful to skip for tasks like testing existing pre-built versions of Nix , doBuild ? true # Run the unit tests as part of the build. See `installUnitTests` for an @@ -74,7 +72,10 @@ # sounds so long as evaluation just takes places within short-lived # processes. (When the process exits, the memory is reclaimed; it is # only leaked *within* the process.) -, enableGC ? true +# +# Temporarily disabled on Windows because the `GC_throw_bad_alloc` +# symbol is missing during linking. +, enableGC ? !stdenv.hostPlatform.isWindows # Whether to enable Markdown rendering in the Nix binary. , enableMarkdown ? !stdenv.hostPlatform.isWindows @@ -87,10 +88,6 @@ # - readline , readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline" -# Whether to build the internal API docs, can be done separately from -# everything else. -, enableInternalAPIDocs ? false - # Whether to install unit tests. This is useful when cross compiling # since we cannot run them natively during the build, but can do so # later. @@ -113,6 +110,8 @@ }: let + inherit (lib) fileset; + version = lib.fileContents ./.version + versionSuffix; # selected attributes with defaults, will be used to define some @@ -161,6 +160,8 @@ in { ./m4 # TODO: do we really need README.md? It doesn't seem used in the build. ./README.md + # This could be put behind a conditional + ./maintainers/local.mk # For make, regardless of what we are building ./local.mk ./Makefile @@ -171,17 +172,11 @@ in { ./doc ./misc ./precompiled-headers.h - ./src + (fileset.difference ./src ./src/perl) ./COPYING ./scripts/local.mk - ] ++ lib.optionals buildUnitTests [ + ] ++ lib.optionals enableManual [ ./doc/manual - ] ++ lib.optionals enableInternalAPIDocs [ - ./doc/internal-api - # Source might not be compiled, but still must be available - # for Doxygen to gather comments. - ./src - ./tests/unit ] ++ lib.optionals buildUnitTests [ ./tests/unit ] ++ lib.optionals doInstallCheck [ @@ -195,8 +190,10 @@ in { ++ lib.optional doBuild "dev" # If we are doing just build or just docs, the one thing will use # "out". We only need additional outputs if we are doing both. - ++ lib.optional (doBuild && (enableManual || enableInternalAPIDocs)) "doc" - ++ lib.optional installUnitTests "check"; + ++ lib.optional (doBuild && enableManual) "doc" + ++ lib.optional installUnitTests "check" + ++ lib.optional doCheck "testresults" + ; nativeBuildInputs = [ autoconf-archive @@ -213,14 +210,14 @@ in { git mercurial openssh + man # for testing `nix-* --help` ] ++ lib.optionals (doInstallCheck || enableManual) [ jq # Also for custom mdBook preprocessor. - ] ++ lib.optional stdenv.hostPlatform.isLinux util-linux - ++ lib.optional enableInternalAPIDocs doxygen + ] ++ lib.optional stdenv.hostPlatform.isStatic unixtools.hexdump ; - buildInputs = lib.optionals doBuild [ - boost + buildInputs = lib.optionals doBuild ( + [ brotli bzip2 curl @@ -229,6 +226,7 @@ in { libsodium openssl sqlite + toml11 xz ({ inherit readline editline; }.${readlineFlavor}) ] ++ lib.optionals enableMarkdown [ @@ -240,46 +238,22 @@ in { ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid # There have been issues building these dependencies ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin)) - (aws-sdk-cpp.override { - apis = ["s3" "transfer"]; - customMemoryManagement = false; - }) - ; + aws-sdk-cpp + ); - propagatedBuildInputs = [ + propagatedBuildInputs = lib.optionals doBuild ([ + boost nlohmann_json - ] ++ lib.optional enableGC boehmgc; + ] ++ lib.optional enableGC boehmgc + ); dontBuild = !attrs.doBuild; doCheck = attrs.doCheck; - disallowedReferences = [ boost ]; - - preConfigure = lib.optionalString (doBuild && ! stdenv.hostPlatform.isStatic) ( - '' - # Copy libboost_context so we don't get all of Boost in our closure. - # https://github.com/NixOS/nixpkgs/issues/45462 - mkdir -p $out/lib - cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib - rm -f $out/lib/*.a - '' + lib.optionalString stdenv.hostPlatform.isLinux '' - chmod u+w $out/lib/*.so.* - patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* - '' + lib.optionalString stdenv.hostPlatform.isDarwin '' - for LIB in $out/lib/*.dylib; do - chmod u+w $LIB - install_name_tool -id $LIB $LIB - install_name_tool -delete_rpath ${boost}/lib/ $LIB || true - done - install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib - '' - ); - configureFlags = [ (lib.enableFeature doBuild "build") (lib.enableFeature buildUnitTests "unit-tests") (lib.enableFeature doInstallCheck "functional-tests") - (lib.enableFeature enableInternalAPIDocs "internal-api-docs") (lib.enableFeature enableManual "doc-gen") (lib.enableFeature enableGC "gc") (lib.enableFeature enableMarkdown "markdown") @@ -303,8 +277,11 @@ in { makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1"; - installTargets = lib.optional doBuild "install" - ++ lib.optional enableInternalAPIDocs "internal-api-html"; + preCheck = '' + mkdir $testresults + ''; + + installTargets = lib.optional doBuild "install"; installFlags = "sysconfdir=$(out)/etc"; @@ -319,18 +296,16 @@ in { lib.optionalString stdenv.hostPlatform.isStatic '' mkdir -p $out/nix-support echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products - '' + lib.optionalString stdenv.isDarwin '' - install_name_tool \ - -change ${boost}/lib/libboost_context.dylib \ - $out/lib/libboost_context.dylib \ - $out/lib/libnixutil.dylib '' ) + lib.optionalString enableManual '' mkdir -p ''${!outputDoc}/nix-support echo "doc manual ''${!outputDoc}/share/doc/nix/manual" >> ''${!outputDoc}/nix-support/hydra-build-products - '' + lib.optionalString enableInternalAPIDocs '' - mkdir -p ''${!outputDoc}/nix-support - echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products + ''; + + # So the check output gets links for DLLs in the out output. + preFixup = lib.optionalString (stdenv.hostPlatform.isWindows && builtins.elem "check" finalAttrs.outputs) '' + ln -s "$check/lib/"*.dll "$check/bin" + ln -s "$out/bin/"*.dll "$check/bin" ''; doInstallCheck = attrs.doInstallCheck; @@ -341,20 +316,26 @@ in { # Work around weird bug where it doesn't think there is a Makefile. installCheckPhase = if (!doBuild && doInstallCheck) then '' + runHook preInstallCheck mkdir -p src/nix-channel make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES '' else null; # Needed for tests if we are not doing a build, but testing existing # built Nix. - preInstallCheck = lib.optionalString (! doBuild) '' - mkdir -p src/nix-channel - ''; + preInstallCheck = + lib.optionalString (! doBuild) '' + mkdir -p src/nix-channel + '' + # See https://github.com/NixOS/nix/issues/2523 + # Occurs often in tests since https://github.com/NixOS/nix/pull/9900 + + lib.optionalString stdenv.hostPlatform.isDarwin '' + export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES + ''; separateDebugInfo = !stdenv.hostPlatform.isStatic; - # TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated - # to work with `strictDeps`. + # TODO Always true after https://github.com/NixOS/nixpkgs/issues/318564 strictDeps = !withCoverageChecks; hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; diff --git a/packaging/components.nix b/packaging/components.nix new file mode 100644 index 000000000..870e9ae61 --- /dev/null +++ b/packaging/components.nix @@ -0,0 +1,43 @@ +scope: +let + inherit (scope) callPackage; +in + +# This becomes the pkgs.nixComponents attribute set +{ + nix = callPackage ../package.nix { }; + + nix-util = callPackage ../src/libutil/package.nix { }; + nix-util-c = callPackage ../src/libutil-c/package.nix { }; + nix-util-test-support = callPackage ../tests/unit/libutil-support/package.nix { }; + nix-util-tests = callPackage ../tests/unit/libutil/package.nix { }; + + nix-store = callPackage ../src/libstore/package.nix { }; + nix-store-c = callPackage ../src/libstore-c/package.nix { }; + nix-store-test-support = callPackage ../tests/unit/libstore-support/package.nix { }; + nix-store-tests = callPackage ../tests/unit/libstore/package.nix { }; + + nix-fetchers = callPackage ../src/libfetchers/package.nix { }; + nix-fetchers-tests = callPackage ../tests/unit/libfetchers/package.nix { }; + + nix-expr = callPackage ../src/libexpr/package.nix { }; + nix-expr-c = callPackage ../src/libexpr-c/package.nix { }; + nix-expr-test-support = callPackage ../tests/unit/libexpr-support/package.nix { }; + nix-expr-tests = callPackage ../tests/unit/libexpr/package.nix { }; + + nix-flake = callPackage ../src/libflake/package.nix { }; + nix-flake-tests = callPackage ../tests/unit/libflake/package.nix { }; + + nix-main = callPackage ../src/libmain/package.nix { }; + nix-main-c = callPackage ../src/libmain-c/package.nix { }; + + nix-cmd = callPackage ../src/libcmd/package.nix { }; + + # Will replace `nix` once the old build system is gone. + nix-ng = callPackage ../src/nix/package.nix { }; + + nix-internal-api-docs = callPackage ../src/internal-api-docs/package.nix { }; + nix-external-api-docs = callPackage ../src/external-api-docs/package.nix { }; + + nix-perl-bindings = callPackage ../src/perl/package.nix { }; +} diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix new file mode 100644 index 000000000..e0737593f --- /dev/null +++ b/packaging/dependencies.nix @@ -0,0 +1,165 @@ +# These overrides are applied to the dependencies of the Nix components. + +{ + # Flake inputs; used for sources + inputs, + + # The raw Nixpkgs, not affected by this scope + pkgs, + + stdenv, + versionSuffix, +}: + +let + prevStdenv = stdenv; +in + +let + inherit (pkgs) lib; + + root = ../.; + + stdenv = if prevStdenv.isDarwin && prevStdenv.isx86_64 + then darwinStdenv + else prevStdenv; + + # Fix the following error with the default x86_64-darwin SDK: + # + # error: aligned allocation function of type 'void *(std::size_t, std::align_val_t)' is only available on macOS 10.13 or newer + # + # Despite the use of the 10.13 deployment target here, the aligned + # allocation function Clang uses with this setting actually works + # all the way back to 10.6. + darwinStdenv = pkgs.overrideSDK prevStdenv { darwinMinVersion = "10.13"; }; + + # Nixpkgs implements this by returning a subpath into the fetched Nix sources. + resolvePath = p: p; + + # Indirection for Nixpkgs to override when package.nix files are vendored + filesetToSource = lib.fileset.toSource; + + localSourceLayer = finalAttrs: prevAttrs: + let + workDirPath = + # Ideally we'd pick finalAttrs.workDir, but for now `mkDerivation` has + # the requirement that everything except passthru and meta must be + # serialized by mkDerivation, which doesn't work for this. + prevAttrs.workDir; + + workDirSubpath = lib.path.removePrefix root workDirPath; + sources = assert prevAttrs.fileset._type == "fileset"; prevAttrs.fileset; + src = lib.fileset.toSource { fileset = sources; inherit root; }; + + in + { + sourceRoot = "${src.name}/" + workDirSubpath; + inherit src; + + # Clear what `derivation` can't/shouldn't serialize; see prevAttrs.workDir. + fileset = null; + workDir = null; + }; + + # Work around weird `--as-needed` linker behavior with BSD, see + # https://github.com/mesonbuild/meson/issues/3593 + bsdNoLinkAsNeeded = finalAttrs: prevAttrs: + lib.optionalAttrs stdenv.hostPlatform.isBSD { + mesonFlags = [ (lib.mesonBool "b_asneeded" false) ] ++ prevAttrs.mesonFlags or []; + }; + + miscGoodPractice = finalAttrs: prevAttrs: + { + strictDeps = prevAttrs.strictDeps or true; + enableParallelBuilding = true; + }; + +in +scope: { + inherit stdenv versionSuffix; + version = lib.fileContents ../.version + versionSuffix; + + aws-sdk-cpp = (pkgs.aws-sdk-cpp.override { + apis = [ "s3" "transfer" ]; + customMemoryManagement = false; + }).overrideAttrs { + # only a stripped down version is built, which takes a lot less resources + # to build, so we don't need a "big-parallel" machine. + requiredSystemFeatures = [ ]; + }; + + libseccomp = pkgs.libseccomp.overrideAttrs (_: rec { + version = "2.5.5"; + src = pkgs.fetchurl { + url = "https://github.com/seccomp/libseccomp/releases/download/v${version}/libseccomp-${version}.tar.gz"; + hash = "sha256-JIosik2bmFiqa69ScSw0r+/PnJ6Ut23OAsHJqiX7M3U="; + }; + }); + + boehmgc = pkgs.boehmgc.override { + enableLargeConfig = true; + }; + + # TODO Hack until https://github.com/NixOS/nixpkgs/issues/45462 is fixed. + boost = (pkgs.boost.override { + extraB2Args = [ + "--with-container" + "--with-context" + "--with-coroutine" + ]; + }).overrideAttrs (old: { + # Need to remove `--with-*` to use `--with-libraries=...` + buildPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.buildPhase; + installPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.installPhase; + }); + + libgit2 = pkgs.libgit2.overrideAttrs (attrs: { + src = inputs.libgit2; + version = inputs.libgit2.lastModifiedDate; + cmakeFlags = attrs.cmakeFlags or [] + ++ [ "-DUSE_SSH=exec" ]; + }); + + busybox-sandbox-shell = pkgs.busybox-sandbox-shell or (pkgs.busybox.override { + useMusl = true; + enableStatic = true; + enableMinimal = true; + extraConfig = '' + CONFIG_FEATURE_FANCY_ECHO y + CONFIG_FEATURE_SH_MATH y + CONFIG_FEATURE_SH_MATH_64 y + + CONFIG_ASH y + CONFIG_ASH_OPTIMIZE_FOR_SIZE y + + CONFIG_ASH_ALIAS y + CONFIG_ASH_BASH_COMPAT y + CONFIG_ASH_CMDCMD y + CONFIG_ASH_ECHO y + CONFIG_ASH_GETOPTS y + CONFIG_ASH_INTERNAL_GLOB y + CONFIG_ASH_JOB_CONTROL y + CONFIG_ASH_PRINTF y + CONFIG_ASH_TEST y + ''; + }); + + # TODO change in Nixpkgs, Windows works fine. First commit of + # https://github.com/NixOS/nixpkgs/pull/322977 backported will fix. + toml11 = pkgs.toml11.overrideAttrs (old: { + meta.platforms = lib.platforms.all; + }); + + inherit resolvePath filesetToSource; + + mkMesonDerivation = f: let + exts = [ + miscGoodPractice + bsdNoLinkAsNeeded + localSourceLayer + ]; + in stdenv.mkDerivation + (lib.extends + (lib.foldr lib.composeExtensions (_: _: {}) exts) + f); +} diff --git a/packaging/hydra.nix b/packaging/hydra.nix new file mode 100644 index 000000000..dbe992476 --- /dev/null +++ b/packaging/hydra.nix @@ -0,0 +1,200 @@ +{ inputs +, binaryTarball +, forAllCrossSystems +, forAllSystems +, lib +, linux64BitSystems +, nixpkgsFor +, self +}: +let + inherit (inputs) nixpkgs nixpkgs-regression; + + installScriptFor = tarballs: + nixpkgsFor.x86_64-linux.native.callPackage ../scripts/installer.nix { + inherit tarballs; + }; + + testNixVersions = pkgs: client: daemon: + pkgs.callPackage ../package.nix { + pname = + "nix-tests" + + lib.optionalString + (lib.versionAtLeast daemon.version "2.4pre20211005" && + lib.versionAtLeast client.version "2.4pre20211005") + "-${client.version}-against-${daemon.version}"; + + test-client = client; + test-daemon = daemon; + + doBuild = false; + }; + + # Technically we could just return `pkgs.nixComponents`, but for Hydra it's + # convention to transpose it, and to transpose it efficiently, we need to + # enumerate them manually, so that we don't evaluate unnecessary package sets. + forAllPackages = lib.genAttrs [ + "nix" + "nix-util" + "nix-util-c" + "nix-util-test-support" + "nix-util-tests" + "nix-store" + "nix-store-c" + "nix-store-test-support" + "nix-store-tests" + "nix-fetchers" + "nix-fetchers-tests" + "nix-expr" + "nix-expr-c" + "nix-expr-test-support" + "nix-expr-tests" + "nix-flake" + "nix-flake-tests" + "nix-main" + "nix-main-c" + "nix-cmd" + "nix-ng" + ]; +in +{ + # Binary package for various platforms. + build = forAllPackages (pkgName: + forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.${pkgName})); + + shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation); + + buildStatic = forAllPackages (pkgName: + lib.genAttrs linux64BitSystems (system: nixpkgsFor.${system}.static.nixComponents.${pkgName})); + + buildCross = forAllPackages (pkgName: + forAllCrossSystems (crossSystem: + lib.genAttrs [ "x86_64-linux" ] (system: nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName}))); + + buildNoGc = forAllSystems (system: + self.packages.${system}.nix.override { enableGC = false; } + ); + + buildNoTests = forAllSystems (system: nixpkgsFor.${system}.native.nix_noTests); + + # Toggles some settings for better coverage. Windows needs these + # library combinations, and Debian build Nix with GNU readline too. + buildReadlineNoMarkdown = forAllSystems (system: + self.packages.${system}.nix.override { + enableMarkdown = false; + readlineFlavor = "readline"; + } + ); + + # Perl bindings for various platforms. + perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.nix-perl-bindings); + + # Binary tarball for various platforms, containing a Nix store + # with the closure of 'nix' package, and the second half of + # the installation script. + binaryTarball = forAllSystems (system: binaryTarball nixpkgsFor.${system}.native.nix nixpkgsFor.${system}.native); + + binaryTarballCross = lib.genAttrs [ "x86_64-linux" ] (system: + forAllCrossSystems (crossSystem: + binaryTarball + nixpkgsFor.${system}.cross.${crossSystem}.nix + nixpkgsFor.${system}.cross.${crossSystem})); + + # The first half of the installation script. This is uploaded + # to https://nixos.org/nix/install. It downloads the binary + # tarball for the user's system and calls the second half of the + # installation script. + installerScript = installScriptFor [ + # Native + self.hydraJobs.binaryTarball."x86_64-linux" + self.hydraJobs.binaryTarball."i686-linux" + self.hydraJobs.binaryTarball."aarch64-linux" + self.hydraJobs.binaryTarball."x86_64-darwin" + self.hydraJobs.binaryTarball."aarch64-darwin" + # Cross + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu" + ]; + installerScriptForGHA = installScriptFor [ + # Native + self.hydraJobs.binaryTarball."x86_64-linux" + self.hydraJobs.binaryTarball."aarch64-darwin" + # Cross + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu" + ]; + + # docker image with Nix inside + dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); + + # Line coverage analysis. + coverage = nixpkgsFor.x86_64-linux.native.nix.override { + pname = "nix-coverage"; + withCoverageChecks = true; + }; + + # API docs for Nix's unstable internal C++ interfaces. + internal-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-internal-api-docs; + + # API docs for Nix's C bindings. + external-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-external-api-docs; + + # System tests. + tests = import ../tests/nixos { inherit lib nixpkgs nixpkgsFor self; } // { + + # Make sure that nix-env still produces the exact same result + # on a particular version of Nixpkgs. + evalNixpkgs = + let + inherit (nixpkgsFor.x86_64-linux.native) runCommand nix; + in + runCommand "eval-nixos" { buildInputs = [ nix ]; } + '' + type -p nix-env + # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593. + ( + set -x + time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages + [[ $(sha1sum < packages | cut -c1-40) = e01b031fc9785a572a38be6bc473957e3b6faad7 ]] + ) + mkdir $out + ''; + + nixpkgsLibTests = + forAllSystems (system: + import (nixpkgs + "/lib/tests/test-with-nix.nix") + { + lib = nixpkgsFor.${system}.native.lib; + nix = self.packages.${system}.nix; + pkgs = nixpkgsFor.${system}.native; + } + ); + }; + + metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" { + pkgs = nixpkgsFor.x86_64-linux.native; + nixpkgs = nixpkgs-regression; + }; + + installTests = forAllSystems (system: + let pkgs = nixpkgsFor.${system}.native; in + pkgs.runCommand "install-tests" + { + againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix; + againstCurrentLatest = + # FIXME: temporarily disable this on macOS because of #3605. + if system == "x86_64-linux" + then testNixVersions pkgs pkgs.nix pkgs.nixVersions.latest + else null; + # Disabled because the latest stable version doesn't handle + # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work + # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable; + } "touch $out"); + + installerTests = import ../tests/installer { + binaryTarballs = self.hydraJobs.binaryTarball; + inherit nixpkgsFor; + }; +} diff --git a/perl/.yath.rc b/perl/.yath.rc deleted file mode 100644 index 118bf80c8..000000000 --- a/perl/.yath.rc +++ /dev/null @@ -1,2 +0,0 @@ -[test] --I=rel(lib/Nix) diff --git a/perl/Makefile b/perl/Makefile deleted file mode 100644 index 832668dd1..000000000 --- a/perl/Makefile +++ /dev/null @@ -1,21 +0,0 @@ -makefiles = local.mk - -GLOBAL_CXXFLAGS += -g -Wall -std=c++2a - -# A convenience for concurrent development of Nix and its Perl bindings. -# Not needed in a standalone build of the Perl bindings. -ifneq ("$(wildcard ../src)", "") - GLOBAL_CXXFLAGS += -I ../src -endif - --include Makefile.config - -OPTIMIZE = 1 - -ifeq ($(OPTIMIZE), 1) - GLOBAL_CXXFLAGS += -O3 -else - GLOBAL_CXXFLAGS += -O0 -endif - -include mk/lib.mk diff --git a/perl/Makefile.config.in b/perl/Makefile.config.in deleted file mode 100644 index d856de3ad..000000000 --- a/perl/Makefile.config.in +++ /dev/null @@ -1,18 +0,0 @@ -HOST_OS = @host_os@ -CC = @CC@ -CFLAGS = @CFLAGS@ -CXX = @CXX@ -CXXFLAGS = @CXXFLAGS@ -PACKAGE_NAME = @PACKAGE_NAME@ -PACKAGE_VERSION = @PACKAGE_VERSION@ -SODIUM_LIBS = @SODIUM_LIBS@ -NIX_CFLAGS = @NIX_CFLAGS@ -NIX_LIBS = @NIX_LIBS@ -nixbindir = @nixbindir@ -curl = @curl@ -nixlibexecdir = @nixlibexecdir@ -nixlocalstatedir = @nixlocalstatedir@ -perl = @perl@ -perllibdir = @perllibdir@ -nixstoredir = @nixstoredir@ -nixsysconfdir = @nixsysconfdir@ diff --git a/perl/configure.ac b/perl/configure.ac deleted file mode 100644 index a02cb06c9..000000000 --- a/perl/configure.ac +++ /dev/null @@ -1,84 +0,0 @@ -AC_INIT(nix-perl, m4_esyscmd([bash -c "echo -n $(cat ../.version)$VERSION_SUFFIX"])) -AC_CONFIG_SRCDIR(MANIFEST) -AC_CONFIG_AUX_DIR(../config) - -CFLAGS= -CXXFLAGS= -AC_PROG_CC -AC_PROG_CXX - -AC_CANONICAL_HOST - -# Use 64-bit file system calls so that we can support files > 2 GiB. -AC_SYS_LARGEFILE - -AC_DEFUN([NEED_PROG], -[ -AC_PATH_PROG($1, $2) -if test -z "$$1"; then - AC_MSG_ERROR([$2 is required]) -fi -]) - -NEED_PROG(perl, perl) -NEED_PROG(curl, curl) -NEED_PROG(bzip2, bzip2) -NEED_PROG(xz, xz) - -# Test that Perl has the open/fork feature (Perl 5.8.0 and beyond). -AC_MSG_CHECKING([whether Perl is recent enough]) -if ! $perl -e 'open(FOO, "-|", "true"); while () { print; }; close FOO or die;'; then - AC_MSG_RESULT(no) - AC_MSG_ERROR([Your Perl version is too old. Nix requires Perl 5.8.0 or newer.]) -fi -AC_MSG_RESULT(yes) - - -# Figure out where to install Perl modules. -AC_MSG_CHECKING([for the Perl installation prefix]) -perlversion=$($perl -e 'use Config; print $Config{version};') -perlarchname=$($perl -e 'use Config; print $Config{archname};') -AC_SUBST(perllibdir, [${libdir}/perl5/site_perl/$perlversion/$perlarchname]) -AC_MSG_RESULT($perllibdir) - -# Look for libsodium. -PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"]) - -# Check for the required Perl dependencies (DBI and DBD::SQLite). -perlFlags="-I$perllibdir" - -AC_ARG_WITH(dbi, AC_HELP_STRING([--with-dbi=PATH], - [prefix of the Perl DBI library]), - perlFlags="$perlFlags -I$withval") - -AC_ARG_WITH(dbd-sqlite, AC_HELP_STRING([--with-dbd-sqlite=PATH], - [prefix of the Perl DBD::SQLite library]), - perlFlags="$perlFlags -I$withval") - -AC_MSG_CHECKING([whether DBD::SQLite works]) -if ! $perl $perlFlags -e 'use DBI; use DBD::SQLite;' 2>&5; then - AC_MSG_RESULT(no) - AC_MSG_FAILURE([The Perl modules DBI and/or DBD::SQLite are missing.]) -fi -AC_MSG_RESULT(yes) - -AC_SUBST(perlFlags) - -PKG_CHECK_MODULES([NIX], [nix-store]) - -NEED_PROG([NIX], [nix]) - -# Expand all variables in config.status. -test "$prefix" = NONE && prefix=$ac_default_prefix -test "$exec_prefix" = NONE && exec_prefix='${prefix}' -for name in $ac_subst_vars; do - declare $name="$(eval echo "${!name}")" - declare $name="$(eval echo "${!name}")" - declare $name="$(eval echo "${!name}")" -done - -rm -f Makefile.config -ln -sfn ../mk mk - -AC_CONFIG_FILES([]) -AC_OUTPUT diff --git a/perl/default.nix b/perl/default.nix deleted file mode 100644 index 7103574c9..000000000 --- a/perl/default.nix +++ /dev/null @@ -1,61 +0,0 @@ -{ lib, fileset -, stdenv -, perl, perlPackages -, autoconf-archive, autoreconfHook, pkg-config -, nix, curl, bzip2, xz, boost, libsodium, darwin -}: - -perl.pkgs.toPerlModule (stdenv.mkDerivation (finalAttrs: { - name = "nix-perl-${nix.version}"; - - src = fileset.toSource { - root = ../.; - fileset = fileset.unions ([ - ../.version - ../m4 - ../mk - ./MANIFEST - ./Makefile - ./Makefile.config.in - ./configure.ac - ./lib - ./local.mk - ] ++ lib.optionals finalAttrs.doCheck [ - ./.yath.rc - ./t - ]); - }; - - nativeBuildInputs = - [ autoconf-archive - autoreconfHook - pkg-config - ]; - - buildInputs = - [ nix - curl - bzip2 - xz - perl - boost - ] - ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium - ++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security; - - # `perlPackages.Test2Harness` is marked broken for Darwin - doCheck = !stdenv.isDarwin; - - nativeCheckInputs = [ - perlPackages.Test2Harness - ]; - - configureFlags = [ - "--with-dbi=${perlPackages.DBI}/${perl.libPrefix}" - "--with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix}" - ]; - - enableParallelBuilding = true; - - postUnpack = "sourceRoot=$sourceRoot/perl"; -})) diff --git a/perl/local.mk b/perl/local.mk deleted file mode 100644 index ed4764eb9..000000000 --- a/perl/local.mk +++ /dev/null @@ -1,46 +0,0 @@ -nix_perl_sources := \ - lib/Nix/Store.pm \ - lib/Nix/Manifest.pm \ - lib/Nix/SSH.pm \ - lib/Nix/CopyClosure.pm \ - lib/Nix/Config.pm.in \ - lib/Nix/Utils.pm - -nix_perl_modules := $(nix_perl_sources:.in=) - -$(foreach x, $(nix_perl_modules), $(eval $(call install-data-in, $(x), $(perllibdir)/Nix))) - -lib/Nix/Store.cc: lib/Nix/Store.xs - $(trace-gen) xsubpp $^ -output $@ - -libraries += Store - -Store_DIR := lib/Nix - -Store_SOURCES := $(Store_DIR)/Store.cc - -Store_CXXFLAGS = \ - $(NIX_CFLAGS) \ - -I$(shell perl -e 'use Config; print $$Config{archlibexp};')/CORE \ - -D_FILE_OFFSET_BITS=64 \ - -Wno-unknown-warning-option -Wno-unused-variable -Wno-literal-suffix \ - -Wno-reserved-user-defined-literal -Wno-duplicate-decl-specifier -Wno-pointer-bool-conversion - -Store_LDFLAGS := $(SODIUM_LIBS) $(NIX_LIBS) - -ifdef HOST_CYGWIN - archlib = $(shell perl -E 'use Config; print $$Config{archlib};') - libperl = $(shell perl -E 'use Config; print $$Config{libperl};') - Store_LDFLAGS += $(shell find ${archlib} -name ${libperl}) -endif - -Store_ALLOW_UNDEFINED = 1 - -Store_FORCE_INSTALL = 1 - -Store_INSTALL_DIR = $(perllibdir)/auto/Nix/Store - -clean-files += lib/Nix/Config.pm lib/Nix/Store.cc Makefile.config - -check: all - yath test diff --git a/precompiled-headers.h b/precompiled-headers.h index f52f1cab8..e1a3f8cc0 100644 --- a/precompiled-headers.h +++ b/precompiled-headers.h @@ -42,19 +42,22 @@ #include #include #include -#include -#include -#include #include -#include -#include -#include #include #include #include -#include -#include -#include #include +#ifndef _WIN32 +# include +# include +# include +# include +# include +# include +# include +# include +# include +#endif + #include diff --git a/scripts/bigsur-nixbld-user-migration.sh b/scripts/bigsur-nixbld-user-migration.sh index f1619fd56..0eb312e07 100755 --- a/scripts/bigsur-nixbld-user-migration.sh +++ b/scripts/bigsur-nixbld-user-migration.sh @@ -3,7 +3,7 @@ ((NEW_NIX_FIRST_BUILD_UID=301)) id_available(){ - dscl . list /Users UniqueID | grep -E '\b'$1'\b' >/dev/null + dscl . list /Users UniqueID | grep -E '\b'"$1"'\b' >/dev/null } change_nixbld_names_and_ids(){ @@ -26,18 +26,18 @@ change_nixbld_names_and_ids(){ fi done - if [[ $name == _* ]]; then + if [[ "$name" == _* ]]; then echo " It looks like $name has already been renamed--skipping." else # first 3 are cleanup, it's OK if they aren't here - sudo dscl . delete /Users/$name dsAttrTypeNative:_writers_passwd &>/dev/null || true - sudo dscl . change /Users/$name NFSHomeDirectory "/private/var/empty 1" "/var/empty" &>/dev/null || true + sudo dscl . delete "/Users/$name" dsAttrTypeNative:_writers_passwd &>/dev/null || true + sudo dscl . change "/Users/$name" NFSHomeDirectory "/private/var/empty 1" "/var/empty" &>/dev/null || true # remove existing user from group - sudo dseditgroup -o edit -t user -d $name nixbld || true - sudo dscl . change /Users/$name UniqueID $uid $next_id - sudo dscl . change /Users/$name RecordName $name _$name + sudo dseditgroup -o edit -t user -d "$name" nixbld || true + sudo dscl . change "/Users/$name" UniqueID "$uid" "$next_id" + sudo dscl . change "/Users/$name" RecordName "$name" "_$name" # add renamed user to group - sudo dseditgroup -o edit -t user -a _$name nixbld + sudo dseditgroup -o edit -t user -a "_$name" nixbld echo " $name migrated to _$name (uid: $next_id)" fi done < <(dscl . list /Users UniqueID | grep nixbld | sort -n -k2) diff --git a/scripts/check-hydra-status.sh b/scripts/check-hydra-status.sh deleted file mode 100644 index e62705e94..000000000 --- a/scripts/check-hydra-status.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail -# set -x - - -# mapfile BUILDS_FOR_LATEST_EVAL < <( -# curl -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master/evals | \ -# jq -r '.evals[0].builds[] | @sh') -BUILDS_FOR_LATEST_EVAL=$( -curl -sS -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master/evals | \ - jq -r '.evals[0].builds[]') - -someBuildFailed=0 - -for buildId in $BUILDS_FOR_LATEST_EVAL; do - buildInfo=$(curl --fail -sS -H 'Accept: application/json' "https://hydra.nixos.org/build/$buildId") - - finished=$(echo "$buildInfo" | jq -r '.finished') - - if [[ $finished = 0 ]]; then - continue - fi - - buildStatus=$(echo "$buildInfo" | jq -r '.buildstatus') - - if [[ $buildStatus != 0 ]]; then - someBuildFailed=1 - echo "Job “$(echo "$buildInfo" | jq -r '.job')†failed on hydra: $buildInfo" - fi -done - -exit "$someBuildFailed" diff --git a/scripts/flake-regressions.sh b/scripts/flake-regressions.sh new file mode 100755 index 000000000..d76531134 --- /dev/null +++ b/scripts/flake-regressions.sh @@ -0,0 +1,27 @@ +#! /usr/bin/env bash + +set -e + +echo "Nix version:" +nix --version + +cd flake-regressions + +status=0 + +flakes=$(find tests -mindepth 3 -maxdepth 3 -type d -not -path '*/.*' | sort | head -n25) + +echo "Running flake tests..." + +for flake in $flakes; do + + if ! REGENERATE=0 ./eval-flake.sh "$flake"; then + status=1 + echo "⌠$flake" + else + echo "✅ $flake" + fi + +done + +exit "$status" diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index ad3ee8881..6aee073e3 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -754,7 +754,7 @@ I will: (if it does, I will tell you how to clean them up.) - create local users (see the list above for the users I'll make) - create a local group ($NIX_BUILD_GROUP_NAME) - - install Nix in to $NIX_ROOT + - install Nix in $NIX_ROOT - create a configuration file in /etc/nix - set up the "default profile" by creating some Nix-related files in $ROOT_HOME diff --git a/scripts/install-systemd-multi-user.sh b/scripts/install-systemd-multi-user.sh index 202a9bb54..a62ed7e3a 100755 --- a/scripts/install-systemd-multi-user.sh +++ b/scripts/install-systemd-multi-user.sh @@ -35,7 +35,7 @@ escape_systemd_env() { # Gather all non-empty proxy environment variables into a string create_systemd_proxy_env() { - vars="http_proxy https_proxy ftp_proxy no_proxy HTTP_PROXY HTTPS_PROXY FTP_PROXY NO_PROXY" + vars="http_proxy https_proxy ftp_proxy all_proxy no_proxy HTTP_PROXY HTTPS_PROXY FTP_PROXY ALL_PROXY NO_PROXY" for v in $vars; do if [ "x${!v:-}" != "x" ]; then echo "Environment=${v}=$(escape_systemd_env ${!v})" diff --git a/scripts/install.in b/scripts/install.in index 7d2e52b26..b4e808d8e 100755 --- a/scripts/install.in +++ b/scripts/install.in @@ -50,6 +50,11 @@ case "$(uname -s).$(uname -m)" in path=@tarballPath_armv7l-linux@ system=armv7l-linux ;; + Linux.riscv64) + hash=@tarballHash_riscv64-linux@ + path=@tarballPath_riscv64-linux@ + system=riscv64-linux + ;; Darwin.x86_64) hash=@tarballHash_x86_64-darwin@ path=@tarballPath_x86_64-darwin@ diff --git a/scripts/nix-profile-daemon.fish.in b/scripts/nix-profile-daemon.fish.in index c23aa64f0..346dce5dd 100644 --- a/scripts/nix-profile-daemon.fish.in +++ b/scripts/nix-profile-daemon.fish.in @@ -28,7 +28,7 @@ else end # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. -if test -n "$NIX_SSH_CERT_FILE" +if test -n "$NIX_SSL_CERT_FILE" : # Allow users to override the NIX_SSL_CERT_FILE else if test -e /etc/ssl/certs/ca-certificates.crt # NixOS, Ubuntu, Debian, Gentoo, Arch set --export NIX_SSL_CERT_FILE /etc/ssl/certs/ca-certificates.crt @@ -44,7 +44,7 @@ else if test -e "$NIX_LINK/etc/ca-bundle.crt" # old cacert in Nix profile set --export NIX_SSL_CERT_FILE "$NIX_LINK/etc/ca-bundle.crt" else # Fall back to what is in the nix profiles, favouring whatever is defined last. - for i in $NIX_PROFILES + for i in (string split ' ' $NIX_PROFILES) if test -e "$i/etc/ssl/certs/ca-bundle.crt" set --export NIX_SSL_CERT_FILE "$i/etc/ssl/certs/ca-bundle.crt" end diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index d256b24ed..eb124c0b5 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -1,4 +1,5 @@ # Only execute this file once per shell. +# This file is tested by tests/installer/default.nix. if [ -n "${__ETC_PROFILE_NIX_SOURCED:-}" ]; then return; fi __ETC_PROFILE_NIX_SOURCED=1 @@ -9,11 +10,9 @@ else NIX_LINK_NEW=$HOME/.local/state/nix/profile fi if [ -e "$NIX_LINK_NEW" ]; then - NIX_LINK="$NIX_LINK_NEW" -else - if [ -t 2 ] && [ -e "$NIX_LINK_NEW" ]; then + if [ -t 2 ] && [ -e "$NIX_LINK" ]; then warning="\033[1;35mwarning:\033[0m" - printf "$warning Both %s and legacy %s exist; using the latter.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2 + printf "$warning Both %s and legacy %s exist; using the former.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2 if [ "$(realpath "$NIX_LINK")" = "$(realpath "$NIX_LINK_NEW")" ]; then printf " Since the profiles match, you can safely delete either of them.\n" 1>&2 else @@ -26,6 +25,7 @@ else printf "$warning Profiles do not match. You should manually migrate from %s to %s.\n" "$NIX_LINK" "$NIX_LINK_NEW" 1>&2 fi fi + NIX_LINK="$NIX_LINK_NEW" fi export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK" @@ -69,4 +69,4 @@ else fi export PATH="$NIX_LINK/bin:@localstatedir@/nix/profiles/default/bin:$PATH" -unset NIX_LINK +unset NIX_LINK NIX_LINK_NEW diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index 44bc96e89..e868399b1 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -1,3 +1,4 @@ +# This file is tested by tests/installer/default.nix. if [ -n "$HOME" ] && [ -n "$USER" ]; then # Set up the per-user profile. @@ -9,11 +10,9 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then NIX_LINK_NEW="$HOME/.local/state/nix/profile" fi if [ -e "$NIX_LINK_NEW" ]; then - NIX_LINK="$NIX_LINK_NEW" - else - if [ -t 2 ] && [ -e "$NIX_LINK_NEW" ]; then + if [ -t 2 ] && [ -e "$NIX_LINK" ]; then warning="\033[1;35mwarning:\033[0m" - printf "$warning Both %s and legacy %s exist; using the latter.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2 + printf "$warning Both %s and legacy %s exist; using the former.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2 if [ "$(realpath "$NIX_LINK")" = "$(realpath "$NIX_LINK_NEW")" ]; then printf " Since the profiles match, you can safely delete either of them.\n" 1>&2 else @@ -26,6 +25,7 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then printf "$warning Profiles do not match. You should manually migrate from %s to %s.\n" "$NIX_LINK" "$NIX_LINK_NEW" 1>&2 fi fi + NIX_LINK="$NIX_LINK_NEW" fi # Set up environment. diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 118468477..82ad7d862 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -11,11 +11,13 @@ #include "machines.hh" #include "shared.hh" +#include "plugin.hh" #include "pathlocks.hh" #include "globals.hh" #include "serialise.hh" #include "build-result.hh" #include "store-api.hh" +#include "strings.hh" #include "derivations.hh" #include "local-store.hh" #include "legacy.hh" @@ -37,7 +39,7 @@ static std::string currentLoad; static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot) { - return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true); + return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri.render()), slot), true); } static bool allSupportedLocally(Store & store, const std::set& requiredFeatures) { @@ -135,7 +137,7 @@ static int main_build_remote(int argc, char * * argv) Machine * bestMachine = nullptr; uint64_t bestLoad = 0; for (auto & m : machines) { - debug("considering building on remote machine '%s'", m.storeUri); + debug("considering building on remote machine '%s'", m.storeUri.render()); if (m.enabled && m.systemSupported(neededSystem) && @@ -202,7 +204,7 @@ static int main_build_remote(int argc, char * * argv) else drvstr = ""; - auto error = HintFmt(errorText); + auto error = HintFmt::fromFormatString(errorText); error % drvstr % neededSystem @@ -232,17 +234,16 @@ static int main_build_remote(int argc, char * * argv) lock = -1; try { + storeUri = bestMachine->storeUri.render(); - Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri)); + Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", storeUri)); sshStore = bestMachine->openStore(); sshStore->connect(); - storeUri = bestMachine->storeUri; - } catch (std::exception & e) { auto msg = chomp(drainFD(5, false)); printError("cannot build on '%s': %s%s", - bestMachine->storeUri, e.what(), + storeUri, e.what(), msg.empty() ? "" : ": " + msg); bestMachine->enabled = false; continue; @@ -262,7 +263,20 @@ connected: auto inputs = readStrings(source); auto wantedOutputs = readStrings(source); - AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true); + AutoCloseFD uploadLock; + { + auto setUpdateLock = [&](auto && fileName){ + uploadLock = openLockFile(currentLoad + "/" + escapeUri(fileName) + ".upload-lock", true); + }; + try { + setUpdateLock(storeUri); + } catch (SysError & e) { + if (e.errNo != ENAMETOOLONG) throw; + // Try again hashing the store URL so we have a shorter path + auto h = hashString(HashAlgorithm::MD5, storeUri); + setUpdateLock(h.to_string(HashFormat::Base64, false)); + } + } { Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri)); diff --git a/doc/internal-api/.gitignore b/src/external-api-docs/.gitignore similarity index 100% rename from doc/internal-api/.gitignore rename to src/external-api-docs/.gitignore diff --git a/src/external-api-docs/.version b/src/external-api-docs/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/external-api-docs/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/external-api-docs/README.md b/src/external-api-docs/README.md new file mode 100644 index 000000000..8760ac88b --- /dev/null +++ b/src/external-api-docs/README.md @@ -0,0 +1,121 @@ +# Getting started + +> **Warning** These bindings are **experimental**, which means they can change +> at any time or be removed outright; nevertheless the plan is to provide a +> stable external C API to the Nix language and the Nix store. + +The language library allows evaluating Nix expressions and interacting with Nix +language values. The Nix store API is still rudimentary, and only allows +initialising and connecting to a store for the Nix language evaluator to +interact with. + +Currently there are two ways to interface with the Nix language evaluator +programmatically: + +1. Embedding the evaluator +2. Writing language plug-ins + +Embedding means you link the Nix C libraries in your program and use them from +there. Adding a plug-in means you make a library that gets loaded by the Nix +language evaluator, specified through a configuration option. + +Many of the components and mechanisms involved are not yet documented, therefore +please refer to the [Nix source code](https://github.com/NixOS/nix/) for +details. Additions to in-code documentation and the reference manual are highly +appreciated. + +The following examples, for simplicity, don't include error handling. See the +[Handling errors](@ref errors) section for more information. + +# Embedding the Nix Evaluator{#nix_evaluator_example} + +In this example we programmatically start the Nix language evaluator with a +dummy store (that has no store paths and cannot be written to), and evaluate the +Nix expression `builtins.nixVersion`. + +**main.c:** + +```C +#include +#include +#include +#include +#include +#include + +// NOTE: This example lacks all error handling. Production code must check for +// errors, as some return values will be undefined. + +void my_get_string_cb(const char * start, unsigned int n, void * user_data) +{ + *((char **) user_data) = strdup(start); +} + +int main() +{ + nix_libexpr_init(NULL); + + Store * store = nix_store_open(NULL, "dummy://", NULL); + EvalState * state = nix_state_create(NULL, NULL, store); // empty search path (NIX_PATH) + Value * value = nix_alloc_value(NULL, state); + + nix_expr_eval_from_string(NULL, state, "builtins.nixVersion", ".", value); + nix_value_force(NULL, state, value); + + char * version; + nix_get_string(NULL, value, my_get_string_cb, &version); + printf("Nix version: %s\n", version); + + free(version); + nix_gc_decref(NULL, value); + nix_state_free(state); + nix_store_free(store); + return 0; +} +``` + +**Usage:** + +```ShellSession +$ gcc main.c $(pkg-config nix-expr-c --libs --cflags) -o main +$ ./main +Nix version: 2.17 +``` + +# Writing a Nix language plug-in + +In this example we add a custom primitive operation (_primop_) to `builtins`. It +will increment the argument if it is an integer and throw an error otherwise. + +**plugin.c:** + +```C +#include +#include +#include + +void increment(void* user_data, nix_c_context* ctx, EvalState* state, Value** args, Value* v) { + nix_value_force(NULL, state, args[0]); + if (nix_get_type(NULL, args[0]) == NIX_TYPE_INT) { + nix_init_int(NULL, v, nix_get_int(NULL, args[0]) + 1); + } else { + nix_set_err_msg(ctx, NIX_ERR_UNKNOWN, "First argument should be an integer."); + } +} + +void nix_plugin_entry() { + const char* args[] = {"n", NULL}; + PrimOp *p = nix_alloc_primop(NULL, increment, 1, "increment", args, "Example custom built-in function: increments an integer", NULL); + nix_register_primop(NULL, p); + nix_gc_decref(NULL, p); +} +``` + +**Usage:** + +```ShellSession +$ gcc plugin.c $(pkg-config nix-expr-c --libs --cflags) -shared -o plugin.so +$ nix --plugin-files ./plugin.so repl +nix-repl> builtins.increment 1 +2 +``` diff --git a/src/external-api-docs/doxygen.cfg.in b/src/external-api-docs/doxygen.cfg.in new file mode 100644 index 000000000..1be71d895 --- /dev/null +++ b/src/external-api-docs/doxygen.cfg.in @@ -0,0 +1,58 @@ +# Doxyfile 1.9.5 + +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by +# double-quotes, unless you are using Doxywizard) that should identify the +# project for which the documentation is generated. This name is used in the +# title of most generated pages and in a few other places. +# The default value is: My Project. + +PROJECT_NAME = "Nix" + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. This +# could be handy for archiving the generated documentation or if some version +# control system is used. + +PROJECT_NUMBER = @PROJECT_NUMBER@ + +OUTPUT_DIRECTORY = @OUTPUT_DIRECTORY@ + +# Using the PROJECT_BRIEF tag one can provide an optional one line description +# for a project that appears at the top of each page and should give viewer a +# quick idea about the purpose of the project. Keep the description short. + +PROJECT_BRIEF = "Nix, the purely functional package manager: C API (experimental)" + +# If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output. +# The default value is: YES. + +GENERATE_LATEX = NO + +# The INPUT tag is used to specify the files and/or directories that contain +# documented source files. You may enter file names like myfile.cpp or +# directories like /usr/src/myproject. Separate the files or directories with +# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING +# Note: If this tag is empty the current directory is searched. + +# FIXME Make this list more maintainable somehow. We could maybe generate this +# in the Makefile, but we would need to change how `.in` files are preprocessed +# so they can expand variables despite configure variables. + +INPUT = \ + @src@/src/libutil-c \ + @src@/src/libexpr-c \ + @src@/src/libstore-c \ + @src@/doc/external-api/README.md + +FILE_PATTERNS = nix_api_*.h *.md + +# The INCLUDE_PATH tag can be used to specify one or more directories that +# contain include files that are not input files but should be processed by the +# preprocessor. Note that the INCLUDE_PATH is not recursive, so the setting of +# RECURSIVE has no effect here. +# This tag requires that the tag SEARCH_INCLUDES is set to YES. + +EXCLUDE_PATTERNS = *_internal.h +GENERATE_TREEVIEW = YES +OPTIMIZE_OUTPUT_FOR_C = YES + +USE_MDFILE_AS_MAINPAGE = doc/external-api/README.md diff --git a/src/external-api-docs/meson.build b/src/external-api-docs/meson.build new file mode 100644 index 000000000..62474ffe4 --- /dev/null +++ b/src/external-api-docs/meson.build @@ -0,0 +1,31 @@ +project('nix-external-api-docs', + version : files('.version'), + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +fs = import('fs') + +doxygen_cfg = configure_file( + input : 'doxygen.cfg.in', + output : 'doxygen.cfg', + configuration : { + 'PROJECT_NUMBER': meson.project_version(), + 'OUTPUT_DIRECTORY' : meson.current_build_dir(), + 'src' : fs.parent(fs.parent(meson.project_source_root())), + }, +) + +doxygen = find_program('doxygen', native : true, required : true) + +custom_target( + 'external-api-docs', + command : [ doxygen , doxygen_cfg ], + input : [ + doxygen_cfg, + ], + output : 'html', + install : true, + install_dir : get_option('datadir') / 'doc/nix/external-api', + build_always_stale : true, +) diff --git a/src/external-api-docs/package.nix b/src/external-api-docs/package.nix new file mode 100644 index 000000000..743b3e9b7 --- /dev/null +++ b/src/external-api-docs/package.nix @@ -0,0 +1,59 @@ +{ lib +, mkMesonDerivation + +, meson +, ninja +, doxygen + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-external-api-docs"; + inherit version; + + workDir = ./.; + fileset = + let + cpp = fileset.fileFilter (file: file.hasExt "cc" || file.hasExt "h"); + in + fileset.unions [ + ./.version + ../../.version + ./meson.build + ./doxygen.cfg.in + ./README.md + # Source is not compiled, but still must be available for Doxygen + # to gather comments. + (cpp ../libexpr-c) + (cpp ../libstore-c) + (cpp ../libutil-c) + ]; + + nativeBuildInputs = [ + meson + ninja + doxygen + ]; + + preConfigure = + '' + chmod u+w ./.version + echo ${finalAttrs.version} > ./.version + ''; + + postInstall = '' + mkdir -p ''${!outputDoc}/nix-support + echo "doc external-api-docs $out/share/doc/nix/external-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products + ''; + + meta = { + platforms = lib.platforms.all; + }; +}) diff --git a/src/internal-api-docs/.gitignore b/src/internal-api-docs/.gitignore new file mode 100644 index 000000000..dab28b6b0 --- /dev/null +++ b/src/internal-api-docs/.gitignore @@ -0,0 +1,3 @@ +/doxygen.cfg +/html +/latex diff --git a/src/internal-api-docs/.version b/src/internal-api-docs/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/internal-api-docs/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/doc/internal-api/doxygen.cfg.in b/src/internal-api-docs/doxygen.cfg.in similarity index 84% rename from doc/internal-api/doxygen.cfg.in rename to src/internal-api-docs/doxygen.cfg.in index 6c6c325bd..f1ef75b38 100644 --- a/doc/internal-api/doxygen.cfg.in +++ b/src/internal-api-docs/doxygen.cfg.in @@ -12,7 +12,9 @@ PROJECT_NAME = "Nix" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = @PACKAGE_VERSION@ +PROJECT_NUMBER = @PROJECT_NUMBER@ + +OUTPUT_DIRECTORY = @OUTPUT_DIRECTORY@ # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a @@ -36,27 +38,27 @@ GENERATE_LATEX = NO # so they can expand variables despite configure variables. INPUT = \ - src/libcmd \ - src/libexpr \ - src/libexpr/flake \ - tests/unit/libexpr \ - tests/unit/libexpr/value \ - tests/unit/libexpr/test \ - tests/unit/libexpr/test/value \ - src/libexpr/value \ - src/libfetchers \ - src/libmain \ - src/libstore \ - src/libstore/build \ - src/libstore/builtins \ - tests/unit/libstore \ - tests/unit/libstore/test \ - src/libutil \ - tests/unit/libutil \ - tests/unit/libutil/test \ - src/nix \ - src/nix-env \ - src/nix-store + @src@/libcmd \ + @src@/libexpr \ + @src@/libexpr/flake \ + @src@/nix-expr-tests \ + @src@/nix-expr-tests/value \ + @src@/nix-expr-test-support/test \ + @src@/nix-expr-test-support/test/value \ + @src@/libexpr/value \ + @src@/libfetchers \ + @src@/libmain \ + @src@/libstore \ + @src@/libstore/build \ + @src@/libstore/builtins \ + @src@/nix-store-tests \ + @src@/nix-store-test-support/test \ + @src@/libutil \ + @src@/nix-util-tests \ + @src@/nix-util-test-support/test \ + @src@/nix \ + @src@/nix-env \ + @src@/nix-store # If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names # in the source code. If set to NO, only conditional compilation will be diff --git a/src/internal-api-docs/meson.build b/src/internal-api-docs/meson.build new file mode 100644 index 000000000..54eb7e5dd --- /dev/null +++ b/src/internal-api-docs/meson.build @@ -0,0 +1,31 @@ +project('nix-internal-api-docs', + version : files('.version'), + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +fs = import('fs') + +doxygen_cfg = configure_file( + input : 'doxygen.cfg.in', + output : 'doxygen.cfg', + configuration : { + 'PROJECT_NUMBER': meson.project_version(), + 'OUTPUT_DIRECTORY' : meson.current_build_dir(), + 'src' : fs.parent(fs.parent(meson.project_source_root())) / 'src', + }, +) + +doxygen = find_program('doxygen', native : true, required : true) + +custom_target( + 'internal-api-docs', + command : [ doxygen , doxygen_cfg ], + input : [ + doxygen_cfg, + ], + output : 'html', + install : true, + install_dir : get_option('datadir') / 'doc/nix/internal-api', + build_always_stale : true, +) diff --git a/src/internal-api-docs/package.nix b/src/internal-api-docs/package.nix new file mode 100644 index 000000000..07ca6d4d9 --- /dev/null +++ b/src/internal-api-docs/package.nix @@ -0,0 +1,54 @@ +{ lib +, mkMesonDerivation + +, meson +, ninja +, doxygen + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-internal-api-docs"; + inherit version; + + workDir = ./.; + fileset = let + cpp = fileset.fileFilter (file: file.hasExt "cc" || file.hasExt "hh"); + in fileset.unions [ + ./.version + ../../.version + ./meson.build + ./doxygen.cfg.in + # Source is not compiled, but still must be available for Doxygen + # to gather comments. + (cpp ../.) + ]; + + nativeBuildInputs = [ + meson + ninja + doxygen + ]; + + preConfigure = + '' + chmod u+w ./.version + echo ${finalAttrs.version} > ./.version + ''; + + postInstall = '' + mkdir -p ''${!outputDoc}/nix-support + echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products + ''; + + meta = { + platforms = lib.platforms.all; + }; +}) diff --git a/src/libcmd/.version b/src/libcmd/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libcmd/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libcmd/build-utils-meson b/src/libcmd/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libcmd/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc index c5eb93c5d..905e70f32 100644 --- a/src/libcmd/built-path.cc +++ b/src/libcmd/built-path.cc @@ -1,6 +1,7 @@ #include "built-path.hh" #include "derivations.hh" #include "store-api.hh" +#include "comparator.hh" #include @@ -8,30 +9,24 @@ namespace nix { -#define CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, COMPARATOR) \ - bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \ - { \ - const MY_TYPE* me = this; \ - auto fields1 = std::tie(*me->drvPath, me->FIELD); \ - me = &other; \ - auto fields2 = std::tie(*me->drvPath, me->FIELD); \ - return fields1 COMPARATOR fields2; \ - } -#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, ==) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <) +// Custom implementation to avoid `ref` ptr equality +GENERATE_CMP_EXT( + , + std::strong_ordering, + SingleBuiltPathBuilt, + *me->drvPath, + me->output); -#define FIELD_TYPE std::pair -CMP(SingleBuiltPath, SingleBuiltPathBuilt, output) -#undef FIELD_TYPE +// Custom implementation to avoid `ref` ptr equality -#define FIELD_TYPE std::map -CMP(SingleBuiltPath, BuiltPathBuilt, outputs) -#undef FIELD_TYPE - -#undef CMP -#undef CMP_ONE +// TODO no `GENERATE_CMP_EXT` because no `std::set::operator<=>` on +// Darwin, per header. +GENERATE_EQUAL( + , + BuiltPathBuilt ::, + BuiltPathBuilt, + *me->drvPath, + me->outputs); StorePath SingleBuiltPath::outPath() const { diff --git a/src/libcmd/built-path.hh b/src/libcmd/built-path.hh index 99917e0ee..dc78d3e59 100644 --- a/src/libcmd/built-path.hh +++ b/src/libcmd/built-path.hh @@ -18,7 +18,8 @@ struct SingleBuiltPathBuilt { static SingleBuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view); nlohmann::json toJSON(const StoreDirConfig & store) const; - DECLARE_CMP(SingleBuiltPathBuilt); + bool operator ==(const SingleBuiltPathBuilt &) const noexcept; + std::strong_ordering operator <=>(const SingleBuiltPathBuilt &) const noexcept; }; using _SingleBuiltPathRaw = std::variant< @@ -33,6 +34,9 @@ struct SingleBuiltPath : _SingleBuiltPathRaw { using Opaque = DerivedPathOpaque; using Built = SingleBuiltPathBuilt; + bool operator == (const SingleBuiltPath &) const = default; + auto operator <=> (const SingleBuiltPath &) const = default; + inline const Raw & raw() const { return static_cast(*this); } @@ -59,11 +63,13 @@ struct BuiltPathBuilt { ref drvPath; std::map outputs; + bool operator == (const BuiltPathBuilt &) const noexcept; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //std::strong_ordering operator <=> (const BuiltPathBuilt &) const noexcept; + std::string to_string(const StoreDirConfig & store) const; static BuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view); nlohmann::json toJSON(const StoreDirConfig & store) const; - - DECLARE_CMP(BuiltPathBuilt); }; using _BuiltPathRaw = std::variant< @@ -82,6 +88,10 @@ struct BuiltPath : _BuiltPathRaw { using Opaque = DerivedPathOpaque; using Built = BuiltPathBuilt; + bool operator == (const BuiltPath &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=> (const BuiltPath &) const = default; + inline const Raw & raw() const { return static_cast(*this); } diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 369fa6004..67fef1909 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -1,3 +1,5 @@ +#include + #include "command.hh" #include "markdown.hh" #include "store-api.hh" @@ -6,8 +8,7 @@ #include "nixexpr.hh" #include "profiles.hh" #include "repl.hh" - -#include +#include "strings.hh" extern char * * environ __attribute__((weak)); @@ -127,12 +128,12 @@ ref EvalCommand::getEvalState() if (!evalState) { evalState = #if HAVE_BOEHMGC - std::allocate_shared(traceable_allocator(), - searchPath, getEvalStore(), getStore()) + std::allocate_shared( + traceable_allocator(), #else std::make_shared( - searchPath, getEvalStore(), getStore()) #endif + lookupPath, getEvalStore(), fetchSettings, evalSettings, getStore()) ; evalState->repair = repair; @@ -148,7 +149,7 @@ MixOperateOnOptions::MixOperateOnOptions() { addFlag({ .longName = "derivation", - .description = "Operate on the [store derivation](../../glossary.md#gloss-store-derivation) rather than its outputs.", + .description = "Operate on the [store derivation](@docroot@/glossary.md#gloss-store-derivation) rather than its outputs.", .category = installablesCategory, .handler = {&operateOn, OperateOn::Derivation}, }); diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 444ff81c9..fcef92487 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -1,18 +1,57 @@ +#include "fetch-settings.hh" #include "eval-settings.hh" #include "common-eval-args.hh" #include "shared.hh" +#include "config-global.hh" #include "filetransfer.hh" #include "eval.hh" #include "fetchers.hh" #include "registry.hh" #include "flake/flakeref.hh" +#include "flake/settings.hh" #include "store-api.hh" #include "command.hh" #include "tarball.hh" #include "fetch-to-store.hh" +#include "compatibility-settings.hh" +#include "eval-settings.hh" namespace nix { +fetchers::Settings fetchSettings; + +static GlobalConfig::Register rFetchSettings(&fetchSettings); + +EvalSettings evalSettings { + settings.readOnlyMode, + { + { + "flake", + [](ref store, std::string_view rest) { + experimentalFeatureSettings.require(Xp::Flakes); + // FIXME `parseFlakeRef` should take a `std::string_view`. + auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false); + debug("fetching flake search path element '%s''", rest); + auto storePath = flakeRef.resolve(store).fetchTree(store).first; + return store->toRealPath(storePath); + }, + }, + }, +}; + +static GlobalConfig::Register rEvalSettings(&evalSettings); + + +flake::Settings flakeSettings; + +static GlobalConfig::Register rFlakeSettings(&flakeSettings); + + +CompatibilitySettings compatibilitySettings {}; + +static GlobalConfig::Register rCompatibilitySettings(&compatibilitySettings); + + MixEvalArgs::MixEvalArgs() { addFlag({ @@ -20,7 +59,7 @@ MixEvalArgs::MixEvalArgs() .description = "Pass the value *expr* as the argument *name* to Nix functions.", .category = category, .labels = {"name", "expr"}, - .handler = {[&](std::string name, std::string expr) { autoArgs[name] = 'E' + expr; }} + .handler = {[&](std::string name, std::string expr) { autoArgs.insert_or_assign(name, AutoArg{AutoArgExpr{expr}}); }} }); addFlag({ @@ -28,87 +67,40 @@ MixEvalArgs::MixEvalArgs() .description = "Pass the string *string* as the argument *name* to Nix functions.", .category = category, .labels = {"name", "string"}, - .handler = {[&](std::string name, std::string s) { autoArgs[name] = 'S' + s; }}, + .handler = {[&](std::string name, std::string s) { autoArgs.insert_or_assign(name, AutoArg{AutoArgString{s}}); }}, + }); + + addFlag({ + .longName = "arg-from-file", + .description = "Pass the contents of file *path* as the argument *name* to Nix functions.", + .category = category, + .labels = {"name", "path"}, + .handler = {[&](std::string name, std::string path) { autoArgs.insert_or_assign(name, AutoArg{AutoArgFile{path}}); }}, + .completer = completePath + }); + + addFlag({ + .longName = "arg-from-stdin", + .description = "Pass the contents of stdin as the argument *name* to Nix functions.", + .category = category, + .labels = {"name"}, + .handler = {[&](std::string name) { autoArgs.insert_or_assign(name, AutoArg{AutoArgStdin{}}); }}, }); addFlag({ .longName = "include", .shortName = 'I', .description = R"( - Add *path* to the Nix search path. The Nix search path is - initialized from the colon-separated [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) environment - variable, and is used to look up the location of Nix expressions using [paths](@docroot@/language/values.md#type-path) enclosed in angle - brackets (i.e., ``). + Add *path* to search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md) - For instance, passing + This option may be given multiple times. - ``` - -I /home/eelco/Dev - -I /etc/nixos - ``` - - will cause Nix to look for paths relative to `/home/eelco/Dev` and - `/etc/nixos`, in that order. This is equivalent to setting the - `NIX_PATH` environment variable to - - ``` - /home/eelco/Dev:/etc/nixos - ``` - - It is also possible to match paths against a prefix. For example, - passing - - ``` - -I nixpkgs=/home/eelco/Dev/nixpkgs-branch - -I /etc/nixos - ``` - - will cause Nix to search for `` in - `/home/eelco/Dev/nixpkgs-branch/path` and `/etc/nixos/nixpkgs/path`. - - If a path in the Nix search path starts with `http://` or `https://`, - it is interpreted as the URL of a tarball that will be downloaded and - unpacked to a temporary location. The tarball must consist of a single - top-level directory. For example, passing - - ``` - -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/master.tar.gz - ``` - - tells Nix to download and use the current contents of the `master` - branch in the `nixpkgs` repository. - - The URLs of the tarballs from the official `nixos.org` channels - (see [the manual page for `nix-channel`](../nix-channel.md)) can be - abbreviated as `channel:`. For instance, the - following two flags are equivalent: - - ``` - -I nixpkgs=channel:nixos-21.05 - -I nixpkgs=https://nixos.org/channels/nixos-21.05/nixexprs.tar.xz - ``` - - You can also fetch source trees using [flake URLs](./nix3-flake.md#url-like-syntax) and add them to the - search path. For instance, - - ``` - -I nixpkgs=flake:nixpkgs - ``` - - specifies that the prefix `nixpkgs` shall refer to the source tree - downloaded from the `nixpkgs` entry in the flake registry. Similarly, - - ``` - -I nixpkgs=flake:github:NixOS/nixpkgs/nixos-22.05 - ``` - - makes `` refer to a particular branch of the - `NixOS/nixpkgs` repository on GitHub. + Paths added through `-I` take precedence over the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path) and the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH). )", .category = category, .labels = {"path"}, .handler = {[&](std::string s) { - searchPath.elements.emplace_back(SearchPath::Elem::parse(s)); + lookupPath.elements.emplace_back(LookupPath::Elem::parse(s)); }} }); @@ -127,8 +119,8 @@ MixEvalArgs::MixEvalArgs() .category = category, .labels = {"original-ref", "resolved-ref"}, .handler = {[&](std::string _from, std::string _to) { - auto from = parseFlakeRef(_from, absPath(".")); - auto to = parseFlakeRef(_to, absPath(".")); + auto from = parseFlakeRef(fetchSettings, _from, absPath(".")); + auto to = parseFlakeRef(fetchSettings, _to, absPath(".")); fetchers::Attrs extraAttrs; if (to.subdir != "") extraAttrs["dir"] = to.subdir; fetchers::overrideRegistry(from.input, to.input, extraAttrs); @@ -154,13 +146,23 @@ MixEvalArgs::MixEvalArgs() Bindings * MixEvalArgs::getAutoArgs(EvalState & state) { auto res = state.buildBindings(autoArgs.size()); - for (auto & i : autoArgs) { + for (auto & [name, arg] : autoArgs) { auto v = state.allocValue(); - if (i.second[0] == 'E') - state.mkThunk_(*v, state.parseExprFromString(i.second.substr(1), state.rootPath("."))); - else - v->mkString(((std::string_view) i.second).substr(1)); - res.insert(state.symbols.create(i.first), v); + std::visit(overloaded { + [&](const AutoArgExpr & arg) { + state.mkThunk_(*v, state.parseExprFromString(arg.expr, compatibilitySettings.nixShellShebangArgumentsRelativeToScript ? state.rootPath(absPath(getCommandBaseDir())) : state.rootPath("."))); + }, + [&](const AutoArgString & arg) { + v->mkString(arg.s); + }, + [&](const AutoArgFile & arg) { + v->mkString(readFile(arg.path.string())); + }, + [&](const AutoArgStdin & arg) { + v->mkString(readFile(STDIN_FILENO)); + } + }, arg); + res.insert(state.symbols.create(name), v); } return res.finish(); } @@ -176,7 +178,7 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas else if (hasPrefix(s, "flake:")) { experimentalFeatureSettings.require(Xp::Flakes); - auto flakeRef = parseFlakeRef(std::string(s.substr(6)), {}, true, false); + auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false); auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first; return state.rootPath(CanonPath(state.store->toRealPath(storePath))); } diff --git a/src/libcmd/common-eval-args.hh b/src/libcmd/common-eval-args.hh index 2eb63e15d..c62365b32 100644 --- a/src/libcmd/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh @@ -6,13 +6,42 @@ #include "common-args.hh" #include "search-path.hh" +#include + namespace nix { class Store; + +namespace fetchers { struct Settings; } + class EvalState; +struct EvalSettings; +struct CompatibilitySettings; class Bindings; struct SourcePath; +namespace flake { struct Settings; } + +/** + * @todo Get rid of global setttings variables + */ +extern fetchers::Settings fetchSettings; + +/** + * @todo Get rid of global setttings variables + */ +extern EvalSettings evalSettings; + +/** + * @todo Get rid of global setttings variables + */ +extern flake::Settings flakeSettings; + +/** + * Settings that control behaviors that have changed since Nix 2.3. + */ +extern CompatibilitySettings compatibilitySettings; + struct MixEvalArgs : virtual Args, virtual MixRepair { static constexpr auto category = "Common evaluation options"; @@ -21,14 +50,24 @@ struct MixEvalArgs : virtual Args, virtual MixRepair Bindings * getAutoArgs(EvalState & state); - SearchPath searchPath; + LookupPath lookupPath; std::optional evalStoreUrl; private: - std::map autoArgs; + struct AutoArgExpr { std::string expr; }; + struct AutoArgString { std::string s; }; + struct AutoArgFile { std::filesystem::path path; }; + struct AutoArgStdin { }; + + using AutoArg = std::variant; + + std::map autoArgs; }; +/** + * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) + */ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * baseDir = nullptr); } diff --git a/src/libcmd/compatibility-settings.hh b/src/libcmd/compatibility-settings.hh new file mode 100644 index 000000000..a129a957a --- /dev/null +++ b/src/libcmd/compatibility-settings.hh @@ -0,0 +1,36 @@ +#pragma once +#include "config.hh" + +namespace nix { +struct CompatibilitySettings : public Config +{ + + CompatibilitySettings() = default; + + // Added in Nix 2.24, July 2024. + Setting nixShellAlwaysLooksForShellNix{this, true, "nix-shell-always-looks-for-shell-nix", R"( + Before Nix 2.24, [`nix-shell`](@docroot@/command-ref/nix-shell.md) would only look at `shell.nix` if it was in the working directory - when no file was specified. + + Since Nix 2.24, `nix-shell` always looks for a `shell.nix`, whether that's in the working directory, or in a directory that was passed as an argument. + + You may set this to `false` to temporarily revert to the behavior of Nix 2.23 and older. + + Using this setting is not recommended. + It will be deprecated and removed. + )"}; + + // Added in Nix 2.24, July 2024. + Setting nixShellShebangArgumentsRelativeToScript{ + this, true, "nix-shell-shebang-arguments-relative-to-script", R"( + Before Nix 2.24, relative file path expressions in arguments in a `nix-shell` shebang were resolved relative to the working directory. + + Since Nix 2.24, `nix-shell` resolves these paths in a manner that is relative to the [base directory](@docroot@/glossary.md#gloss-base-directory), defined as the script's directory. + + You may set this to `false` to temporarily revert to the behavior of Nix 2.23 and older. + + Using this setting is not recommended. + It will be deprecated and removed. + )"}; +}; + +}; diff --git a/src/libcmd/installable-attr-path.cc b/src/libcmd/installable-attr-path.cc index 3ec1c1614..8917e7a01 100644 --- a/src/libcmd/installable-attr-path.cc +++ b/src/libcmd/installable-attr-path.cc @@ -75,6 +75,8 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths() std::set outputsToInstall; for (auto & output : packageInfo.queryOutputs(false, true)) outputsToInstall.insert(output.first); + if (outputsToInstall.empty()) + outputsToInstall.insert("out"); return OutputsSpec::Names { std::move(outputsToInstall) }; }, [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec { diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index ddec7537b..8796ad5ba 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -43,20 +43,6 @@ std::vector InstallableFlake::getActualAttrPaths() return res; } -Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake) -{ - auto vFlake = state.allocValue(); - - callFlake(state, lockedFlake, *vFlake); - - auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); - assert(aOutputs); - - state.forceValue(*aOutputs->value, aOutputs->value->determinePos(noPos)); - - return aOutputs->value; -} - static std::string showAttrPaths(const std::vector & paths) { std::string s; @@ -106,9 +92,14 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() fmt("while evaluating the flake output attribute '%s'", attrPath))) { return { *derivedPathWithInfo }; + } else { + throw Error( + "expected flake output attribute '%s' to be a derivation or path but found %s: %s", + attrPath, + showType(v), + ValuePrinter(*this->state, v, errorPrintOptions) + ); } - else - throw Error("flake output attribute '%s' is not a derivation or path", attrPath); } auto drvPath = attr->forceDerivation(); @@ -205,7 +196,8 @@ std::shared_ptr InstallableFlake::getLockedFlake() const flake::LockFlags lockFlagsApplyConfig = lockFlags; // FIXME why this side effect? lockFlagsApplyConfig.applyNixConfig = true; - _lockedFlake = std::make_shared(lockFlake(*state, flakeRef, lockFlagsApplyConfig)); + _lockedFlake = std::make_shared(lockFlake( + flakeSettings, *state, flakeRef, lockFlagsApplyConfig)); } return _lockedFlake; } diff --git a/src/libcmd/installable-flake.hh b/src/libcmd/installable-flake.hh index 314918c14..8e0a232ef 100644 --- a/src/libcmd/installable-flake.hh +++ b/src/libcmd/installable-flake.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "common-eval-args.hh" #include "installable-value.hh" namespace nix { @@ -52,8 +53,6 @@ struct InstallableFlake : InstallableValue std::vector getActualAttrPaths(); - Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake); - DerivedPathsWithInfo toDerivedPaths() override; std::pair toValue(EvalState & state) override; @@ -80,7 +79,7 @@ struct InstallableFlake : InstallableValue */ static inline FlakeRef defaultNixpkgsFlakeRef() { - return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}}); + return FlakeRef::fromAttrs(fetchSettings, {{"type","indirect"}, {"id", "nixpkgs"}}); } ref openEvalCache( diff --git a/src/libcmd/installable-value.hh b/src/libcmd/installable-value.hh index f300d392b..798cb5e1a 100644 --- a/src/libcmd/installable-value.hh +++ b/src/libcmd/installable-value.hh @@ -66,7 +66,7 @@ struct ExtraPathInfoValue : ExtraPathInfo }; /** - * An Installable which corresponds a Nix langauge value, in addition to + * An Installable which corresponds a Nix language value, in addition to * a collection of \ref DerivedPath "derived paths". */ struct InstallableValue : Installable diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 16d25d3cf..0fe956ec0 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -27,6 +27,8 @@ #include +#include "strings-inline.hh" + namespace nix { void completeFlakeInputPath( @@ -129,7 +131,7 @@ MixFlakeOptions::MixFlakeOptions() lockFlags.writeLockFile = false; lockFlags.inputOverrides.insert_or_assign( flake::parseInputPath(inputPath), - parseFlakeRef(flakeRef, absPath(getCommandBaseDir()), true)); + parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir()), true)); }}, .completer = {[&](AddCompletions & completions, size_t n, std::string_view prefix) { if (n == 0) { @@ -146,7 +148,7 @@ MixFlakeOptions::MixFlakeOptions() .category = category, .labels = {"flake-lock-path"}, .handler = {[&](std::string lockFilePath) { - lockFlags.referenceLockFilePath = lockFilePath; + lockFlags.referenceLockFilePath = {getFSSourceAccessor(), CanonPath(absPath(lockFilePath))}; }}, .completer = completePath }); @@ -170,14 +172,15 @@ MixFlakeOptions::MixFlakeOptions() .handler = {[&](std::string flakeRef) { auto evalState = getEvalState(); auto flake = flake::lockFlake( + flakeSettings, *evalState, - parseFlakeRef(flakeRef, absPath(getCommandBaseDir())), + parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir())), { .writeLockFile = false }); for (auto & [inputName, input] : flake.lockFile.root->inputs) { auto input2 = flake.lockFile.findInput({inputName}); // resolve 'follows' nodes if (auto input3 = std::dynamic_pointer_cast(input2)) { overrideRegistry( - fetchers::Input::fromAttrs({{"type","indirect"}, {"id", inputName}}), + fetchers::Input::fromAttrs(fetchSettings, {{"type","indirect"}, {"id", inputName}}), input3->lockedRef.input, {}); } @@ -288,11 +291,11 @@ void SourceExprCommand::completeInstallable(AddCompletions & completions, std::s state->autoCallFunction(*autoArgs, v1, v2); if (v2.type() == nAttrs) { - for (auto & i : *v2.attrs) { - std::string name = state->symbols[i.name]; + for (auto & i : *v2.attrs()) { + std::string_view name = state->symbols[i.name]; if (name.find(searchWord) == 0) { if (prefix_ == "") - completions.add(name); + completions.add(std::string(name)); else completions.add(prefix_ + "." + name); } @@ -338,10 +341,11 @@ void completeFlakeRefWithFragment( auto flakeRefS = std::string(prefix.substr(0, hash)); // TODO: ideally this would use the command base directory instead of assuming ".". - auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath(".")); + auto flakeRef = parseFlakeRef(fetchSettings, expandTilde(flakeRefS), absPath(".")); auto evalCache = openEvalCache(*evalState, - std::make_shared(lockFlake(*evalState, flakeRef, lockFlags))); + std::make_shared(lockFlake( + flakeSettings, *evalState, flakeRef, lockFlags))); auto root = evalCache->getRoot(); @@ -372,6 +376,7 @@ void completeFlakeRefWithFragment( auto attrPath2 = (*attr)->getAttrPath(attr2); /* Strip the attrpath prefix. */ attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size()); + // FIXME: handle names with dots completions.add(flakeRefS + "#" + prefixRoot + concatStringsSep(".", evalState->symbols.resolve(attrPath2))); } } @@ -403,7 +408,7 @@ void completeFlakeRef(AddCompletions & completions, ref store, std::strin Args::completeDir(completions, 0, prefix); /* Look for registry entries that match the prefix. */ - for (auto & registry : fetchers::getRegistries(store)) { + for (auto & registry : fetchers::getRegistries(fetchSettings, store)) { for (auto & entry : registry->entries) { auto from = entry.from.to_string(); if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) { @@ -442,13 +447,10 @@ ref openEvalCache( EvalState & state, std::shared_ptr lockedFlake) { - auto fingerprint = lockedFlake->getFingerprint(); - return make_ref( - evalSettings.useEvalCache && evalSettings.pureEval - ? std::optional { std::cref(fingerprint) } - : std::nullopt, - state, - [&state, lockedFlake]() + auto fingerprint = evalSettings.useEvalCache && evalSettings.pureEval + ? lockedFlake->getFingerprint(state.store) + : std::nullopt; + auto rootLoader = [&state, lockedFlake]() { /* For testing whether the evaluation cache is complete. */ @@ -460,11 +462,21 @@ ref openEvalCache( state.forceAttrs(*vFlake, noPos, "while parsing cached flake data"); - auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); + auto aOutputs = vFlake->attrs()->get(state.symbols.create("outputs")); assert(aOutputs); return aOutputs->value; - }); + }; + + if (fingerprint) { + auto search = state.evalCaches.find(fingerprint.value()); + if (search == state.evalCaches.end()) { + search = state.evalCaches.emplace(fingerprint.value(), make_ref(fingerprint, state, rootLoader)).first; + } + return search->second; + } else { + return make_ref(std::nullopt, state, rootLoader); + } } Installables SourceExprCommand::parseInstallables( @@ -527,7 +539,8 @@ Installables SourceExprCommand::parseInstallables( } try { - auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, absPath(getCommandBaseDir())); + auto [flakeRef, fragment] = parseFlakeRefWithFragment( + fetchSettings, std::string { prefix }, absPath(getCommandBaseDir())); result.push_back(make_ref( this, getEvalState(), @@ -594,6 +607,37 @@ std::vector Installable::build( return res; } +static void throwBuildErrors( + std::vector & buildResults, + const Store & store) +{ + std::vector failed; + for (auto & buildResult : buildResults) { + if (!buildResult.success()) { + failed.push_back(buildResult); + } + } + + auto failedResult = failed.begin(); + if (failedResult != failed.end()) { + if (failed.size() == 1) { + failedResult->rethrow(); + } else { + StringSet failedPaths; + for (; failedResult != failed.end(); failedResult++) { + if (!failedResult->errorMsg.empty()) { + logError(ErrorInfo{ + .level = lvlError, + .msg = failedResult->errorMsg, + }); + } + failedPaths.insert(failedResult->path.to_string(store)); + } + throw Error("build of %s failed", concatStringsSep(", ", quoteStrings(failedPaths))); + } + } +} + std::vector, BuiltPathWithResult>> Installable::build2( ref evalStore, ref store, @@ -655,10 +699,9 @@ std::vector, BuiltPathWithResult>> Installable::build if (settings.printMissing) printMissing(store, pathsToBuild, lvlInfo); - for (auto & buildResult : store->buildPathsWithResults(pathsToBuild, bMode, evalStore)) { - if (!buildResult.success()) - buildResult.rethrow(); - + auto buildResults = store->buildPathsWithResults(pathsToBuild, bMode, evalStore); + throwBuildErrors(buildResults, *store); + for (auto & buildResult : buildResults) { for (auto & aux : backmap[buildResult.path]) { std::visit(overloaded { [&](const DerivedPath::Built & bfd) { @@ -814,6 +857,7 @@ std::vector RawInstallablesCommand::getFlakeRefsForCompletion() std::vector res; for (auto i : rawInstallables) res.push_back(parseFlakeRefWithFragment( + fetchSettings, expandTilde(i), absPath(getCommandBaseDir())).first); return res; @@ -836,6 +880,7 @@ std::vector InstallableCommand::getFlakeRefsForCompletion() { return { parseFlakeRefWithFragment( + fetchSettings, expandTilde(_installable), absPath(getCommandBaseDir())).first }; diff --git a/src/libcmd/local.mk b/src/libcmd/local.mk index abb7459a7..a270333f4 100644 --- a/src/libcmd/local.mk +++ b/src/libcmd/local.mk @@ -6,10 +6,10 @@ libcmd_DIR := $(d) libcmd_SOURCES := $(wildcard $(d)/*.cc) -libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers +libcmd_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) $(INCLUDE_libflake) $(INCLUDE_libmain) libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) $(THREAD_LDFLAGS) -libcmd_LIBS = libstore libutil libexpr libmain libfetchers +libcmd_LIBS = libutil libstore libfetchers libflake libexpr libmain $(eval $(call install-file-in, $(buildprefix)$(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644)) diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc index a4e3c5a77..6a0d05d9f 100644 --- a/src/libcmd/markdown.cc +++ b/src/libcmd/markdown.cc @@ -1,21 +1,23 @@ #include "markdown.hh" -#include "util.hh" +#include "environment-variables.hh" +#include "error.hh" #include "finally.hh" #include "terminal.hh" -#include #if HAVE_LOWDOWN -#include +# include +# include #endif namespace nix { -std::string renderMarkdownToTerminal(std::string_view markdown) -{ #if HAVE_LOWDOWN +static std::string doRenderMarkdownToTerminal(std::string_view markdown) +{ int windowWidth = getWindowSize().second; - struct lowdown_opts opts { + struct lowdown_opts opts + { .type = LOWDOWN_TERM, .maxdepth = 20, .cols = (size_t) std::max(windowWidth - 5, 60), @@ -50,10 +52,22 @@ std::string renderMarkdownToTerminal(std::string_view markdown) if (!rndr_res) throw Error("allocation error while rendering Markdown"); - return filterANSIEscapes(std::string(buf->data, buf->size), !shouldANSI()); -#else - return std::string(markdown); -#endif + return filterANSIEscapes(std::string(buf->data, buf->size), !isTTY()); } +std::string renderMarkdownToTerminal(std::string_view markdown) +{ + if (auto e = getEnv("_NIX_TEST_RAW_MARKDOWN"); e && *e == "1") + return std::string(markdown); + else + return doRenderMarkdownToTerminal(markdown); } + +#else +std::string renderMarkdownToTerminal(std::string_view markdown) +{ + return std::string(markdown); +} +#endif + +} // namespace nix diff --git a/src/libcmd/markdown.hh b/src/libcmd/markdown.hh index a04d32a4f..66db1736c 100644 --- a/src/libcmd/markdown.hh +++ b/src/libcmd/markdown.hh @@ -1,10 +1,17 @@ #pragma once ///@file -#include "types.hh" +#include namespace nix { +/** + * Render the given Markdown text to the terminal. + * + * If Nix is compiled without Markdown support, this function will return the input text as-is. + * + * The renderer takes into account the terminal width, and wraps text accordingly. + */ std::string renderMarkdownToTerminal(std::string_view markdown); } diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build new file mode 100644 index 000000000..c484cf998 --- /dev/null +++ b/src/libcmd/meson.build @@ -0,0 +1,130 @@ +project('nix-cmd', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ +] +deps_public_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), + dependency('nix-fetchers'), + dependency('nix-expr'), + dependency('nix-flake'), + dependency('nix-main'), +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') +deps_public += nlohmann_json + +lowdown = dependency('lowdown', version : '>= 0.9.0', required : get_option('markdown')) +deps_private += lowdown +configdata.set('HAVE_LOWDOWN', lowdown.found().to_int()) + +readline_flavor = get_option('readline-flavor') +if readline_flavor == 'editline' + editline = dependency('libeditline', 'editline', version : '>=1.14') + deps_private += editline +elif readline_flavor == 'readline' + readline = dependency('readline') + deps_private += readline + configdata.set( + 'USE_READLINE', + 1, + description: 'Use readline instead of editline', + ) +else + error('illegal editline flavor', readline_flavor) +endif + +config_h = configure_file( + configuration : configdata, + output : 'config-cmd.hh', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + # '-include', 'config-fetchers.h', + '-include', 'config-expr.hh', + '-include', 'config-main.hh', + '-include', 'config-cmd.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'built-path.cc', + 'command-installable-value.cc', + 'command.cc', + 'common-eval-args.cc', + 'editor-for.cc', + 'installable-attr-path.cc', + 'installable-derived-path.cc', + 'installable-flake.cc', + 'installable-value.cc', + 'installables.cc', + 'legacy.cc', + 'markdown.cc', + 'misc-store-flags.cc', + 'network-proxy.cc', + 'repl-interacter.cc', + 'repl.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'built-path.hh', + 'command-installable-value.hh', + 'command.hh', + 'common-eval-args.hh', + 'compatibility-settings.hh', + 'editor-for.hh', + 'installable-attr-path.hh', + 'installable-derived-path.hh', + 'installable-flake.hh', + 'installable-value.hh', + 'installables.hh', + 'legacy.hh', + 'markdown.hh', + 'misc-store-flags.hh', + 'network-proxy.hh', + 'repl-interacter.hh', + 'repl.hh', +) + +this_library = library( + 'nixcmd', + sources, + dependencies : deps_public + deps_private + deps_other, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libcmd/meson.options b/src/libcmd/meson.options new file mode 100644 index 000000000..79ae4fa55 --- /dev/null +++ b/src/libcmd/meson.options @@ -0,0 +1,15 @@ +# vim: filetype=meson + +option( + 'markdown', + type: 'feature', + description: 'Enable Markdown rendering in the Nix binary (requires lowdown)', +) + +option( + 'readline-flavor', + type : 'combo', + choices : ['editline', 'readline'], + value : 'editline', + description : 'Which library to use for nice line editing with the Nix language REPL', +) diff --git a/src/libcmd/misc-store-flags.cc b/src/libcmd/misc-store-flags.cc index e66d3f63b..06552c032 100644 --- a/src/libcmd/misc-store-flags.cc +++ b/src/libcmd/misc-store-flags.cc @@ -81,9 +81,15 @@ Args::Flag fileIngestionMethod(FileIngestionMethod * method) How to compute the hash of the input. One of: - - `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function. + - `nar` (the default): + Serialises the input as a + [Nix Archive](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) + and passes that to the hash function. - - `flat`: Assumes that the input is a single file and directly passes it to the hash function; + - `flat`: + Assumes that the input is a single file and + [directly passes](@docroot@/store/file-system-object/content-address.md#serial-flat) + it to the hash function. )", .labels = {"file-ingestion-method"}, .handler = {[method](std::string s) { @@ -101,15 +107,23 @@ Args::Flag contentAddressMethod(ContentAddressMethod * method) How to compute the content-address of the store object. One of: - - `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function. + - [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive) + (the default): + Serialises the input as a + [Nix Archive](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) + and passes that to the hash function. - - `flat`: Assumes that the input is a single file and directly passes it to the hash function; + - [`flat`](@docroot@/store/store-object/content-address.md#method-flat): + Assumes that the input is a single file and + [directly passes](@docroot@/store/file-system-object/content-address.md#serial-flat) + it to the hash function. - - `text`: Like `flat`, but used for - [derivations](@docroot@/glossary.md#store-derivation) serialized in store object and + - [`text`](@docroot@/store/store-object/content-address.md#method-text): + Like `flat`, but used for + [derivations](@docroot@/glossary.md#store-derivation) serialized in store object and [`builtins.toFile`](@docroot@/language/builtins.html#builtins-toFile). For advanced use-cases only; - for regular usage prefer `nar` and `flat. + for regular usage prefer `nar` and `flat`. )", .labels = {"content-address-method"}, .handler = {[method](std::string s) { diff --git a/src/libcmd/network-proxy.cc b/src/libcmd/network-proxy.cc new file mode 100644 index 000000000..738bf6147 --- /dev/null +++ b/src/libcmd/network-proxy.cc @@ -0,0 +1,50 @@ +#include "network-proxy.hh" + +#include + +#include "environment-variables.hh" + +namespace nix { + +static const StringSet lowercaseVariables{"http_proxy", "https_proxy", "ftp_proxy", "all_proxy", "no_proxy"}; + +static StringSet getAllVariables() +{ + StringSet variables = lowercaseVariables; + for (const auto & variable : lowercaseVariables) { + std::string upperVariable; + std::transform( + variable.begin(), variable.end(), upperVariable.begin(), [](unsigned char c) { return std::toupper(c); }); + variables.insert(std::move(upperVariable)); + } + return variables; +} + +const StringSet networkProxyVariables = getAllVariables(); + +static StringSet getExcludingNoProxyVariables() +{ + static const StringSet excludeVariables{"no_proxy", "NO_PROXY"}; + StringSet variables; + std::set_difference( + networkProxyVariables.begin(), + networkProxyVariables.end(), + excludeVariables.begin(), + excludeVariables.end(), + std::inserter(variables, variables.begin())); + return variables; +} + +static const StringSet excludingNoProxyVariables = getExcludingNoProxyVariables(); + +bool haveNetworkProxyConnection() +{ + for (const auto & variable : excludingNoProxyVariables) { + if (getEnv(variable).has_value()) { + return true; + } + } + return false; +} + +} diff --git a/src/libcmd/network-proxy.hh b/src/libcmd/network-proxy.hh new file mode 100644 index 000000000..0b6856acb --- /dev/null +++ b/src/libcmd/network-proxy.hh @@ -0,0 +1,22 @@ +#pragma once +///@file + +#include "types.hh" + +namespace nix { + +/** + * Environment variables relating to network proxying. These are used by + * a few misc commands. + * + * See the Environment section of https://curl.se/docs/manpage.html for details. + */ +extern const StringSet networkProxyVariables; + +/** + * Heuristically check if there is a proxy connection by checking for defined + * proxy variables. + */ +bool haveNetworkProxyConnection(); + +} diff --git a/src/libcmd/package.nix b/src/libcmd/package.nix new file mode 100644 index 000000000..cde494901 --- /dev/null +++ b/src/libcmd/package.nix @@ -0,0 +1,104 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-util +, nix-store +, nix-fetchers +, nix-expr +, nix-flake +, nix-main +, editline +, readline +, lowdown +, nlohmann_json + +# Configuration Options + +, version + +# Whether to enable Markdown rendering in the Nix binary. +, enableMarkdown ? !stdenv.hostPlatform.isWindows + +# Which interactive line editor library to use for Nix's repl. +# +# Currently supported choices are: +# +# - editline (default) +# - readline +, readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline" +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-cmd"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + ({ inherit editline readline; }.${readlineFlavor}) + ] ++ lib.optional enableMarkdown lowdown; + + propagatedBuildInputs = [ + nix-util + nix-store + nix-fetchers + nix-expr + nix-flake + nix-main + nlohmann_json + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + mesonFlags = [ + (lib.mesonEnable "markdown" enableMarkdown) + (lib.mesonOption "readline-flavor" readlineFlavor) + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libcmd/repl-interacter.cc b/src/libcmd/repl-interacter.cc new file mode 100644 index 000000000..187af46ea --- /dev/null +++ b/src/libcmd/repl-interacter.cc @@ -0,0 +1,206 @@ +#include + +#ifdef USE_READLINE +#include +#include +#else +// editline < 1.15.2 don't wrap their API for C++ usage +// (added in https://github.com/troglobit/editline/commit/91398ceb3427b730995357e9d120539fb9bb7461). +// This results in linker errors due to to name-mangling of editline C symbols. +// For compatibility with these versions, we wrap the API here +// (wrapping multiple times on newer versions is no problem). +extern "C" { +#include +} +#endif + +#include "signals.hh" +#include "finally.hh" +#include "repl-interacter.hh" +#include "file-system.hh" +#include "repl.hh" +#include "environment-variables.hh" + +namespace nix { + +namespace { +// Used to communicate to NixRepl::getLine whether a signal occurred in ::readline. +volatile sig_atomic_t g_signal_received = 0; + +void sigintHandler(int signo) +{ + g_signal_received = signo; +} +}; + +static detail::ReplCompleterMixin * curRepl; // ugly + +#ifndef USE_READLINE +static char * completionCallback(char * s, int * match) +{ + auto possible = curRepl->completePrefix(s); + if (possible.size() == 1) { + *match = 1; + auto * res = strdup(possible.begin()->c_str() + strlen(s)); + if (!res) + throw Error("allocation failure"); + return res; + } else if (possible.size() > 1) { + auto checkAllHaveSameAt = [&](size_t pos) { + auto & first = *possible.begin(); + for (auto & p : possible) { + if (p.size() <= pos || p[pos] != first[pos]) + return false; + } + return true; + }; + size_t start = strlen(s); + size_t len = 0; + while (checkAllHaveSameAt(start + len)) + ++len; + if (len > 0) { + *match = 1; + auto * res = strdup(std::string(*possible.begin(), start, len).c_str()); + if (!res) + throw Error("allocation failure"); + return res; + } + } + + *match = 0; + return nullptr; +} + +static int listPossibleCallback(char * s, char *** avp) +{ + auto possible = curRepl->completePrefix(s); + + if (possible.size() > (std::numeric_limits::max() / sizeof(char *))) + throw Error("too many completions"); + + int ac = 0; + char ** vp = nullptr; + + auto check = [&](auto * p) { + if (!p) { + if (vp) { + while (--ac >= 0) + free(vp[ac]); + free(vp); + } + throw Error("allocation failure"); + } + return p; + }; + + vp = check((char **) malloc(possible.size() * sizeof(char *))); + + for (auto & p : possible) + vp[ac++] = check(strdup(p.c_str())); + + *avp = vp; + + return ac; +} +#endif + +ReadlineLikeInteracter::Guard ReadlineLikeInteracter::init(detail::ReplCompleterMixin * repl) +{ + // Allow nix-repl specific settings in .inputrc + rl_readline_name = "nix-repl"; + try { + createDirs(dirOf(historyFile)); + } catch (SystemError & e) { + logWarning(e.info()); + } +#ifndef USE_READLINE + el_hist_size = 1000; +#endif + read_history(historyFile.c_str()); + auto oldRepl = curRepl; + curRepl = repl; + Guard restoreRepl([oldRepl] { curRepl = oldRepl; }); +#ifndef USE_READLINE + rl_set_complete_func(completionCallback); + rl_set_list_possib_func(listPossibleCallback); +#endif + return restoreRepl; +} + +static constexpr const char * promptForType(ReplPromptType promptType) +{ + switch (promptType) { + case ReplPromptType::ReplPrompt: + return "nix-repl> "; + case ReplPromptType::ContinuationPrompt: + return " "; + } + assert(false); +} + +bool ReadlineLikeInteracter::getLine(std::string & input, ReplPromptType promptType) +{ +#ifndef _WIN32 // TODO use more signals.hh for this + struct sigaction act, old; + sigset_t savedSignalMask, set; + + auto setupSignals = [&]() { + act.sa_handler = sigintHandler; + sigfillset(&act.sa_mask); + act.sa_flags = 0; + if (sigaction(SIGINT, &act, &old)) + throw SysError("installing handler for SIGINT"); + + sigemptyset(&set); + sigaddset(&set, SIGINT); + if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask)) + throw SysError("unblocking SIGINT"); + }; + auto restoreSignals = [&]() { + if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) + throw SysError("restoring signals"); + + if (sigaction(SIGINT, &old, 0)) + throw SysError("restoring handler for SIGINT"); + }; + + setupSignals(); +#endif + char * s = readline(promptForType(promptType)); + Finally doFree([&]() { free(s); }); +#ifndef _WIN32 // TODO use more signals.hh for this + restoreSignals(); +#endif + + if (g_signal_received) { + g_signal_received = 0; + input.clear(); + return true; + } + + // editline doesn't echo the input to the output when non-interactive, unlike readline + // this results in a different behavior when running tests. The echoing is + // quite useful for reading the test output, so we add it here. + if (auto e = getEnv("_NIX_TEST_REPL_ECHO"); s && e && *e == "1") + { +#ifndef USE_READLINE + // This is probably not right for multi-line input, but we don't use that + // in the characterisation tests, so it's fine. + std::cout << promptForType(promptType) << s << std::endl; +#endif + } + + if (!s) + return false; + input += s; + input += '\n'; + + return true; +} + +ReadlineLikeInteracter::~ReadlineLikeInteracter() +{ + write_history(historyFile.c_str()); +} + +}; diff --git a/src/libcmd/repl-interacter.hh b/src/libcmd/repl-interacter.hh new file mode 100644 index 000000000..cc70efd07 --- /dev/null +++ b/src/libcmd/repl-interacter.hh @@ -0,0 +1,48 @@ +#pragma once +/// @file + +#include "finally.hh" +#include "types.hh" +#include +#include + +namespace nix { + +namespace detail { +/** Provides the completion hooks for the repl, without exposing its complete + * internals. */ +struct ReplCompleterMixin { + virtual StringSet completePrefix(const std::string & prefix) = 0; +}; +}; + +enum class ReplPromptType { + ReplPrompt, + ContinuationPrompt, +}; + +class ReplInteracter +{ +public: + using Guard = Finally>; + + virtual Guard init(detail::ReplCompleterMixin * repl) = 0; + /** Returns a boolean of whether the interacter got EOF */ + virtual bool getLine(std::string & input, ReplPromptType promptType) = 0; + virtual ~ReplInteracter(){}; +}; + +class ReadlineLikeInteracter : public virtual ReplInteracter +{ + std::string historyFile; +public: + ReadlineLikeInteracter(std::string historyFile) + : historyFile(historyFile) + { + } + virtual Guard init(detail::ReplCompleterMixin * repl) override; + virtual bool getLine(std::string & input, ReplPromptType promptType) override; + virtual ~ReadlineLikeInteracter() override; +}; + +}; diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 8b83608fa..efc04b029 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -1,34 +1,17 @@ #include #include #include -#include - -#include - -#ifdef USE_READLINE -#include -#include -#else -// editline < 1.15.2 don't wrap their API for C++ usage -// (added in https://github.com/troglobit/editline/commit/91398ceb3427b730995357e9d120539fb9bb7461). -// This results in linker errors due to to name-mangling of editline C symbols. -// For compatibility with these versions, we wrap the API here -// (wrapping multiple times on newer versions is no problem). -extern "C" { -#include -} -#endif +#include "repl-interacter.hh" #include "repl.hh" #include "ansicolor.hh" -#include "signals.hh" #include "shared.hh" +#include "config-global.hh" #include "eval.hh" -#include "eval-cache.hh" -#include "eval-inline.hh" #include "eval-settings.hh" #include "attr-path.hh" +#include "signals.hh" #include "store-api.hh" #include "log-store.hh" #include "common-eval-args.hh" @@ -38,18 +21,21 @@ extern "C" { #include "flake/flake.hh" #include "flake/lockfile.hh" #include "users.hh" -#include "terminal.hh" #include "editor-for.hh" #include "finally.hh" #include "markdown.hh" #include "local-fs-store.hh" #include "print.hh" +#include "ref.hh" +#include "value.hh" #if HAVE_BOEHMGC #define GC_INCLUDE_NEW #include #endif +#include "strings.hh" + namespace nix { /** @@ -75,6 +61,7 @@ enum class ProcessLineResult { struct NixRepl : AbstractNixRepl + , detail::ReplCompleterMixin #if HAVE_BOEHMGC , gc #endif @@ -90,17 +77,16 @@ struct NixRepl int displ; StringSet varNames; - const Path historyFile; + std::unique_ptr interacter; - NixRepl(const SearchPath & searchPath, nix::ref store,ref state, + NixRepl(const LookupPath & lookupPath, nix::ref store,ref state, std::function getValues); - virtual ~NixRepl(); + virtual ~NixRepl() = default; ReplExitStatus mainLoop() override; void initEnv() override; - StringSet completePrefix(const std::string & prefix); - bool getLine(std::string & input, const std::string & prompt); + virtual StringSet completePrefix(const std::string & prefix) override; StorePath getDerivationPath(Value & v); ProcessLineResult processLine(std::string line); @@ -123,7 +109,8 @@ struct NixRepl .force = true, .derivationPaths = true, .maxDepth = maxDepth, - .prettyIndent = 2 + .prettyIndent = 2, + .errors = ErrorPrintBehavior::ThrowTopLevel, }); } }; @@ -137,111 +124,33 @@ std::string removeWhitespace(std::string s) } -NixRepl::NixRepl(const SearchPath & searchPath, nix::ref store, ref state, +NixRepl::NixRepl(const LookupPath & lookupPath, nix::ref store, ref state, std::function getValues) : AbstractNixRepl(state) , debugTraceIndex(0) , getValues(getValues) , staticEnv(new StaticEnv(nullptr, state->staticBaseEnv.get())) - , historyFile(getDataDir() + "/nix/repl-history") + , interacter(make_unique(getDataDir() + "/nix/repl-history")) { } - -NixRepl::~NixRepl() -{ - write_history(historyFile.c_str()); -} - void runNix(Path program, const Strings & args, const std::optional & input = {}) { auto subprocessEnv = getEnv(); subprocessEnv["NIX_CONFIG"] = globalConfig.toKeyValue(); - + //isInteractive avoid grabling interactive commands runProgram2(RunOptions { .program = settings.nixBinDir+ "/" + program, .args = args, .environment = subprocessEnv, .input = input, + .isInteractive = true, }); return; } -static NixRepl * curRepl; // ugly - -static char * completionCallback(char * s, int *match) { - auto possible = curRepl->completePrefix(s); - if (possible.size() == 1) { - *match = 1; - auto *res = strdup(possible.begin()->c_str() + strlen(s)); - if (!res) throw Error("allocation failure"); - return res; - } else if (possible.size() > 1) { - auto checkAllHaveSameAt = [&](size_t pos) { - auto &first = *possible.begin(); - for (auto &p : possible) { - if (p.size() <= pos || p[pos] != first[pos]) - return false; - } - return true; - }; - size_t start = strlen(s); - size_t len = 0; - while (checkAllHaveSameAt(start + len)) ++len; - if (len > 0) { - *match = 1; - auto *res = strdup(std::string(*possible.begin(), start, len).c_str()); - if (!res) throw Error("allocation failure"); - return res; - } - } - - *match = 0; - return nullptr; -} - -static int listPossibleCallback(char *s, char ***avp) { - auto possible = curRepl->completePrefix(s); - - if (possible.size() > (INT_MAX / sizeof(char*))) - throw Error("too many completions"); - - int ac = 0; - char **vp = nullptr; - - auto check = [&](auto *p) { - if (!p) { - if (vp) { - while (--ac >= 0) - free(vp[ac]); - free(vp); - } - throw Error("allocation failure"); - } - return p; - }; - - vp = check((char **)malloc(possible.size() * sizeof(char*))); - - for (auto & p : possible) - vp[ac++] = check(strdup(p.c_str())); - - *avp = vp; - - return ac; -} - -namespace { - // Used to communicate to NixRepl::getLine whether a signal occurred in ::readline. - volatile sig_atomic_t g_signal_received = 0; - - void sigintHandler(int signo) { - g_signal_received = signo; - } -} - static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positions, const DebugTrace & dt) { if (dt.isError) @@ -281,24 +190,7 @@ ReplExitStatus NixRepl::mainLoop() loadFiles(); - // Allow nix-repl specific settings in .inputrc - rl_readline_name = "nix-repl"; - try { - createDirs(dirOf(historyFile)); - } catch (SystemError & e) { - logWarning(e.info()); - } -#ifndef USE_READLINE - el_hist_size = 1000; -#endif - read_history(historyFile.c_str()); - auto oldRepl = curRepl; - curRepl = this; - Finally restoreRepl([&] { curRepl = oldRepl; }); -#ifndef USE_READLINE - rl_set_complete_func(completionCallback); - rl_set_list_possib_func(listPossibleCallback); -#endif + auto _guard = interacter->init(static_cast(this)); std::string input; @@ -307,7 +199,7 @@ ReplExitStatus NixRepl::mainLoop() logger->pause(); // When continuing input from previous lines, don't print a prompt, just align to the same // number of chars as the prompt. - if (!getLine(input, input.empty() ? "nix-repl> " : " ")) { + if (!interacter->getLine(input, input.empty() ? ReplPromptType::ReplPrompt : ReplPromptType::ContinuationPrompt)) { // Ctrl-D should exit the debugger. state->debugStop = false; logger->cout(""); @@ -325,7 +217,7 @@ ReplExitStatus NixRepl::mainLoop() case ProcessLineResult::PromptAgain: break; default: - abort(); + unreachable(); } } catch (ParseError & e) { if (e.msg().find("unexpected end of file") != std::string::npos) { @@ -336,13 +228,7 @@ ReplExitStatus NixRepl::mainLoop() printMsg(lvlError, e.msg()); } } catch (EvalError & e) { - // in debugger mode, an EvalError should trigger another repl session. - // when that session returns the exception will land here. No need to show it again; - // show the error for this repl session instead. - if (state->debugRepl && !state->debugTraces.empty()) - showDebugTrace(std::cout, state->positions, state->debugTraces.front()); - else - printMsg(lvlError, e.msg()); + printMsg(lvlError, e.msg()); } catch (Error & e) { printMsg(lvlError, e.msg()); } catch (Interrupted & e) { @@ -356,51 +242,6 @@ ReplExitStatus NixRepl::mainLoop() } } - -bool NixRepl::getLine(std::string & input, const std::string & prompt) -{ - struct sigaction act, old; - sigset_t savedSignalMask, set; - - auto setupSignals = [&]() { - act.sa_handler = sigintHandler; - sigfillset(&act.sa_mask); - act.sa_flags = 0; - if (sigaction(SIGINT, &act, &old)) - throw SysError("installing handler for SIGINT"); - - sigemptyset(&set); - sigaddset(&set, SIGINT); - if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask)) - throw SysError("unblocking SIGINT"); - }; - auto restoreSignals = [&]() { - if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) - throw SysError("restoring signals"); - - if (sigaction(SIGINT, &old, 0)) - throw SysError("restoring handler for SIGINT"); - }; - - setupSignals(); - char * s = readline(prompt.c_str()); - Finally doFree([&]() { free(s); }); - restoreSignals(); - - if (g_signal_received) { - g_signal_received = 0; - input.clear(); - return true; - } - - if (!s) - return false; - input += s; - input += '\n'; - return true; -} - - StringSet NixRepl::completePrefix(const std::string & prefix) { StringSet completions; @@ -421,11 +262,14 @@ StringSet NixRepl::completePrefix(const std::string & prefix) try { auto dir = std::string(cur, 0, slash); auto prefix2 = std::string(cur, slash + 1); - for (auto & entry : readDirectory(dir == "" ? "/" : dir)) { - if (entry.name[0] != '.' && hasPrefix(entry.name, prefix2)) - completions.insert(prev + dir + "/" + entry.name); + for (auto & entry : std::filesystem::directory_iterator{dir == "" ? "/" : dir}) { + checkInterrupt(); + auto name = entry.path().filename().string(); + if (name[0] != '.' && hasPrefix(name, prefix2)) + completions.insert(prev + entry.path().string()); } } catch (Error &) { + } catch (std::filesystem::filesystem_error &) { } } else if ((dot = cur.rfind('.')) == std::string::npos) { /* This is a variable name; look it up in the current scope. */ @@ -452,7 +296,7 @@ StringSet NixRepl::completePrefix(const std::string & prefix) e->eval(*state, *env, v); state->forceAttrs(v, noPos, "while evaluating an attrset for the purpose of completion (this error should not be displayed; file an issue?)"); - for (auto & i : *v.attrs) { + for (auto & i : *v.attrs()) { std::string_view name = state->symbols[i.name]; if (name.substr(0, cur2.size()) != cur2) continue; completions.insert(concatStrings(prev, expr, ".", name)); @@ -464,6 +308,8 @@ StringSet NixRepl::completePrefix(const std::string & prefix) // Quietly ignore evaluation errors. } catch (BadURL & e) { // Quietly ignore BadURL flake-related errors. + } catch (FileNotFound & e) { + // Quietly ignore non-existent file beeing `import`-ed. } } @@ -519,7 +365,7 @@ ProcessLineResult NixRepl::processLine(std::string line) if (line.empty()) return ProcessLineResult::PromptAgain; - _isInterrupted = false; + setInterrupted(false); std::string command, arg; @@ -548,6 +394,7 @@ ProcessLineResult NixRepl::processLine(std::string line) << " :l, :load Load Nix expression and add it to scope\n" << " :lf, :load-flake Load Nix flake and add it to scope\n" << " :p, :print Evaluate and print expression recursively\n" + << " Strings are printed directly, without escaping.\n" << " :q, :quit Exit nix-repl\n" << " :r, :reload Reload all files\n" << " :sh Build dependencies of derivation, then start\n" @@ -651,7 +498,7 @@ ProcessLineResult NixRepl::processLine(std::string line) auto path = state->coerceToPath(noPos, v, context, "while evaluating the filename to edit"); return {path, 0}; } else if (v.isLambda()) { - auto pos = state->positions[v.lambda.fun->pos]; + auto pos = state->positions[v.payload.lambda.fun->pos]; if (auto path = std::get_if(&pos.origin)) return {*path, pos.line}; else @@ -669,7 +516,7 @@ ProcessLineResult NixRepl::processLine(std::string line) // runProgram redirects stdout to a StringSink, // using runProgram2 to allow editors to display their UI - runProgram2(RunOptions { .program = editor, .searchPath = true, .args = args }); + runProgram2(RunOptions { .program = editor, .lookupPath = true, .args = args , .isInteractive = true }); // Reload right after exiting the editor state->resetFileCache(); @@ -755,7 +602,11 @@ ProcessLineResult NixRepl::processLine(std::string line) else if (command == ":p" || command == ":print") { Value v; evalString(arg, v); - printValue(std::cout, v); + if (v.type() == nString) { + std::cout << v.string_view(); + } else { + printValue(std::cout, v); + } std::cout << std::endl; } @@ -766,6 +617,35 @@ ProcessLineResult NixRepl::processLine(std::string line) else if (command == ":doc") { Value v; + + auto expr = parseString(arg); + std::string fallbackName; + PosIdx fallbackPos; + DocComment fallbackDoc; + if (auto select = dynamic_cast(expr)) { + Value vAttrs; + auto name = select->evalExceptFinalSelect(*state, *env, vAttrs); + fallbackName = state->symbols[name]; + + state->forceAttrs(vAttrs, noPos, "while evaluating an attribute set to look for documentation"); + auto attrs = vAttrs.attrs(); + assert(attrs); + auto attr = attrs->get(name); + if (!attr) { + // When missing, trigger the normal exception + // e.g. :doc builtins.foo + // behaves like + // nix-repl> builtins.foo + // error: attribute 'foo' missing + evalString(arg, v); + assert(false); + } + if (attr->pos) { + fallbackPos = attr->pos; + fallbackDoc = state->getDocCommentForPos(fallbackPos); + } + } + evalString(arg, v); if (auto doc = state->getDoc(v)) { std::string markdown; @@ -783,6 +663,19 @@ ProcessLineResult NixRepl::processLine(std::string line) markdown += stripIndentation(doc->doc); logger->cout(trim(renderMarkdownToTerminal(markdown))); + } else if (fallbackPos) { + std::stringstream ss; + ss << "Attribute `" << fallbackName << "`\n\n"; + ss << " … defined at " << state->positions[fallbackPos] << "\n\n"; + if (fallbackDoc) { + ss << fallbackDoc.getInnerText(state->positions); + } else { + ss << "No documentation found.\n\n"; + } + + auto markdown = ss.str(); + logger->cout(trim(renderMarkdownToTerminal(markdown))); + } else throw Error("value does not have documentation"); } @@ -840,14 +733,14 @@ void NixRepl::loadFlake(const std::string & flakeRefS) if (flakeRefS.empty()) throw Error("cannot use ':load-flake' without a path specified. (Use '.' for the current working directory.)"); - auto flakeRef = parseFlakeRef(flakeRefS, absPath("."), true); + auto flakeRef = parseFlakeRef(fetchSettings, flakeRefS, absPath("."), true); if (evalSettings.pureEval && !flakeRef.input.isLocked()) throw Error("cannot use ':load-flake' on locked flake reference '%s' (use --impure to override)", flakeRefS); Value v; flake::callFlake(*state, - flake::lockFlake(*state, flakeRef, + flake::lockFlake(flakeSettings, *state, flakeRef, flake::LockFlags { .updateLockFile = false, .useRegistries = !evalSettings.pureEval, @@ -899,17 +792,17 @@ void NixRepl::loadFiles() void NixRepl::addAttrsToScope(Value & attrs) { state->forceAttrs(attrs, [&]() { return attrs.determinePos(noPos); }, "while evaluating an attribute set to be merged in the global scope"); - if (displ + attrs.attrs->size() >= envSize) + if (displ + attrs.attrs()->size() >= envSize) throw Error("environment full; cannot add more variables"); - for (auto & i : *attrs.attrs) { + for (auto & i : *attrs.attrs()) { staticEnv->vars.emplace_back(i.name, displ); env->values[displ++] = i.value; varNames.emplace(state->symbols[i.name]); } staticEnv->sort(); staticEnv->deduplicate(); - notice("Added %1% variables.", attrs.attrs->size()); + notice("Added %1% variables.", attrs.attrs()->size()); } @@ -941,11 +834,11 @@ void NixRepl::evalString(std::string s, Value & v) std::unique_ptr AbstractNixRepl::create( - const SearchPath & searchPath, nix::ref store, ref state, + const LookupPath & lookupPath, nix::ref store, ref state, std::function getValues) { return std::make_unique( - searchPath, + lookupPath, openStore(), state, getValues @@ -961,9 +854,9 @@ ReplExitStatus AbstractNixRepl::runSimple( NixRepl::AnnotatedValues values; return values; }; - SearchPath searchPath = {}; + LookupPath lookupPath = {}; auto repl = std::make_unique( - searchPath, + lookupPath, openStore(), evalState, getValues diff --git a/src/libcmd/repl.hh b/src/libcmd/repl.hh index 21aa8bfc7..3fd4b2c39 100644 --- a/src/libcmd/repl.hh +++ b/src/libcmd/repl.hh @@ -3,11 +3,6 @@ #include "eval.hh" -#if HAVE_BOEHMGC -#define GC_INCLUDE_NEW -#include -#endif - namespace nix { struct AbstractNixRepl @@ -25,7 +20,7 @@ struct AbstractNixRepl typedef std::vector> AnnotatedValues; static std::unique_ptr create( - const SearchPath & searchPath, nix::ref store, ref state, + const LookupPath & lookupPath, nix::ref store, ref state, std::function getValues); static ReplExitStatus runSimple( diff --git a/src/libexpr-c/.version b/src/libexpr-c/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libexpr-c/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libexpr-c/build-utils-meson b/src/libexpr-c/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libexpr-c/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libexpr-c/local.mk b/src/libexpr-c/local.mk new file mode 100644 index 000000000..227a4095b --- /dev/null +++ b/src/libexpr-c/local.mk @@ -0,0 +1,25 @@ +libraries += libexprc + +libexprc_NAME = libnixexprc + +libexprc_DIR := $(d) + +libexprc_SOURCES := \ + $(wildcard $(d)/*.cc) \ + +# Not just for this library itself, but also for downstream libraries using this library + +INCLUDE_libexprc := -I $(d) +libexprc_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libutilc) \ + $(INCLUDE_libfetchers) \ + $(INCLUDE_libstore) $(INCLUDE_libstorec) \ + $(INCLUDE_libexpr) $(INCLUDE_libexprc) + +libexprc_LIBS = libutil libutilc libstore libstorec libfetchers libexpr + +libexprc_LDFLAGS += $(THREAD_LDFLAGS) + +$(eval $(call install-file-in, $(d)/nix-expr-c.pc, $(libdir)/pkgconfig, 0644)) + +libexprc_FORCE_INSTALL := 1 + diff --git a/src/libexpr-c/meson.build b/src/libexpr-c/meson.build new file mode 100644 index 000000000..6db5b83b8 --- /dev/null +++ b/src/libexpr-c/meson.build @@ -0,0 +1,93 @@ +project('nix-expr-c', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), + dependency('nix-expr'), +] +deps_public_maybe_subproject = [ + dependency('nix-util-c'), + dependency('nix-store-c'), +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +# TODO rename, because it will conflict with downstream projects +configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) + +config_h = configure_file( + configuration : configdata, + output : 'config-expr.h', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + + # From C++ libraries, only for internals + '-include', 'config-util.hh', + '-include', 'config-store.hh', + '-include', 'config-expr.hh', + + # From C libraries, for our public, installed headers too + '-include', 'config-util.h', + '-include', 'config-store.h', + '-include', 'config-expr.h', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'nix_api_expr.cc', + 'nix_api_external.cc', + 'nix_api_value.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'nix_api_expr.h', + 'nix_api_external.h', + 'nix_api_value.h', +) + +# TODO move this header to libexpr, maybe don't use it in tests? +headers += files('nix_api_expr_internal.h') + +subdir('build-utils-meson/export-all-symbols') + +this_library = library( + 'nixexprc', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + link_args: linker_export_flags, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libexpr-c/nix-expr-c.pc.in b/src/libexpr-c/nix-expr-c.pc.in new file mode 100644 index 000000000..06897064d --- /dev/null +++ b/src/libexpr-c/nix-expr-c.pc.in @@ -0,0 +1,10 @@ +prefix=@prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: Nix +Description: Nix Language Evaluator - C API +Version: @PACKAGE_VERSION@ +Requires: nix-store-c +Libs: -L${libdir} -lnixexprc +Cflags: -I${includedir}/nix diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc new file mode 100644 index 000000000..8f21d7022 --- /dev/null +++ b/src/libexpr-c/nix_api_expr.cc @@ -0,0 +1,213 @@ +#include +#include +#include + +#include "eval.hh" +#include "eval-gc.hh" +#include "globals.hh" +#include "eval-settings.hh" + +#include "nix_api_expr.h" +#include "nix_api_expr_internal.h" +#include "nix_api_store.h" +#include "nix_api_store_internal.h" +#include "nix_api_util.h" +#include "nix_api_util_internal.h" + +#if HAVE_BOEHMGC +# include +# define GC_INCLUDE_NEW 1 +# include "gc_cpp.h" +#endif + +nix_err nix_libexpr_init(nix_c_context * context) +{ + if (context) + context->last_err_code = NIX_OK; + { + auto ret = nix_libutil_init(context); + if (ret != NIX_OK) + return ret; + } + { + auto ret = nix_libstore_init(context); + if (ret != NIX_OK) + return ret; + } + try { + nix::initGC(); + } + NIXC_CATCH_ERRS +} + +nix_err nix_expr_eval_from_string( + nix_c_context * context, EvalState * state, const char * expr, const char * path, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + nix::Expr * parsedExpr = state->state.parseExprFromString(expr, state->state.rootPath(nix::CanonPath(path))); + state->state.eval(parsedExpr, value->value); + state->state.forceValue(value->value, nix::noPos); + } + NIXC_CATCH_ERRS +} + +nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, nix_value * arg, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + state->state.callFunction(fn->value, arg->value, value->value, nix::noPos); + state->state.forceValue(value->value, nix::noPos); + } + NIXC_CATCH_ERRS +} + +nix_err nix_value_call_multi(nix_c_context * context, EvalState * state, nix_value * fn, size_t nargs, nix_value ** args, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + state->state.callFunction(fn->value, nargs, (nix::Value * *)args, value->value, nix::noPos); + state->state.forceValue(value->value, nix::noPos); + } + NIXC_CATCH_ERRS +} + +nix_err nix_value_force(nix_c_context * context, EvalState * state, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + state->state.forceValue(value->value, nix::noPos); + } + NIXC_CATCH_ERRS +} + +nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + state->state.forceValueDeep(value->value); + } + NIXC_CATCH_ERRS +} + +EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c, Store * store) +{ + if (context) + context->last_err_code = NIX_OK; + try { + nix::Strings lookupPath; + if (lookupPath_c != nullptr) + for (size_t i = 0; lookupPath_c[i] != nullptr; i++) + lookupPath.push_back(lookupPath_c[i]); + + void * p = ::operator new( + sizeof(EvalState), + static_cast(alignof(EvalState))); + auto * p2 = static_cast(p); + new (p) EvalState { + .fetchSettings = nix::fetchers::Settings{}, + .settings = nix::EvalSettings{ + nix::settings.readOnlyMode, + }, + .state = nix::EvalState( + nix::LookupPath::parse(lookupPath), + store->ptr, + p2->fetchSettings, + p2->settings), + }; + loadConfFile(p2->settings); + return p2; + } + NIXC_CATCH_ERRS_NULL +} + +void nix_state_free(EvalState * state) +{ + delete state; +} + +#if HAVE_BOEHMGC +std::unordered_map< + const void *, + unsigned int, + std::hash, + std::equal_to, + traceable_allocator
; }' + > + > + > + > + > + > + > + > + > ``` - - `--json`\ - When used with `--eval`, print the resulting value as an JSON - representation of the abstract syntax tree rather than as a Nix expression. +- `--find-file` - - `--xml`\ - When used with `--eval`, print the resulting value as an XML - representation of the abstract syntax tree rather than as a Nix expression. - The schema is the same as that used by the [`toXML` - built-in](../language/builtins.md). + Look up the given files in Nix’s search path (as specified by the + `NIX_PATH` environment variable). If found, print the corresponding + absolute paths on standard output. For instance, if `NIX_PATH` is + `nixpkgs=/home/alice/nixpkgs`, then `nix-instantiate --find-file + nixpkgs/default.nix` will print `/home/alice/nixpkgs/default.nix`. - - `--read-write-mode`\ - When used with `--eval`, perform evaluation in read/write mode so - nix language features that require it will still work (at the cost - of needing to do instantiation of every evaluated derivation). If - this option is not enabled, there may be uninstantiated store paths - in the final output. +- `--strict` + + When used with `--eval`, recursively evaluate list elements and + attributes. Normally, such sub-expressions are left unevaluated + (since the Nix language is lazy). + + > **Warning** + > + > This option can cause non-termination, because lazy data + > structures can be infinitely large. + +- `--json` + + When used with `--eval`, print the resulting value as an JSON + representation of the abstract syntax tree rather than as a Nix expression. + +- `--xml` + + When used with `--eval`, print the resulting value as an XML + representation of the abstract syntax tree rather than as a Nix expression. + The schema is the same as that used by the [`toXML` + built-in](../language/builtins.md). + +- `--read-write-mode` + + When used with `--eval`, perform evaluation in read/write mode so + nix language features that require it will still work (at the cost + of needing to do instantiation of every evaluated derivation). If + this option is not enabled, there may be uninstantiated store paths + in the final output. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-prefetch-url.md b/doc/manual/src/command-ref/nix-prefetch-url.md index 45ef01e02..ffab94b8a 100644 --- a/doc/manual/src/command-ref/nix-prefetch-url.md +++ b/doc/manual/src/command-ref/nix-prefetch-url.md @@ -39,27 +39,32 @@ the path of the downloaded file in the Nix store is also printed. # Options - - `--type` *hashAlgo*\ - Use the specified cryptographic hash algorithm, - which can be one of `md5`, `sha1`, `sha256`, and `sha512`. - The default is `sha256`. +- `--type` *hashAlgo* - - `--print-path`\ - Print the store path of the downloaded file on standard output. + Use the specified cryptographic hash algorithm, + which can be one of `md5`, `sha1`, `sha256`, and `sha512`. + The default is `sha256`. - - `--unpack`\ - Unpack the archive (which must be a tarball or zip file) and add the - result to the Nix store. The resulting hash can be used with - functions such as Nixpkgs’s `fetchzip` or `fetchFromGitHub`. +- `--print-path` - - `--executable`\ - Set the executable bit on the downloaded file. + Print the store path of the downloaded file on standard output. - - `--name` *name*\ - Override the name of the file in the Nix store. By default, this is - `hash-basename`, where *basename* is the last component of *url*. - Overriding the name is necessary when *basename* contains characters - that are not allowed in Nix store paths. +- `--unpack` + + Unpack the archive (which must be a tarball or zip file) and add the + result to the Nix store. The resulting hash can be used with + functions such as Nixpkgs’s `fetchzip` or `fetchFromGitHub`. + +- `--executable` + + Set the executable bit on the downloaded file. + +- `--name` *name* + + Override the name of the file in the Nix store. By default, this is + `hash-basename`, where *basename* is the last component of *url*. + Overriding the name is necessary when *basename* contains characters + that are not allowed in Nix store paths. # Examples diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index 1eaf3c36a..69a711bd5 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -60,55 +60,63 @@ All options not listed here are passed to `nix-store --realise`, except for `--arg` and `--attr` / `-A` which are passed to `nix-instantiate`. - - `--command` *cmd*\ - In the environment of the derivation, run the shell command *cmd*. - This command is executed in an interactive shell. (Use `--run` to - use a non-interactive shell instead.) However, a call to `exit` is - implicitly added to the command, so the shell will exit after - running the command. To prevent this, add `return` at the end; - e.g. `--command "echo Hello; return"` will print `Hello` and then - drop you into the interactive shell. This can be useful for doing - any additional initialisation. +- `--command` *cmd* - - `--run` *cmd*\ - Like `--command`, but executes the command in a non-interactive - shell. This means (among other things) that if you hit Ctrl-C while - the command is running, the shell exits. + In the environment of the derivation, run the shell command *cmd*. + This command is executed in an interactive shell. (Use `--run` to + use a non-interactive shell instead.) However, a call to `exit` is + implicitly added to the command, so the shell will exit after + running the command. To prevent this, add `return` at the end; + e.g. `--command "echo Hello; return"` will print `Hello` and then + drop you into the interactive shell. This can be useful for doing + any additional initialisation. - - `--exclude` *regexp*\ - Do not build any dependencies whose store path matches the regular - expression *regexp*. This option may be specified multiple times. +- `--run` *cmd* - - `--pure`\ - If this flag is specified, the environment is almost entirely - cleared before the interactive shell is started, so you get an - environment that more closely corresponds to the “real†Nix build. A - few variables, in particular `HOME`, `USER` and `DISPLAY`, are - retained. + Like `--command`, but executes the command in a non-interactive + shell. This means (among other things) that if you hit Ctrl-C while + the command is running, the shell exits. - - `--packages` / `-p` *packages*…\ - Set up an environment in which the specified packages are present. - The command line arguments are interpreted as attribute names inside - the Nix Packages collection. Thus, `nix-shell --packages libjpeg openjdk` - will start a shell in which the packages denoted by the attribute - names `libjpeg` and `openjdk` are present. +- `--exclude` *regexp* - - `-i` *interpreter*\ - The chained script interpreter to be invoked by `nix-shell`. Only - applicable in `#!`-scripts (described below). + Do not build any dependencies whose store path matches the regular + expression *regexp*. This option may be specified multiple times. - - `--keep` *name*\ - When a `--pure` shell is started, keep the listed environment - variables. +- `--pure` + + If this flag is specified, the environment is almost entirely + cleared before the interactive shell is started, so you get an + environment that more closely corresponds to the “real†Nix build. A + few variables, in particular `HOME`, `USER` and `DISPLAY`, are + retained. + +- `--packages` / `-p` *packages*… + + Set up an environment in which the specified packages are present. + The command line arguments are interpreted as attribute names inside + the Nix Packages collection. Thus, `nix-shell --packages libjpeg openjdk` + will start a shell in which the packages denoted by the attribute + names `libjpeg` and `openjdk` are present. + +- `-i` *interpreter* + + The chained script interpreter to be invoked by `nix-shell`. Only + applicable in `#!`-scripts (described below). + +- `--keep` *name* + + When a `--pure` shell is started, keep the listed environment + variables. {{#include ./opt-common.md}} # Environment variables - - `NIX_BUILD_SHELL`\ - Shell used to start the interactive environment. Defaults to the - `bash` found in ``, falling back to the `bash` found in - `PATH` if not found. +- `NIX_BUILD_SHELL` + + Shell used to start the interactive environment. Defaults to the + `bash` found in ``, falling back to the `bash` found in + `PATH` if not found. {{#include ./env-common.md}} @@ -202,14 +210,14 @@ For example, here is a Python script that depends on Python and the ```python #! /usr/bin/env nix-shell -#! nix-shell -i python --packages python pythonPackages.prettytable +#! nix-shell -i python3 --packages python3 python3Packages.prettytable import prettytable # Print a simple table. t = prettytable.PrettyTable(["N", "N^2"]) for n in range(1, 10): t.add_row([n, n * n]) -print t +print(t) ``` Similarly, the following is a Perl script that specifies that it @@ -289,3 +297,8 @@ with import {}; runCommand "dummy" { buildInputs = [ python pythonPackages.prettytable ]; } "" ``` + +The script's file name is passed as the first argument to the interpreter specified by the `-i` flag. + +Aside from the very first line, which is a directive to the operating system, the additional `#! nix-shell` lines do not need to be at the beginning of the file. +This allows wrapping them in block comments for languages where `#` does not start a comment, such as ECMAScript, Erlang, PHP, or Ruby. diff --git a/doc/manual/src/command-ref/nix-store/add-fixed.md b/doc/manual/src/command-ref/nix-store/add-fixed.md index d25db091c..bebf15026 100644 --- a/doc/manual/src/command-ref/nix-store/add-fixed.md +++ b/doc/manual/src/command-ref/nix-store/add-fixed.md @@ -16,9 +16,10 @@ public url or broke since the download expression was written. This operation has the following options: - - `--recursive`\ - Use recursive instead of flat hashing mode, used when adding - directories to the store. +- `--recursive` + + Use recursive instead of flat hashing mode, used when adding + directories to the store. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/dump.md b/doc/manual/src/command-ref/nix-store/dump.md index c2f3c42ef..3de0e27b0 100644 --- a/doc/manual/src/command-ref/nix-store/dump.md +++ b/doc/manual/src/command-ref/nix-store/dump.md @@ -1,6 +1,6 @@ # Name -`nix-store --dump` - write a single path to a Nix Archive +`nix-store --dump` - write a single path to a [Nix Archive] ## Synopsis @@ -8,7 +8,7 @@ ## Description -The operation `--dump` produces a NAR (Nix ARchive) file containing the +The operation `--dump` produces a [Nix archive](@docroot@/glossary.md#gloss-nar) (NAR) file containing the contents of the file system tree rooted at *path*. The archive is written to standard output. @@ -30,8 +30,9 @@ NAR archives support filenames of unlimited length and 64-bit file sizes. They can contain regular files, directories, and symbolic links, but not other types of files (such as device nodes). -A Nix archive can be unpacked using `nix-store ---restore`. +A Nix archive can be unpacked using [`nix-store --restore`](@docroot@/command-ref/nix-store/restore.md). + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/export.md b/doc/manual/src/command-ref/nix-store/export.md index 1bc46f53b..ba772eb43 100644 --- a/doc/manual/src/command-ref/nix-store/export.md +++ b/doc/manual/src/command-ref/nix-store/export.md @@ -1,6 +1,6 @@ # Name -`nix-store --export` - export store paths to a Nix Archive +`nix-store --export` - export store paths to a [Nix Archive] ## Synopsis @@ -8,16 +8,22 @@ ## Description -The operation `--export` writes a serialisation of the specified store -paths to standard output in a format that can be imported into another -Nix store with `nix-store --import`. This is like `nix-store ---dump`, except that the NAR archive produced by that command doesn’t -contain the necessary meta-information to allow it to be imported into -another Nix store (namely, the set of references of the path). +The operation `--export` writes a serialisation of the given [store objects](@docroot@/glossary.md#gloss-store-object) to standard output in a format that can be imported into another [Nix store](@docroot@/store/index.md) with [`nix-store --import`](./import.md). -This command does not produce a *closure* of the specified paths, so if -a store path references other store paths that are missing in the target -Nix store, the import will fail. +> **Warning** +> +> This command *does not* produce a [closure](@docroot@/glossary.md#gloss-closure) of the specified store paths. +> Trying to import a store object that refers to store paths not available in the target Nix store will fail. +> +> Use [`nix-store --query`](@docroot@/command-ref/nix-store/query.md) to obtain the closure of a store path. + +This command is different from [`nix-store --dump`](./dump.md), which produces a [Nix archive](@docroot@/glossary.md#gloss-nar) that *does not* contain the set of [references](@docroot@/glossary.md#gloss-reference) of a given store path. + +> **Note** +> +> For efficient transfer of closures to remote machines over SSH, use [`nix-copy-closure`](@docroot@/command-ref/nix-copy-closure.md). + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive {{#include ./opt-common.md}} @@ -27,15 +33,21 @@ Nix store, the import will fail. # Examples -To copy a whole closure, do something -like: - -```console -$ nix-store --export $(nix-store --query --requisites paths) > out -``` - -To import the whole closure again, run: - -```console -$ nix-store --import < out -``` +> **Example** +> +> Deploy GNU Hello to an airgapped machine via USB stick. +> +> Write the closure to the block device on a machine with internet connection: +> +> ```shell-session +> [alice@itchy]$ storePath=$(nix-build '' -I nixpkgs=channel:nixpkgs-unstable -A hello --no-out-link) +> [alice@itchy]$ nix-store --export $(nix-store --query --requisites $storePath) | sudo dd of=/dev/usb +> ``` +> +> Read the closure from the block device on the machine without internet connection: +> +> ```shell-session +> [bob@scratchy]$ hello=$(sudo dd if=/dev/usb | nix-store --import | tail -1) +> [bob@scratchy]$ $hello/bin/hello +> Hello, world! +> ``` diff --git a/doc/manual/src/command-ref/nix-store/gc.md b/doc/manual/src/command-ref/nix-store/gc.md index 7be0d559a..f432e00eb 100644 --- a/doc/manual/src/command-ref/nix-store/gc.md +++ b/doc/manual/src/command-ref/nix-store/gc.md @@ -14,30 +14,34 @@ reachable via file system references from a set of “rootsâ€, are deleted. The following suboperations may be specified: - - `--print-roots`\ - This operation prints on standard output the set of roots used by - the garbage collector. +- `--print-roots` - - `--print-live`\ - This operation prints on standard output the set of “live†store - paths, which are all the store paths reachable from the roots. Live - paths should never be deleted, since that would break consistency — - it would become possible that applications are installed that - reference things that are no longer present in the store. + This operation prints on standard output the set of roots used by + the garbage collector. - - `--print-dead`\ - This operation prints out on standard output the set of “dead†store - paths, which is just the opposite of the set of live paths: any path - in the store that is not live (with respect to the roots) is dead. +- `--print-live` + + This operation prints on standard output the set of “live†store + paths, which are all the store paths reachable from the roots. Live + paths should never be deleted, since that would break consistency — + it would become possible that applications are installed that + reference things that are no longer present in the store. + +- `--print-dead` + + This operation prints out on standard output the set of “dead†store + paths, which is just the opposite of the set of live paths: any path + in the store that is not live (with respect to the roots) is dead. By default, all unreachable paths are deleted. The following options control what gets deleted and in what order: - - `--max-freed` *bytes*\ - Keep deleting paths until at least *bytes* bytes have been deleted, - then stop. The argument *bytes* can be followed by the - multiplicative suffix `K`, `M`, `G` or `T`, denoting KiB, MiB, GiB - or TiB units. +- `--max-freed` *bytes* + + Keep deleting paths until at least *bytes* bytes have been deleted, + then stop. The argument *bytes* can be followed by the + multiplicative suffix `K`, `M`, `G` or `T`, denoting KiB, MiB, GiB + or TiB units. The behaviour of the collector is also influenced by the `keep-outputs` and `keep-derivations` settings in the Nix diff --git a/doc/manual/src/command-ref/nix-store/import.md b/doc/manual/src/command-ref/nix-store/import.md index 2711316a7..3f6b3d076 100644 --- a/doc/manual/src/command-ref/nix-store/import.md +++ b/doc/manual/src/command-ref/nix-store/import.md @@ -1,6 +1,8 @@ # Name -`nix-store --import` - import Nix Archive into the store +`nix-store --import` - import [Nix Archive] into the store + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive # Synopsis @@ -8,14 +10,34 @@ # Description -The operation `--import` reads a serialisation of a set of store paths -produced by `nix-store --export` from standard input and adds those -store paths to the Nix store. Paths that already exist in the Nix store -are ignored. If a path refers to another path that doesn’t exist in the -Nix store, the import fails. +The operation `--import` reads a serialisation of a set of [store objects](@docroot@/glossary.md#gloss-store-object) produced by [`nix-store --export`](./export.md) from standard input, and adds those store objects to the specified [Nix store](@docroot@/store/index.md). +Paths that already exist in the target Nix store are ignored. +If a path [refers](@docroot@/glossary.md#gloss-reference) to another path that doesn’t exist in the target Nix store, the import fails. + +> **Note** +> +> For efficient transfer of closures to remote machines over SSH, use [`nix-copy-closure`](@docroot@/command-ref/nix-copy-closure.md). {{#include ./opt-common.md}} {{#include ../opt-common.md}} {{#include ../env-common.md}} + +# Examples + +> **Example** +> +> Given a closure of GNU Hello as a file: +> +> ```shell-session +> $ storePath="$(nix-build '' -I nixpkgs=channel:nixpkgs-unstable -A hello --no-out-link)" +> $ nix-store --export $(nix-store --query --requisites $storePath) > hello.closure +> ``` +> +> Import the closure into a [remote SSH store](@docroot@/store/types/ssh-store.md) using the [`--store`](@docroot@/command-ref/conf-file.md#conf-store) option: +> +> ```console +> $ nix-store --import --store ssh://alice@itchy.example.org < hello.closure +> ``` + diff --git a/doc/manual/src/command-ref/nix-store/optimise.md b/doc/manual/src/command-ref/nix-store/optimise.md index dc392aeb8..b257466b2 100644 --- a/doc/manual/src/command-ref/nix-store/optimise.md +++ b/doc/manual/src/command-ref/nix-store/optimise.md @@ -12,7 +12,7 @@ The operation `--optimise` reduces Nix store disk space usage by finding identical files in the store and hard-linking them to each other. It typically reduces the size of the store by something like 25-35%. Only regular files and symlinks are hard-linked in this manner. Files are -considered identical when they have the same NAR archive serialisation: +considered identical when they have the same [Nix Archive (NAR)][Nix Archive] serialisation: that is, regular files must have the same contents and permission (executable or non-executable), and symlinks must have the same contents. @@ -38,3 +38,4 @@ hashing files in `/nix/store/qhqx7l2f1kmwihc9bnxs7rc159hsxnf3-gcc-4.1.1' there are 114486 files with equal contents out of 215894 files in total ``` +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive diff --git a/doc/manual/src/command-ref/nix-store/query.md b/doc/manual/src/command-ref/nix-store/query.md index a158c76aa..b4efa734e 100644 --- a/doc/manual/src/command-ref/nix-store/query.md +++ b/doc/manual/src/command-ref/nix-store/query.md @@ -24,122 +24,138 @@ symlink. # Common query options - - `--use-output`; `-u`\ - For each argument to the query that is a [store derivation], apply the - query to the output path of the derivation instead. +- `--use-output` / `-u` - - `--force-realise`; `-f`\ - Realise each argument to the query first (see [`nix-store --realise`](./realise.md)). + For each argument to the query that is a [store derivation], apply the + query to the output path of the derivation instead. + +- `--force-realise` / `-f` + + Realise each argument to the query first (see [`nix-store --realise`](./realise.md)). [store derivation]: @docroot@/glossary.md#gloss-store-derivation # Queries - - `--outputs`\ - Prints out the [output paths] of the store - derivations *paths*. These are the paths that will be produced when - the derivation is built. +- `--outputs` - [output paths]: ../../glossary.md#gloss-output-path + Prints out the [output paths] of the store + derivations *paths*. These are the paths that will be produced when + the derivation is built. - - `--requisites`; `-R`\ - Prints out the [closure] of the store path *paths*. + [output paths]: @docroot@/glossary.md#gloss-output-path - [closure]: ../../glossary.md#gloss-closure +- `--requisites` / `-R` - This query has one option: + Prints out the [closure] of the store path *paths*. - - `--include-outputs` - Also include the existing output paths of [store derivation]s, - and their closures. + [closure]: @docroot@/glossary.md#gloss-closure - This query can be used to implement various kinds of deployment. A - *source deployment* is obtained by distributing the closure of a - store derivation. A *binary deployment* is obtained by distributing - the closure of an output path. A *cache deployment* (combined - source/binary deployment, including binaries of build-time-only - dependencies) is obtained by distributing the closure of a store - derivation and specifying the option `--include-outputs`. + This query has one option: - - `--references`\ - Prints the set of [references] of the store paths - *paths*, that is, their immediate dependencies. (For *all* - dependencies, use `--requisites`.) + - `--include-outputs` + Also include the existing output paths of [store derivation]s, + and their closures. - [references]: ../../glossary.md#gloss-reference + This query can be used to implement various kinds of deployment. A + *source deployment* is obtained by distributing the closure of a + store derivation. A *binary deployment* is obtained by distributing + the closure of an output path. A *cache deployment* (combined + source/binary deployment, including binaries of build-time-only + dependencies) is obtained by distributing the closure of a store + derivation and specifying the option `--include-outputs`. - - `--referrers`\ - Prints the set of *referrers* of the store paths *paths*, that is, - the store paths currently existing in the Nix store that refer to - one of *paths*. Note that contrary to the references, the set of - referrers is not constant; it can change as store paths are added or - removed. +- `--references` - - `--referrers-closure`\ - Prints the closure of the set of store paths *paths* under the - referrers relation; that is, all store paths that directly or - indirectly refer to one of *paths*. These are all the path currently - in the Nix store that are dependent on *paths*. + Prints the set of [references] of the store paths + *paths*, that is, their immediate dependencies. (For *all* + dependencies, use `--requisites`.) - - `--deriver`; `-d`\ - Prints the [deriver] that was used to build the store paths *paths*. If - the path has no deriver (e.g., if it is a source file), or if the - deriver is not known (e.g., in the case of a binary-only - deployment), the string `unknown-deriver` is printed. - The returned deriver is not guaranteed to exist in the local store, for - example when *paths* were substituted from a binary cache. - Use `--valid-derivers` instead to obtain valid paths only. + [references]: @docroot@/glossary.md#gloss-reference - [deriver]: ../../glossary.md#gloss-deriver +- `--referrers` - - `--valid-derivers`\ - Prints a set of derivation files (`.drv`) which are supposed produce - said paths when realized. Might print nothing, for example for source paths - or paths subsituted from a binary cache. + Prints the set of *referrers* of the store paths *paths*, that is, + the store paths currently existing in the Nix store that refer to + one of *paths*. Note that contrary to the references, the set of + referrers is not constant; it can change as store paths are added or + removed. - - `--graph`\ - Prints the references graph of the store paths *paths* in the format - of the `dot` tool of AT\&T's [Graphviz - package](http://www.graphviz.org/). This can be used to visualise - dependency graphs. To obtain a build-time dependency graph, apply - this to a store derivation. To obtain a runtime dependency graph, - apply it to an output path. +- `--referrers-closure` - - `--tree`\ - Prints the references graph of the store paths *paths* as a nested - ASCII tree. References are ordered by descending closure size; this - tends to flatten the tree, making it more readable. The query only - recurses into a store path when it is first encountered; this - prevents a blowup of the tree representation of the graph. + Prints the closure of the set of store paths *paths* under the + referrers relation; that is, all store paths that directly or + indirectly refer to one of *paths*. These are all the path currently + in the Nix store that are dependent on *paths*. - - `--graphml`\ - Prints the references graph of the store paths *paths* in the - [GraphML](http://graphml.graphdrawing.org/) file format. This can be - used to visualise dependency graphs. To obtain a build-time - dependency graph, apply this to a [store derivation]. To obtain a - runtime dependency graph, apply it to an output path. +- `--deriver` / `-d` - - `--binding` *name*; `-b` *name*\ - Prints the value of the attribute *name* (i.e., environment - variable) of the [store derivation]s *paths*. It is an error for a - derivation to not have the specified attribute. + Prints the [deriver] that was used to build the store paths *paths*. If + the path has no deriver (e.g., if it is a source file), or if the + deriver is not known (e.g., in the case of a binary-only + deployment), the string `unknown-deriver` is printed. + The returned deriver is not guaranteed to exist in the local store, for + example when *paths* were substituted from a binary cache. + Use `--valid-derivers` instead to obtain valid paths only. - - `--hash`\ - Prints the SHA-256 hash of the contents of the store paths *paths* - (that is, the hash of the output of `nix-store --dump` on the given - paths). Since the hash is stored in the Nix database, this is a fast - operation. + [deriver]: @docroot@/glossary.md#gloss-deriver - - `--size`\ - Prints the size in bytes of the contents of the store paths *paths* - — to be precise, the size of the output of `nix-store --dump` on - the given paths. Note that the actual disk space required by the - store paths may be higher, especially on filesystems with large - cluster sizes. +- `--valid-derivers` - - `--roots`\ - Prints the garbage collector roots that point, directly or - indirectly, at the store paths *paths*. + Prints a set of derivation files (`.drv`) which are supposed produce + said paths when realized. Might print nothing, for example for source paths + or paths subsituted from a binary cache. + +- `--graph` + + Prints the references graph of the store paths *paths* in the format + of the `dot` tool of AT\&T's [Graphviz + package](http://www.graphviz.org/). This can be used to visualise + dependency graphs. To obtain a build-time dependency graph, apply + this to a store derivation. To obtain a runtime dependency graph, + apply it to an output path. + +- `--tree` + + Prints the references graph of the store paths *paths* as a nested + ASCII tree. References are ordered by descending closure size; this + tends to flatten the tree, making it more readable. The query only + recurses into a store path when it is first encountered; this + prevents a blowup of the tree representation of the graph. + +- `--graphml` + + Prints the references graph of the store paths *paths* in the + [GraphML](http://graphml.graphdrawing.org/) file format. This can be + used to visualise dependency graphs. To obtain a build-time + dependency graph, apply this to a [store derivation]. To obtain a + runtime dependency graph, apply it to an output path. + +- `--binding` *name* / `-b` *name* + + Prints the value of the attribute *name* (i.e., environment + variable) of the [store derivation]s *paths*. It is an error for a + derivation to not have the specified attribute. + +- `--hash` + + Prints the SHA-256 hash of the contents of the store paths *paths* + (that is, the hash of the output of `nix-store --dump` on the given + paths). Since the hash is stored in the Nix database, this is a fast + operation. + +- `--size` + + Prints the size in bytes of the contents of the store paths *paths* + — to be precise, the size of the output of `nix-store --dump` on + the given paths. Note that the actual disk space required by the + store paths may be higher, especially on filesystems with large + cluster sizes. + +- `--roots` + + Prints the garbage collector roots that point, directly or + indirectly, at the store paths *paths*. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index 5428d57fa..a899758df 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -25,14 +25,14 @@ Each of *paths* is processed as follows: If no substitutes are available and no store derivation is given, realisation fails. -[store paths]: @docroot@/glossary.md#gloss-store-path +[store paths]: @docroot@/store/store-path.md [valid]: @docroot@/glossary.md#gloss-validity [store derivation]: @docroot@/glossary.md#gloss-store-derivation [output paths]: @docroot@/glossary.md#gloss-output-path -[store objects]: @docroot@/glossary.md#gloss-store-object +[store objects]: @docroot@/store/store-object.md [closure]: @docroot@/glossary.md#gloss-closure [substituters]: @docroot@/command-ref/conf-file.md#conf-substituters -[content-addressed derivations]: @docroot@/contributing/experimental-features.md#xp-feature-ca-derivations +[content-addressed derivations]: @docroot@/development/experimental-features.md#xp-feature-ca-derivations [Nix database]: @docroot@/glossary.md#gloss-nix-database The resulting paths are printed on standard output. @@ -42,23 +42,26 @@ For non-derivation arguments, the argument itself is printed. # Options - - `--dry-run`\ - Print on standard error a description of what packages would be - built or downloaded, without actually performing the operation. +- `--dry-run` - - `--ignore-unknown`\ - If a non-derivation path does not have a substitute, then silently - ignore it. + Print on standard error a description of what packages would be + built or downloaded, without actually performing the operation. - - `--check`\ - This option allows you to check whether a derivation is - deterministic. It rebuilds the specified derivation and checks - whether the result is bitwise-identical with the existing outputs, - printing an error if that’s not the case. The outputs of the - specified derivation must already exist. When used with `-K`, if an - output path is not identical to the corresponding output from the - previous build, the new output path is left in - `/nix/store/name.check.` +- `--ignore-unknown` + + If a non-derivation path does not have a substitute, then silently + ignore it. + +- `--check` + + This option allows you to check whether a derivation is + deterministic. It rebuilds the specified derivation and checks + whether the result is bitwise-identical with the existing outputs, + printing an error if that’s not the case. The outputs of the + specified derivation must already exist. When used with `-K`, if an + output path is not identical to the corresponding output from the + previous build, the new output path is left in + `/nix/store/name.check.` {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/restore.md b/doc/manual/src/command-ref/nix-store/restore.md index fcba43df4..2d0aa3127 100644 --- a/doc/manual/src/command-ref/nix-store/restore.md +++ b/doc/manual/src/command-ref/nix-store/restore.md @@ -8,9 +8,11 @@ ## Description -The operation `--restore` unpacks a NAR archive to *path*, which must +The operation `--restore` unpacks a [Nix Archive (NAR)][Nix Archive] to *path*, which must not already exist. The archive is read from standard input. +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive + {{#include ./opt-common.md}} {{#include ../opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/serve.md b/doc/manual/src/command-ref/nix-store/serve.md index 0f90f65ae..9a4cf5216 100644 --- a/doc/manual/src/command-ref/nix-store/serve.md +++ b/doc/manual/src/command-ref/nix-store/serve.md @@ -14,10 +14,11 @@ access to a restricted ssh user. The following flags are available: - - `--write`\ - Allow the connected client to request the realization of - derivations. In effect, this can be used to make the host act as a - remote builder. +- `--write` + + Allow the connected client to request the realization of + derivations. In effect, this can be used to make the host act as a + remote builder. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/verify.md b/doc/manual/src/command-ref/nix-store/verify.md index 2695b3361..40c9180db 100644 --- a/doc/manual/src/command-ref/nix-store/verify.md +++ b/doc/manual/src/command-ref/nix-store/verify.md @@ -16,18 +16,20 @@ being modified by non-Nix tools, or of bugs in Nix itself. This operation has the following options: - - `--check-contents`\ - Checks that the contents of every valid store path has not been - altered by computing a SHA-256 hash of the contents and comparing it - with the hash stored in the Nix database at build time. Paths that - have been modified are printed out. For large stores, - `--check-contents` is obviously quite slow. +- `--check-contents` - - `--repair`\ - If any valid path is missing from the store, or (if - `--check-contents` is given) the contents of a valid path has been - modified, then try to repair the path by redownloading it. See - `nix-store --repair-path` for details. + Checks that the contents of every valid store path has not been + altered by computing a SHA-256 hash of the contents and comparing it + with the hash stored in the Nix database at build time. Paths that + have been modified are printed out. For large stores, + `--check-contents` is obviously quite slow. + +- `--repair` + + If any valid path is missing from the store, or (if + `--check-contents` is given) the contents of a valid path has been + modified, then try to repair the path by redownloading it. See + `nix-store --repair-path` for details. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index 114b292f9..69a700207 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -37,7 +37,7 @@ Most Nix commands accept the following command-line options: Print even more informational messages. - `4` “Debug†- + Print debug information. - `5` “Vomit†@@ -143,7 +143,7 @@ Most Nix commands accept the following command-line options: This option is accepted by `nix-env`, `nix-instantiate`, `nix-shell` and `nix-build`. When evaluating Nix expressions, the expression evaluator will automatically try to call functions that it encounters. - It can automatically call functions for which every argument has a [default value](@docroot@/language/constructs.md#functions) (e.g., `{ argName ? defaultValue }: ...`). + It can automatically call functions for which every argument has a [default value](@docroot@/language/syntax.md#functions) (e.g., `{ argName ? defaultValue }: ...`). With `--arg`, you can also call functions that have arguments without a default value (or override a default value). That is, if the evaluator encounters a function with an argument named *name*, it will call it with value *value*. @@ -187,11 +187,12 @@ Most Nix commands accept the following command-line options: For `nix-shell`, this option is commonly used to give you a shell in which you can build the packages returned by the expression. If you want to get a shell which contain the *built* packages ready for use, give your expression to the `nix-shell --packages ` convenience flag instead. -- [`-I`](#opt-I) *path* +- [`-I` / `--include`](#opt-I) *path* - Add an entry to the [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path). + Add an entry to the list of search paths used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). This option may be given multiple times. - Paths added through `-I` take precedence over [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH). + + Paths added through `-I` take precedence over the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path) and the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH). - [`--option`](#opt-option) *name* *value* diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/development/building.md similarity index 67% rename from doc/manual/src/contributing/hacking.md rename to doc/manual/src/development/building.md index 916ec3077..5a5fb3368 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/development/building.md @@ -1,24 +1,67 @@ -# Hacking +# Building Nix -This section provides some notes on how to hack on Nix. To get the -latest version of Nix from GitHub: +This section provides some notes on how to start hacking on Nix. +To get the latest version of Nix from GitHub: ```console $ git clone https://github.com/NixOS/nix.git $ cd nix ``` -The following instructions assume you already have some version of Nix installed locally, so that you can use it to set up the development environment. If you don't have it installed, follow the [installation instructions]. +> **Note** +> +> The following instructions assume you already have some version of Nix installed locally, so that you can use it to set up the development environment. +> If you don't have it installed, follow the [installation instructions](../installation/index.md). -[installation instructions]: ../installation/index.md + +To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: + +```console +$ nix-shell +``` + +To get a shell with one of the other [supported compilation environments](#compilation-environments): + +```console +$ nix-shell --attr devShells.x86_64-linux.native-clangStdenvPackages +``` + +> **Note** +> +> You can use `native-ccacheStdenvPackages` to drastically improve rebuild time. +> By default, [ccache](https://ccache.dev) keeps artifacts in `~/.cache/ccache/`. + +To build Nix itself in this shell: + +```console +[nix-shell]$ autoreconfPhase +[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out +[nix-shell]$ make -j $NIX_BUILD_CORES +``` + +To install it in `$(pwd)/outputs` and test it: + +```console +[nix-shell]$ make install +[nix-shell]$ make installcheck -j $NIX_BUILD_CORES +[nix-shell]$ ./outputs/out/bin/nix --version +nix (Nix) 2.12 +``` + +To build a release version of Nix for the current operating system and CPU architecture: + +```console +$ nix-build +``` + +You can also build Nix for one of the [supported platforms](#platforms). ## Building Nix with flakes This section assumes you are using Nix with the [`flakes`] and [`nix-command`] experimental features enabled. -See the [Building Nix](#building-nix) section for equivalent instructions using stable Nix interfaces. -[`flakes`]: @docroot@/contributing/experimental-features.md#xp-feature-flakes -[`nix-command`]: @docroot@/contributing/experimental-features.md#xp-nix-command +[`flakes`]: @docroot@/development/experimental-features.md#xp-feature-flakes +[`nix-command`]: @docroot@/development/experimental-features.md#xp-nix-command To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: @@ -67,50 +110,6 @@ $ nix build You can also build Nix for one of the [supported platforms](#platforms). -## Building Nix - -To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: - -```console -$ nix-shell -``` - -To get a shell with one of the other [supported compilation environments](#compilation-environments): - -```console -$ nix-shell --attr devShells.x86_64-linux.native-clangStdenvPackages -``` - -> **Note** -> -> You can use `native-ccacheStdenvPackages` to drastically improve rebuild time. -> By default, [ccache](https://ccache.dev) keeps artifacts in `~/.cache/ccache/`. - -To build Nix itself in this shell: - -```console -[nix-shell]$ autoreconfPhase -[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out -[nix-shell]$ make -j $NIX_BUILD_CORES -``` - -To install it in `$(pwd)/outputs` and test it: - -```console -[nix-shell]$ make install -[nix-shell]$ make installcheck -j $NIX_BUILD_CORES -[nix-shell]$ ./outputs/out/bin/nix --version -nix (Nix) 2.12 -``` - -To build a release version of Nix for the current operating system and CPU architecture: - -```console -$ nix-build -``` - -You can also build Nix for one of the [supported platforms](#platforms). - ## Makefile variables You may need `profiledir=$out/etc/profile.d` and `sysconfdir=$out/etc` to run `make install`. @@ -122,7 +121,6 @@ Run `make` with [`-e` / `--environment-overrides`](https://www.gnu.org/software/ The docs can take a while to build, so you may want to disable this for local development. - `ENABLE_FUNCTIONAL_TESTS=yes` to enable building the functional tests. -- `ENABLE_UNIT_TESTS=yes` to enable building the unit tests. - `OPTIMIZE=1` to enable optimizations. - `libraries=libutil programs=` to only build a specific library. @@ -144,6 +142,7 @@ Nix can be built for various platforms, as specified in [`flake.nix`]: - `aarch64-darwin` - `armv6l-linux` - `armv7l-linux` +- `riscv64-linux` In order to build Nix for a different platform than the one you're currently on, you need a way for your current Nix installation to build code for that @@ -166,7 +165,10 @@ or for Nix with the [`flakes`] and [`nix-command`] experimental features enabled $ nix build .#packages.aarch64-linux.default ``` -Cross-compiled builds are available for ARMv6 (`armv6l-linux`) and ARMv7 (`armv7l-linux`). +Cross-compiled builds are available for: +- `armv6l-linux` +- `armv7l-linux` +- `riscv64-linux` Add more [system types](#system-type) to `crossSystems` in `flake.nix` to bootstrap Nix on unsupported platforms. ### Building for multiple platforms at once @@ -196,7 +198,7 @@ In order to facilitate this, Nix has some support for being built out of tree ## System type -Nix uses a string with he following format to identify the *system type* or *platform* it runs on: +Nix uses a string with the following format to identify the *system type* or *platform* it runs on: ``` -[-] @@ -258,10 +260,10 @@ See [supported compilation environments](#compilation-environments) and instruct To use the LSP with your editor, you first need to [set up `clangd`](https://clangd.llvm.org/installation#project-setup) by running: ```console -make clean && bear -- make -j$NIX_BUILD_CORES default check install +make compile_commands.json ``` -Configure your editor to use the `clangd` from the shell, either by running it inside the development shell, or by using [nix-direnv](https://github.com/nix-community/nix-direnv) and [the appropriate editor plugin](https://github.com/direnv/direnv/wiki#editor-integration). +Configure your editor to use the `clangd` from the `.#native-clangStdenvPackages` shell. You can do that either by running it inside the development shell, or by using [nix-direnv](https://github.com/nix-community/nix-direnv) and [the appropriate editor plugin](https://github.com/direnv/direnv/wiki#editor-integration). > **Note** > @@ -269,80 +271,25 @@ Configure your editor to use the `clangd` from the shell, either by running it i > Some other editors (e.g. Emacs, Vim) need a plugin to support LSP servers in general (e.g. [lsp-mode](https://github.com/emacs-lsp/lsp-mode) for Emacs and [vim-lsp](https://github.com/prabirshrestha/vim-lsp) for vim). > Editor-specific setup is typically opinionated, so we will not cover it here in more detail. -## Add a release note +## Formatting and pre-commit hooks -`doc/manual/rl-next` contains release notes entries for all unreleased changes. +You may run the formatters as a one-off using: -User-visible changes should come with a release note. - -### Add an entry - -Here's what a complete entry looks like. The file name is not incorporated in the document. - -``` ---- -synopsis: Basically a title -issues: 1234 -prs: 1238 ---- - -Here's one or more paragraphs that describe the change. - -- It's markdown -- Add references to the manual using @docroot@ +```console +make format ``` -Significant changes should add the following header, which moves them to the top. +If you'd like to run the formatters before every commit, install the hooks: ``` -significance: significant +pre-commit-hooks-install ``` - -See also the [format documentation](https://github.com/haskell/cabal/blob/master/CONTRIBUTING.md#changelog). +This installs [pre-commit](https://pre-commit.com) using [cachix/git-hooks.nix](https://github.com/cachix/git-hooks.nix). -### Build process +When making a commit, pay attention to the console output. +If it fails, run `git add --patch` to approve the suggestions _and commit again_. -Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`. - -## Branches - -- [`master`](https://github.com/NixOS/nix/commits/master) - - The main development branch. All changes are approved and merged here. - When developing a change, create a branch based on the latest `master`. - - Maintainers try to [keep it in a release-worthy state](#reverting). - -- [`maintenance-*.*`](https://github.com/NixOS/nix/branches/all?query=maintenance) - - These branches are the subject of backports only, and are - also [kept](#reverting) in a release-worthy state. - - See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) - -- [`latest-release`](https://github.com/NixOS/nix/tree/latest-release) - - The latest patch release of the latest minor version. - - See [`maintainers/release-process.md`](https://github.com/NixOS/nix/blob/master/maintainers/release-process.md) - -- [`backport-*-to-*`](https://github.com/NixOS/nix/branches/all?query=backport) - - Generally branches created by the backport action. - - See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) - -- [_other_](https://github.com/NixOS/nix/branches/all) - - Branches that do not conform to the above patterns should be feature branches. - -## Reverting - -If a change turns out to be merged by mistake, or contain a regression, it may be reverted. -A revert is not a rejection of the contribution, but merely part of an effective development process. -It makes sure that development keeps running smoothly, with minimal uncertainty, and less overhead. -If maintainers have to worry too much about avoiding reverts, they would not be able to merge as much. -By embracing reverts as a good part of the development process, everyone wins. - -However, taking a step back may be frustrating, so maintainers will be extra supportive on the next try. +To refresh pre-commit hook's config file, do the following: +1. Exit the development shell and start it again by running `nix develop`. +2. If you also use the pre-commit hook, also run `pre-commit-hooks-install` again. diff --git a/doc/manual/src/contributing/cli-guideline.md b/doc/manual/src/development/cli-guideline.md similarity index 88% rename from doc/manual/src/contributing/cli-guideline.md rename to doc/manual/src/development/cli-guideline.md index e90d6de8d..23df844ec 100644 --- a/doc/manual/src/contributing/cli-guideline.md +++ b/doc/manual/src/development/cli-guideline.md @@ -389,88 +389,6 @@ colors, no emojis and using ASCII instead of Unicode symbols). The same should happen when TTY is not detected on STDERR. We should not display progress / status section, but only print warnings and errors. -## Returning future proof JSON - -The schema of JSON output should allow for backwards compatible extension. This section explains how to achieve this. - -Two definitions are helpful here, because while JSON only defines one "key-value" -object type, we use it to cover two use cases: - - - **dictionary**: a map from names to value that all have the same type. In - C++ this would be a `std::map` with string keys. - - **record**: a fixed set of attributes each with their own type. In C++, this - would be represented by a `struct`. - -It is best not to mix these use cases, as that may lead to incompatibilities when the schema changes. For example, adding a record field to a dictionary breaks consumers that assume all JSON object fields to have the same meaning and type. - -This leads to the following guidelines: - - - The top-level (root) value must be a record. - - Otherwise, one can not change the structure of a command's output. - - - The value of a dictionary item must be a record. - - Otherwise, the item type can not be extended. - - - List items should be records. - - Otherwise, one can not change the structure of the list items. - - If the order of the items does not matter, and each item has a unique key that is a string, consider representing the list as a dictionary instead. If the order of the items needs to be preserved, return a list of records. - - - Streaming JSON should return records. - - An example of a streaming JSON format is [JSON lines](https://jsonlines.org/), where each line represents a JSON value. These JSON values can be considered top-level values or list items, and they must be records. - -### Examples - - -This is bad, because all keys must be assumed to be store types: - -```json -{ - "local": { ... }, - "remote": { ... }, - "http": { ... } -} -``` - -This is good, because the it is extensible at the root, and is somewhat self-documenting: - -```json -{ - "storeTypes": { "local": { ... }, ... }, - "pluginSupport": true -} -``` - -While the dictionary of store types seems like a very complete response at first, a use case may arise that warrants returning additional information. -For example, the presence of plugin support may be crucial information for a client to proceed when their desired store type is missing. - - - -The following representation is bad because it is not extensible: - -```json -{ "outputs": [ "out" "bin" ] } -``` - -However, simply converting everything to records is not enough, because the order of outputs must be preserved: - -```json -{ "outputs": { "bin": {}, "out": {} } } -``` - -The first item is the default output. Deriving this information from the outputs ordering is not great, but this is how Nix currently happens to work. -While it is possible for a JSON parser to preserve the order of fields, we can not rely on this capability to be present in all JSON libraries. - -This representation is extensible and preserves the ordering: - -```json -{ "outputs": [ { "outputName": "out" }, { "outputName": "bin" } ] } -``` - ## Dialog with the user CLIs don't always make it clear when an action has taken place. For every diff --git a/doc/manual/src/development/contributing.md b/doc/manual/src/development/contributing.md new file mode 100644 index 000000000..7de7489dc --- /dev/null +++ b/doc/manual/src/development/contributing.md @@ -0,0 +1,79 @@ +# Contributing + +## Add a release note + +`doc/manual/rl-next` contains release notes entries for all unreleased changes. + +User-visible changes should come with a release note. + +### Add an entry + +Here's what a complete entry looks like. The file name is not incorporated in the document. + +``` +--- +synopsis: Basically a title +issues: 1234 +prs: 1238 +--- + +Here's one or more paragraphs that describe the change. + +- It's markdown +- Add references to the manual using @docroot@ +``` + +Significant changes should add the following header, which moves them to the top. + +``` +significance: significant +``` + + +See also the [format documentation](https://github.com/haskell/cabal/blob/master/CONTRIBUTING.md#changelog). + +### Build process + +Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`. + +## Branches + +- [`master`](https://github.com/NixOS/nix/commits/master) + + The main development branch. All changes are approved and merged here. + When developing a change, create a branch based on the latest `master`. + + Maintainers try to [keep it in a release-worthy state](#reverting). + +- [`maintenance-*.*`](https://github.com/NixOS/nix/branches/all?query=maintenance) + + These branches are the subject of backports only, and are + also [kept](#reverting) in a release-worthy state. + + See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) + +- [`latest-release`](https://github.com/NixOS/nix/tree/latest-release) + + The latest patch release of the latest minor version. + + See [`maintainers/release-process.md`](https://github.com/NixOS/nix/blob/master/maintainers/release-process.md) + +- [`backport-*-to-*`](https://github.com/NixOS/nix/branches/all?query=backport) + + Generally branches created by the backport action. + + See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) + +- [_other_](https://github.com/NixOS/nix/branches/all) + + Branches that do not conform to the above patterns should be feature branches. + +## Reverting + +If a change turns out to be merged by mistake, or contain a regression, it may be reverted. +A revert is not a rejection of the contribution, but merely part of an effective development process. +It makes sure that development keeps running smoothly, with minimal uncertainty, and less overhead. +If maintainers have to worry too much about avoiding reverts, they would not be able to merge as much. +By embracing reverts as a good part of the development process, everyone wins. + +However, taking a step back may be frustrating, so maintainers will be extra supportive on the next try. diff --git a/doc/manual/src/contributing/cxx.md b/doc/manual/src/development/cxx.md similarity index 100% rename from doc/manual/src/contributing/cxx.md rename to doc/manual/src/development/cxx.md diff --git a/doc/manual/src/contributing/documentation.md b/doc/manual/src/development/documentation.md similarity index 89% rename from doc/manual/src/contributing/documentation.md rename to doc/manual/src/development/documentation.md index 1dddb207c..63f574ab7 100644 --- a/doc/manual/src/contributing/documentation.md +++ b/doc/manual/src/development/documentation.md @@ -24,14 +24,12 @@ nix build .#^doc and open `./result-doc/share/doc/nix/manual/index.html`. -To build the manual incrementally, [enter the development shell](./hacking.md) and run: +To build the manual incrementally, [enter the development shell](./building.md) and run: ```console -make manual-html -j $NIX_BUILD_CORES +make manual-html-open -j $NIX_BUILD_CORES ``` -and open `./outputs/out/share/doc/nix/manual/language/index.html`. - In order to reflect changes to the [Makefile for the manual], clear all generated files before re-building: [Makefile for the manual]: https://github.com/NixOS/nix/blob/master/doc/manual/local.mk @@ -149,7 +147,7 @@ Please observe these guidelines to ease reviews: ``` A [store object] contains a [file system object] and [references] to other store objects. - [store object]: @docroot@/glossary.md#gloss-store-object + [store object]: @docroot@/store/store-object.md [file system object]: @docroot@/architecture/file-system-object.md [references]: @docroot@/glossary.md#gloss-reference ``` @@ -198,13 +196,35 @@ You can also build and view it yourself: [Doxygen API documentation]: https://hydra.nixos.org/job/nix/master/internal-api-docs/latest/download-by-type/doc/internal-api-docs ```console -# nix build .#hydraJobs.internal-api-docs -# xdg-open ./result/share/doc/nix/internal-api/html/index.html +$ nix build .#hydraJobs.internal-api-docs +$ xdg-open ./result/share/doc/nix/internal-api/html/index.html +``` + +or inside `nix-shell` or `nix develop`: + +```console +$ mesonConfigurePhase +$ ninja src/internal-api-docs/html +$ xdg-open src/internal-api-docs/html/index.html +``` + +## C API documentation + +Note that the C API is not yet stable. +[C API documentation] is available online. +You can also build and view it yourself: + +[C API documentation]: https://hydra.nixos.org/job/nix/master/external-api-docs/latest/download-by-type/doc/external-api-docs + +```console +$ nix build .#hydraJobs.external-api-docs +$ xdg-open ./result/share/doc/nix/external-api/html/index.html ``` or inside `nix-shell` or `nix develop`: ``` -# make internal-api-html -# xdg-open ./outputs/doc/share/doc/nix/internal-api/html/index.html +$ mesonConfigurePhase +$ ninja src/external-api-docs/html +$ xdg-open src/external-api-docs/html/index.html ``` diff --git a/doc/manual/src/contributing/experimental-features.md b/doc/manual/src/development/experimental-features.md similarity index 100% rename from doc/manual/src/contributing/experimental-features.md rename to doc/manual/src/development/experimental-features.md diff --git a/doc/manual/src/contributing/index.md b/doc/manual/src/development/index.md similarity index 77% rename from doc/manual/src/contributing/index.md rename to doc/manual/src/development/index.md index 4d55c17a4..6403c3e66 100644 --- a/doc/manual/src/contributing/index.md +++ b/doc/manual/src/development/index.md @@ -5,4 +5,4 @@ Check the [contributing guide](https://github.com/NixOS/nix/blob/master/CONTRIBU This chapter is a collection of guides for making changes to the code and documentation. -If you're not sure where to start, try to [compile Nix from source](./hacking.md) and consider [making improvements to documentation](./documentation.md). +If you're not sure where to start, try to [compile Nix from source](./building.md) and consider [making improvements to documentation](./documentation.md). diff --git a/doc/manual/src/development/json-guideline.md b/doc/manual/src/development/json-guideline.md new file mode 100644 index 000000000..b4bc92af9 --- /dev/null +++ b/doc/manual/src/development/json-guideline.md @@ -0,0 +1,128 @@ +# JSON guideline + +Nix consumes and produces JSON in a variety of contexts. +These guidelines ensure consistent practices for all our JSON interfaces, for ease of use, and so that experience in one part carries over to another. + +## Extensibility + +The schema of JSON input and output should allow for backwards compatible extension. +This section explains how to achieve this. + +Two definitions are helpful here, because while JSON only defines one "key-value" object type, we use it to cover two use cases: + + - **dictionary**: a map from names to value that all have the same type. + In C++ this would be a `std::map` with string keys. + + - **record**: a fixed set of attributes each with their own type. + In C++, this would be represented by a `struct`. + +It is best not to mix these use cases, as that may lead to incompatibilities when the schema changes. +For example, adding a record field to a dictionary breaks consumers that assume all JSON object fields to have the same meaning and type, and dictionary items with a colliding name can not be represented anymore. + +This leads to the following guidelines: + + - The top-level (root) value must be a record. + + Otherwise, one can not change the structure of a command's output. + + - The value of a dictionary item must be a record. + + Otherwise, the item type can not be extended. + + - List items should be records. + + Otherwise, one can not change the structure of the list items. + + If the order of the items does not matter, and each item has a unique key that is a string, consider representing the list as a dictionary instead. + If the order of the items needs to be preserved, return a list of records. + + - Streaming JSON should return records. + + An example of a streaming JSON format is [JSON lines](https://jsonlines.org/), where each line represents a JSON value. + These JSON values can be considered top-level values or list items, and they must be records. + +### Examples + +This is bad, because all keys must be assumed to be store types: + +```json +{ + "local": { ... }, + "remote": { ... }, + "http": { ... } +} +``` + +This is good, because the it is extensible at the root, and is somewhat self-documenting: + +```json +{ + "storeTypes": { "local": { ... }, ... }, + "pluginSupport": true +} +``` + +While the dictionary of store types seems like a very complete response at first, a use case may arise that warrants returning additional information. +For example, the presence of plugin support may be crucial information for a client to proceed when their desired store type is missing. + + + +The following representation is bad because it is not extensible: + +```json +{ "outputs": [ "out" "bin" ] } +``` + +However, simply converting everything to records is not enough, because the order of outputs must be preserved: + +```json +{ "outputs": { "bin": {}, "out": {} } } +``` + +The first item is the default output. Deriving this information from the outputs ordering is not great, but this is how Nix currently happens to work. +While it is possible for a JSON parser to preserve the order of fields, we can not rely on this capability to be present in all JSON libraries. + +This representation is extensible and preserves the ordering: + +```json +{ "outputs": [ { "outputName": "out" }, { "outputName": "bin" } ] } +``` + +## Self-describing values + +As described in the previous section, it's crucial that schemas can be extended with with new fields without breaking compatibility. +However, that should *not* mean we use the presence/absence of fields to indicate optional information *within* a version of the schema. +Instead, always include the field, and use `null` to indicate the "nothing" case. + +### Examples + +Here are two JSON objects: + +```json +{ + "foo": {} +} +``` +```json +{ + "foo": {}, + "bar": {} +} +``` + +Since they differ in which fields they contain, they should *not* both be valid values of the same schema. +At most, they can match two different schemas where the second (with `foo` and `bar`) is considered a newer version of the first (with just `foo`). +Within each version, all fields are mandatory (always `foo`, and always `foo` and `bar`). +Only *between* each version, `bar` gets added as a new mandatory field. + +Here are another two JSON objects: + +```json +{ "foo": null } +``` +```json +{ "foo": { "bar": 1 } } +``` + +Since they both contain a `foo` field, they could be valid values of the same schema. +The schema would have `foo` has an optional field, which is either `null` or an object where `bar` is an integer. diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/development/testing.md similarity index 81% rename from doc/manual/src/contributing/testing.md rename to doc/manual/src/development/testing.md index 31c39c16c..3949164d5 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/development/testing.md @@ -59,15 +59,15 @@ The unit tests are defined using the [googletest] and [rapidcheck] frameworks. > … > ``` -The tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `tests/unit/${library_name_without-nix}`. -Given a interface (header) and implementation pair in the original library, say, `src/libexpr/value/context.{hh,cc}`, we write tests for it in `tests/unit/libexpr/tests/value/context.cc`, and (possibly) declare/define additional interfaces for testing purposes in `tests/unit/libexpr-support/tests/value/context.{hh,cc}`. +The tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `src/${library_name_without-nix}-test`. +Given an interface (header) and implementation pair in the original library, say, `src/libexpr/value/context.{hh,cc}`, we write tests for it in `src/nix-expr-tests/value/context.cc`, and (possibly) declare/define additional interfaces for testing purposes in `src/nix-expr-test-support/tests/value/context.{hh,cc}`. Data for unit tests is stored in a `data` subdir of the directory for each unit test executable. -For example, `libnixstore` code is in `src/libstore`, and its test data is in `tests/unit/libstore/data`. -The path to the `tests/unit/data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`. +For example, `libnixstore` code is in `src/libstore`, and its test data is in `src/nix-store-tests/data`. +The path to the `src/${library_name_without-nix}-test/data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`. Note that each executable only gets the data for its tests. -The unit test libraries are in `tests/unit/${library_name_without-nix}-lib`. +The unit test libraries are in `src/${library_name_without-nix}-test-support`. All headers are in a `tests` subdirectory so they are included with `#include "tests/"`. The use of all these separate directories for the unit tests might seem inconvenient, as for example the tests are not "right next to" the part of the code they are testing. @@ -76,8 +76,25 @@ there is no risk of any build-system wildcards for the library accidentally pick ### Running tests -You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. -Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option, or the `GTEST_FILTER` environment variable, e.g. `GTEST_FILTER='ErrorTraceTest.*' make check`. +You can run the whole testsuite with `meson test` from the Meson build directory, or the tests for a specific component with `meson test nix-store-tests`. +A environment variables that Google Test accepts are also worth knowing: + +1. [`GTEST_FILTER`](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) + + This is used for finer-grained filtering of which tests to run. + + +2. [`GTEST_BRIEF`](https://google.github.io/googletest/advanced.html#suppressing-test-passes) + + This is used to avoid logging passing tests. + +Putting the two together, one might run + +```bash +GTEST_BRIEF=1 GTEST_FILTER='ErrorTraceTest.*' meson test nix-expr-tests -v +``` + +for short but comprensive output. ### Characterisation testing { #characaterisation-testing-unit } @@ -86,7 +103,7 @@ See [functional characterisation testing](#characterisation-testing-functional) Like with the functional characterisation, `_NIX_TEST_ACCEPT=1` is also used. For example: ```shell-session -$ _NIX_TEST_ACCEPT=1 make libstore-tests_RUN +$ _NIX_TEST_ACCEPT=1 meson test nix-store-tests -v ... [ SKIPPED ] WorkerProtoTest.string_read [ SKIPPED ] WorkerProtoTest.string_write @@ -114,6 +131,8 @@ On other platforms they wouldn't be run at all. The functional tests reside under the `tests/functional` directory and are listed in `tests/functional/local.mk`. Each test is a bash script. +Functional tests are run during `installCheck` in the `nix` package build, as well as separately from the build, in VM tests. + ### Running the whole test suite The whole test suite can be run with: @@ -162,14 +181,14 @@ ran test tests/functional/${testName}.sh... [PASS] or without `make`: ```shell-session -$ ./mk/run-test.sh tests/functional/${testName}.sh tests/functional/init.sh +$ ./mk/run-test.sh tests/functional/${testName}.sh ran test tests/functional/${testName}.sh... [PASS] ``` To see the complete output, one can also run: ```shell-session -$ ./mk/debug-test.sh tests/functional/${testName}.sh tests/functional/init.sh +$ ./mk/debug-test.sh tests/functional/${testName}.sh +(${testName}.sh:1) foo output from foo +(${testName}.sh:2) bar @@ -204,7 +223,7 @@ edit it like so: Then, running the test with `./mk/debug-test.sh` will drop you into GDB once the script reaches that point: ```shell-session -$ ./mk/debug-test.sh tests/functional/${testName}.sh tests/functional/init.sh +$ ./mk/debug-test.sh tests/functional/${testName}.sh ... + gdb blash blub GNU gdb (GDB) 12.1 @@ -252,13 +271,30 @@ Regressions are caught, and improvements always show up in code review. To ensure that characterisation testing doesn't make it harder to intentionally change these interfaces, there always must be an easy way to regenerate the expected output, as we do with `_NIX_TEST_ACCEPT=1`. +### Running functional tests on NixOS + +We run the functional tests not just in the build, but also in VM tests. +This helps us ensure that Nix works correctly on NixOS, and environments that have similar characteristics that are hard to reproduce in a build environment. + +The recommended way to run these tests during development is: + +```shell +nix build .#hydraJobs.tests.functional_user.quickBuild +``` + +The `quickBuild` attribute configures the test to use a `nix` package that's built without integration tests, so that you can iterate on the tests without performing recompilations due to the changed sources for `installCheck`. + +Generally, this build is sufficient, but in nightly or CI we also test the attributes `functional_root` and `functional_trusted`, in which the test suite is run with different levels of authorization. + ## Integration tests The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute. These tests include everything that needs to interact with external services or run Nix in a non-trivial distributed setup. Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on ). -You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}` +You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}`. + +If you are testing a build of `nix` that you haven't compiled yet, you may iterate faster by appending the `quickBuild` attribute: `nix build .#hydraJobs.tests.{testName}.quickBuild`. ## Installer tests diff --git a/doc/manual/src/favicon.png b/doc/manual/src/favicon.png new file mode 100644 index 000000000..1ed2b5fe0 Binary files /dev/null and b/doc/manual/src/favicon.png differ diff --git a/doc/manual/src/favicon.svg b/doc/manual/src/favicon.svg new file mode 100644 index 000000000..1d2a6e835 --- /dev/null +++ b/doc/manual/src/favicon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index a71b2e2b3..877c4668b 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -1,5 +1,24 @@ # Glossary +- [content address]{#gloss-content-address} + + A + [*content address*](https://en.wikipedia.org/wiki/Content-addressable_storage) + is a secure way to reference immutable data. + The reference is calculated directly from the content of the data being referenced, which means the reference is + [*tamper proof*](https://en.wikipedia.org/wiki/Tamperproofing) + --- variations of the data should always calculate to distinct content addresses. + + For how Nix uses content addresses, see: + + - [Content-Addressing File System Objects](@docroot@/store/file-system-object/content-address.md) + - [Content-Addressing Store Objects](@docroot@/store/store-object/content-address.md) + - [content-addressed derivation](#gloss-content-addressed-derivation) + + Software Heritage's writing on [*Intrinsic and Extrinsic identifiers*](https://www.softwareheritage.org/2020/07/09/intrinsic-vs-extrinsic-identifiers) is also a good introduction to the value of content-addressing over other referencing schemes. + + Besides content addressing, the Nix store also uses [input addressing](#gloss-input-addressed-store-object). + - [derivation]{#gloss-derivation} A description of a build task. The result of a derivation is a @@ -52,10 +71,9 @@ [`__contentAddressed`](./language/advanced-attributes.md#adv-attr-__contentAddressed) attribute set to `true`. -- [fixed-output derivation]{#gloss-fixed-output-derivation} +- [fixed-output derivation]{#gloss-fixed-output-derivation} (FOD) - A derivation which includes the - [`outputHash`](./language/advanced-attributes.md#adv-attr-outputHash) attribute. + A [derivation] where a cryptographic hash of the [output] is determined in advance using the [`outputHash`](./language/advanced-attributes.md#adv-attr-outputHash) attribute, and where the [`builder`](@docroot@/language/derivations.md#attr-builder) executable has access to the network. - [store]{#gloss-store} @@ -86,7 +104,7 @@ [store path]: #gloss-store-path -- [file system object]{#gloss-store-object} +- [file system object]{#gloss-file-system-object} The Nix data model for representing simplified file system data. @@ -118,9 +136,12 @@ - [content-addressed store object]{#gloss-content-addressed-store-object} - A [store object] whose [store path] is determined by its contents. + A [store object] which is [content-addressed](#gloss-content-address), + i.e. whose [store path] is determined by its contents. This includes derivations, the outputs of [content-addressed derivations](#gloss-content-addressed-derivation), and the outputs of [fixed-output derivations](#gloss-fixed-output-derivation). + See [Content-Addressing Store Objects](@docroot@/store/store-object/content-address.md) for details. + - [substitute]{#gloss-substitute} A substitute is a command invocation stored in the [Nix database] that @@ -147,7 +168,7 @@ - [impure derivation]{#gloss-impure-derivation} - [An experimental feature](#@docroot@/contributing/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, + [An experimental feature](#@docroot@/development/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, so that they are always rebuilt, and their outputs not reused by subsequent calls to realise them. - [Nix database]{#gloss-nix-database} @@ -215,6 +236,20 @@ [output path]: #gloss-output-path +- [output closure]{#gloss-output-closure}\ + The [closure] of an [output path]. It only contains what is [reachable] from the output. + +- [deriving path]{#gloss-deriving-path} + + Deriving paths are a way to refer to [store objects][store object] that ar not yet [realised][realise]. + This is necessary because, in general and particularly for [content-addressed derivations][content-addressed derivation], the [output path] of an [output] is not known in advance. + There are two forms: + + - *constant*: just a [store path] + It can be made [valid][validity] by copying it into the store: from the evaluator, command line interface or another store. + + - *output*: a pair of a [store path] to a [derivation] and an [output] name. + - [deriver]{#gloss-deriver} The [store derivation] that produced an [output path]. @@ -252,13 +287,15 @@ See [installables](./command-ref/new-cli/nix.md#installables) for [`nix` commands](./command-ref/new-cli/nix.md) (experimental) for details. -- [NAR]{#gloss-nar} +- [Nix Archive (NAR)]{#gloss-nar} A *N*ix *AR*chive. This is a serialisation of a path in the Nix store. It can contain regular files, directories and symbolic links. NARs are generated and unpacked using `nix-store --dump` and `nix-store --restore`. + See [Nix Archive](store/file-system-object/content-address.html#serial-nix-archive) for details. + - [`∅`]{#gloss-emtpy-set} The empty set symbol. In the context of profile history, this denotes a package is not present in a particular version of the profile. @@ -275,7 +312,7 @@ - [package attribute set]{#package-attribute-set} - An [attribute set](@docroot@/language/values.md#attribute-set) containing the attribute `type = "derivation";` (derivation for historical reasons), as well as other attributes, such as + An [attribute set](@docroot@/language/types.md#attribute-set) containing the attribute `type = "derivation";` (derivation for historical reasons), as well as other attributes, such as - attributes that refer to the files of a [package], typically in the form of [derivation outputs](#output), - attributes that declare something about how the package is supposed to be installed or used, - other metadata or arbitrary attributes. @@ -288,16 +325,35 @@ See [String interpolation](./language/string-interpolation.md) for details. - [string]: ./language/values.md#type-string - [path]: ./language/values.md#type-path - [attribute name]: ./language/values.md#attribute-set + [string]: ./language/types.md#type-string + [path]: ./language/types.md#type-path + [attribute name]: ./language/types.md#attribute-set + +- [base directory]{#gloss-base-directory} + + The location from which relative paths are resolved. + + - For expressions in a file, the base directory is the directory containing that file. + This is analogous to the directory of a [base URL](https://datatracker.ietf.org/doc/html/rfc1808#section-3.3). + + + + - For expressions written in command line arguments with [`--expr`](@docroot@/command-ref/opt-common.html#opt-expr), the base directory is the current working directory. + + [base directory]: #gloss-base-directory - [experimental feature]{#gloss-experimental-feature} Not yet stabilized functionality guarded by named experimental feature flags. These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting. - See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/contributing/experimental-features.md). + See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/development/experimental-features.md). [Nix language]: ./language/index.md diff --git a/doc/manual/src/installation/env-variables.md b/doc/manual/src/installation/env-variables.md index db98f52ff..035090421 100644 --- a/doc/manual/src/installation/env-variables.md +++ b/doc/manual/src/installation/env-variables.md @@ -53,7 +53,8 @@ ssl-cert-file = /etc/ssl/my-certificate-bundle.crt The Nix installer has special handling for these proxy-related environment variables: `http_proxy`, `https_proxy`, `ftp_proxy`, -`no_proxy`, `HTTP_PROXY`, `HTTPS_PROXY`, `FTP_PROXY`, `NO_PROXY`. +`all_proxy`, `no_proxy`, `HTTP_PROXY`, `HTTPS_PROXY`, `FTP_PROXY`, +`ALL_PROXY`, `NO_PROXY`. If any of these variables are set when running the Nix installer, then the installer will create an override file at diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index 0dc989159..6a168ff3d 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -50,7 +50,7 @@ Supported systems: To explicitly instruct the installer to perform a multi-user installation on your system: ```console -$ curl -L https://nixos.org/nix/install | sh -s -- --daemon +$ bash <(curl -L https://nixos.org/nix/install) --daemon ``` You can run this under your usual user account or `root`. @@ -61,7 +61,7 @@ The script will invoke `sudo` as needed. To explicitly select a single-user installation on your system: ```console -$ curl -L https://nixos.org/nix/install | sh -s -- --no-daemon +$ bash <(curl -L https://nixos.org/nix/install) --no-daemon ``` In a single-user installation, `/nix` is owned by the invoking user. @@ -77,7 +77,7 @@ $ su root # Installing from a binary tarball You can also download a binary tarball that contains Nix and all its dependencies: -- Choose a [version](https://releases.nixos.org/?prefix=nix/) and [system type](../contributing/hacking.md#platforms) +- Choose a [version](https://releases.nixos.org/?prefix=nix/) and [system type](../development/building.md#platforms) - Download and unpack the tarball - Run the installer diff --git a/doc/manual/src/installation/uninstall.md b/doc/manual/src/installation/uninstall.md index 9ead5e53c..590327fea 100644 --- a/doc/manual/src/installation/uninstall.md +++ b/doc/manual/src/installation/uninstall.md @@ -1,16 +1,8 @@ # Uninstalling Nix -## Single User - -If you have a [single-user installation](./installing-binary.md#single-user-installation) of Nix, uninstall it by running: - -```console -$ rm -rf /nix -``` - ## Multi User -Removing a [multi-user installation](./installing-binary.md#multi-user-installation) of Nix is more involved, and depends on the operating system. +Removing a [multi-user installation](./installing-binary.md#multi-user-installation) depends on the operating system. ### Linux @@ -51,7 +43,15 @@ which you may remove. ### macOS -1. Edit `/etc/zshrc`, `/etc/bashrc`, and `/etc/bash.bashrc` to remove the lines sourcing `nix-daemon.sh`, which should look like this: +1. If system-wide shell initialisation files haven't been altered since installing Nix, use the backups made by the installer: + + ```console + sudo mv /etc/zshrc.backup-before-nix /etc/zshrc + sudo mv /etc/bashrc.backup-before-nix /etc/bashrc + sudo mv /etc/bash.bashrc.backup-before-nix /etc/bash.bashrc + ``` + + Otherwise, edit `/etc/zshrc`, `/etc/bashrc`, and `/etc/bash.bashrc` to remove the lines sourcing `nix-daemon.sh`, which should look like this: ```bash # Nix @@ -61,18 +61,6 @@ which you may remove. # End Nix ``` - If these files haven't been altered since installing Nix you can simply put - the backups back in place: - - ```console - sudo mv /etc/zshrc.backup-before-nix /etc/zshrc - sudo mv /etc/bashrc.backup-before-nix /etc/bashrc - sudo mv /etc/bash.bashrc.backup-before-nix /etc/bash.bashrc - ``` - - This will stop shells from sourcing the file and bringing everything you - installed using Nix in scope. - 2. Stop and remove the Nix daemon services: ```console @@ -82,8 +70,7 @@ which you may remove. sudo rm /Library/LaunchDaemons/org.nixos.darwin-store.plist ``` - This stops the Nix daemon and prevents it from being started next time you - boot the system. + This stops the Nix daemon and prevents it from being started next time you boot the system. 3. Remove the `nixbld` group and the `_nixbuildN` users: @@ -94,25 +81,42 @@ which you may remove. This will remove all the build users that no longer serve a purpose. -4. Edit fstab using `sudo vifs` to remove the line mounting the Nix Store - volume on `/nix`, which looks like - `UUID= /nix apfs rw,noauto,nobrowse,suid,owners` or - `LABEL=Nix\040Store /nix apfs rw,nobrowse`. This will prevent automatic - mounting of the Nix Store volume. +4. Edit fstab using `sudo vifs` to remove the line mounting the Nix Store volume on `/nix`, which looks like -5. Edit `/etc/synthetic.conf` to remove the `nix` line. If this is the only - line in the file you can remove it entirely, `sudo rm /etc/synthetic.conf`. - This will prevent the creation of the empty `/nix` directory to provide a - mountpoint for the Nix Store volume. + ``` + UUID= /nix apfs rw,noauto,nobrowse,suid,owners + ``` + or -6. Remove the files Nix added to your system: + ``` + LABEL=Nix\040Store /nix apfs rw,nobrowse + ``` + + by setting the cursor on the respective line using the error keys, and pressing `dd`, and then `:wq` to save the file. + + This will prevent automatic mounting of the Nix Store volume. + +5. Edit `/etc/synthetic.conf` to remove the `nix` line. + If this is the only line in the file you can remove it entirely: + + ```bash + if [ -f /etc/synthetic.conf ]; then + if [ "$(cat /etc/synthetic.conf)" = "nix" ]; then + sudo rm /etc/synthetic.conf + else + sudo vi /etc/synthetic.conf + fi + fi + ``` + + This will prevent the creation of the empty `/nix` directory. + +6. Remove the files Nix added to your system, except for the store: ```console sudo rm -rf /etc/nix /var/root/.nix-profile /var/root/.nix-defexpr /var/root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels ``` - This gets rid of any data Nix may have created except for the store which is - removed next. 7. Remove the Nix Store volume: @@ -120,29 +124,32 @@ which you may remove. sudo diskutil apfs deleteVolume /nix ``` - This will remove the Nix Store volume and everything that was added to the - store. + This will remove the Nix Store volume and everything that was added to the store. - If the output indicates that the command couldn't remove the volume, you should - make sure you don't have an _unmounted_ Nix Store volume. Look for a - "Nix Store" volume in the output of the following command: + If the output indicates that the command couldn't remove the volume, you should make sure you don't have an _unmounted_ Nix Store volume. + Look for a "Nix Store" volume in the output of the following command: ```console diskutil list ``` - If you _do_ see a "Nix Store" volume, delete it by re-running the diskutil - deleteVolume command, but replace `/nix` with the store volume's `diskXsY` - identifier. + If you _do_ find a "Nix Store" volume, delete it by running `diskutil deleteVolume` with the store volume's `diskXsY` identifier. > **Note** > -> After you complete the steps here, you will still have an empty `/nix` -> directory. This is an expected sign of a successful uninstall. The empty -> `/nix` directory will disappear the next time you reboot. +> After you complete the steps here, you will still have an empty `/nix` directory. +> This is an expected sign of a successful uninstall. +> The empty `/nix` directory will disappear the next time you reboot. > -> You do not have to reboot to finish uninstalling Nix. The uninstall is -> complete. macOS (Catalina+) directly controls root directories and its -> read-only root will prevent you from manually deleting the empty `/nix` -> mountpoint. +> You do not have to reboot to finish uninstalling Nix. +> The uninstall is complete. +> macOS (Catalina+) directly controls root directories, and its read-only root will prevent you from manually deleting the empty `/nix` mountpoint. +## Single User + +To remove a [single-user installation](./installing-binary.md#single-user-installation) of Nix, run: + +```console +$ rm -rf /nix ~/.nix-channels ~/.nix-defexpr ~/.nix-profile +``` +You might also want to manually remove references to Nix from your `~/.profile`. diff --git a/doc/manual/src/installation/upgrading.md b/doc/manual/src/installation/upgrading.md index 38edcdbc5..a433f1d30 100644 --- a/doc/manual/src/installation/upgrading.md +++ b/doc/manual/src/installation/upgrading.md @@ -28,7 +28,7 @@ $ sudo su ## macOS multi-user ```console -$ sudo nix-env --install --file '' --attr nix -I nixpkgs=channel:nixpkgs-unstable +$ sudo nix-env --install --file '' --attr nix cacert -I nixpkgs=channel:nixpkgs-unstable $ sudo launchctl remove org.nixos.nix-daemon $ sudo launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist ``` diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 7306fc182..51b83fc8a 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -113,19 +113,18 @@ Derivations can declare some infrequently used optional attributes. > `nix-build`. If the [`configurable-impure-env` experimental - feature](@docroot@/contributing/experimental-features.md#xp-feature-configurable-impure-env) + feature](@docroot@/development/experimental-features.md#xp-feature-configurable-impure-env) is enabled, these environment variables can also be controlled through the [`impure-env`](@docroot@/command-ref/conf-file.md#conf-impure-env) configuration setting. - [`outputHash`]{#adv-attr-outputHash}; [`outputHashAlgo`]{#adv-attr-outputHashAlgo}; [`outputHashMode`]{#adv-attr-outputHashMode}\ - These attributes declare that the derivation is a so-called - *fixed-output derivation*, which means that a cryptographic hash of - the output is already known in advance. When the build of a - fixed-output derivation finishes, Nix computes the cryptographic - hash of the output and compares it to the hash declared with these - attributes. If there is a mismatch, the build fails. + These attributes declare that the derivation is a so-called *fixed-output derivation* (FOD), which means that a cryptographic hash of the output is already known in advance. + + As opposed to regular derivations, the [`builder`] executable of a fixed-output derivation has access to the network. + Nix computes a cryptographic hash of its output and compares that to the hash declared with these attributes. + If there is a mismatch, the derivation fails. The rationale for fixed-output derivations is derivations such as those produced by the `fetchurl` function. This function downloads a @@ -188,38 +187,49 @@ Derivations can declare some infrequently used optional attributes. } ``` - The `outputHashAlgo` attribute specifies the hash algorithm used to - compute the hash. It can currently be `"sha1"`, `"sha256"` or - `"sha512"`. + The `outputHash` attribute must be a string containing the hash in either hexadecimal or "nix32" encoding, or following the format for integrity metadata as defined by [SRI](https://www.w3.org/TR/SRI/). + The "nix32" encoding is an adaptation of base-32 encoding. + The [`convertHash`](@docroot@/language/builtins.md#builtins-convertHash) function shows how to convert between different encodings, and the [`nix-hash` command](../command-ref/nix-hash.md) has information about obtaining the hash for some contents, as well as converting to and from encodings. + + The `outputHashAlgo` attribute specifies the hash algorithm used to compute the hash. + It can currently be `"sha1"`, `"sha256"`, `"sha512"`, or `null`. + `outputHashAlgo` can only be `null` when `outputHash` follows the SRI format. The `outputHashMode` attribute determines how the hash is computed. - It must be one of the following two values: + It must be one of the following values: - - `"flat"`\ - The output must be a non-executable regular file. If it isn’t, - the build fails. The hash is simply computed over the contents - of that file (so it’s equal to what Unix commands like - `sha256sum` or `sha1sum` produce). + - [`"flat"`](@docroot@/store/store-object/content-address.md#method-flat) This is the default. - - `"recursive"`\ - The hash is computed over the NAR archive dump of the output - (i.e., the result of [`nix-store --dump`](@docroot@/command-ref/nix-store/dump.md)). In - this case, the output can be anything, including a directory - tree. + - [`"recursive"` or `"nar"`](@docroot@/store/store-object/content-address.md#method-nix-archive) - The `outputHash` attribute, finally, must be a string containing - the hash in either hexadecimal or base-32 notation. (See the - [`nix-hash` command](../command-ref/nix-hash.md) for information - about converting to and from base-32 notation.) + > **Compatibility** + > + > `"recursive"` is the traditional way of indicating this, + > and is supported since 2005 (virtually the entire history of Nix). + > `"nar"` is more clear, and consistent with other parts of Nix (such as the CLI), + > however support for it is only added in Nix version 2.21. + + - [`"text"`](@docroot@/store/store-object/content-address.md#method-text) + + > **Warning** + > + > The use of this method for derivation outputs is part of the [`dynamic-derivations`][xp-feature-dynamic-derivations] experimental feature. + + - [`"git"`](@docroot@/store/store-object/content-address.md#method-git) + + > **Warning** + > + > This method is part of the [`git-hashing`][xp-feature-git-hashing] experimental feature. - [`__contentAddressed`]{#adv-attr-__contentAddressed} + > **Warning** - > This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md). + > This attribute is part of an [experimental feature](@docroot@/development/experimental-features.md). > > To use this attribute, you must enable the - > [`ca-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-ca-derivations) experimental feature. + > [`ca-derivations`][xp-feature-ca-derivations] experimental feature. > For example, in [nix.conf](../command-ref/conf-file.md) you could add: > > ``` @@ -268,7 +278,9 @@ Derivations can declare some infrequently used optional attributes. > **Note** > - > If set to `false`, the [`builder`](./derivations.md#attr-builder) should be able to run on the system type specified in the [`system` attribute](./derivations.md#attr-system), since the derivation cannot be substituted. + > If set to `false`, the [`builder`] should be able to run on the system type specified in the [`system` attribute](./derivations.md#attr-system), since the derivation cannot be substituted. + + [`builder`]: ./derivations.md#attr-builder - [`__structuredAttrs`]{#adv-attr-structuredAttrs}\ If the special attribute `__structuredAttrs` is set to `true`, the other derivation @@ -290,6 +302,12 @@ Derivations can declare some infrequently used optional attributes. (associative) arrays. For example, the attribute `hardening.format = true` ends up as the Bash associative array element `${hardening[format]}`. + > **Warning** + > + > If set to `true`, other advanced attributes such as [`allowedReferences`](#adv-attr-allowedReferences), [`allowedReferences`](#adv-attr-allowedReferences), [`allowedRequisites`](#adv-attr-allowedRequisites), + [`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites), maxSize, and maxClosureSize. + will have no effect. + - [`outputChecks`]{#adv-attr-outputChecks}\ When using [structured attributes](#adv-attr-structuredAttrs), the `outputChecks` attribute allows defining checks per-output. @@ -299,7 +317,7 @@ Derivations can declare some infrequently used optional attributes. [`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites), the following attributes are available: - - `maxSize` defines the maximum size of the resulting [store object](../glossary.md#gloss-store-object). + - `maxSize` defines the maximum size of the resulting [store object](@docroot@/store/store-object.md). - `maxClosureSize` defines the maximum size of the output's closure. - `ignoreSelfRefs` controls whether self-references should be considered when checking for allowed references/requisites. @@ -351,3 +369,7 @@ Derivations can declare some infrequently used optional attributes. ``` ensures that the derivation can only be built on a machine with the `kvm` feature. + +[xp-feature-ca-derivations]: @docroot@/development/experimental-features.md#xp-feature-ca-derivations +[xp-feature-dynamic-derivations]: @docroot@/development/experimental-features.md#xp-feature-dynamic-derivations +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/doc/manual/src/language/builtin-constants-prefix.md b/doc/manual/src/language/builtin-constants-prefix.md deleted file mode 100644 index 50f43006d..000000000 --- a/doc/manual/src/language/builtin-constants-prefix.md +++ /dev/null @@ -1,5 +0,0 @@ -# Built-in Constants - -These constants are built into the Nix language evaluator: - - diff --git a/doc/manual/src/language/builtin-constants-suffix.md b/doc/manual/src/language/builtin-constants-suffix.md deleted file mode 100644 index a74db2857..000000000 --- a/doc/manual/src/language/builtin-constants-suffix.md +++ /dev/null @@ -1 +0,0 @@ - diff --git a/doc/manual/src/language/builtins-prefix.md b/doc/manual/src/language/builtins-prefix.md index 7b2321466..fb983bb7f 100644 --- a/doc/manual/src/language/builtins-prefix.md +++ b/doc/manual/src/language/builtins-prefix.md @@ -1,9 +1,11 @@ -# Built-in Functions +# Built-ins -This section lists the functions built into the Nix language evaluator. -All built-in functions are available through the global [`builtins`](./builtin-constants.md#builtins-builtins) constant. +This section lists the values and functions built into the Nix language evaluator. +All built-ins are available through the global [`builtins`](#builtins-builtins) constant. -For convenience, some built-ins can be accessed directly: +Some built-ins are also exposed directly in the global scope: + + - [`derivation`](#builtins-derivation) - [`import`](#builtins-import) diff --git a/doc/manual/src/language/constructs.md b/doc/manual/src/language/constructs.md deleted file mode 100644 index a82ec5960..000000000 --- a/doc/manual/src/language/constructs.md +++ /dev/null @@ -1,408 +0,0 @@ -# Language Constructs - -## Recursive sets - -Recursive sets are like normal [attribute sets](./values.md#attribute-set), but the attributes can refer to each other. - -> *rec-attrset* = `rec {` [ *name* `=` *expr* `;` `]`... `}` - -Example: - -```nix -rec { - x = y; - y = 123; -}.x -``` - -This evaluates to `123`. - -Note that without `rec` the binding `x = y;` would -refer to the variable `y` in the surrounding scope, if one exists, and -would be invalid if no such variable exists. That is, in a normal -(non-recursive) set, attributes are not added to the lexical scope; in a -recursive set, they are. - -Recursive sets of course introduce the danger of infinite recursion. For -example, the expression - -```nix -rec { - x = y; - y = x; -}.x -``` - -will crash with an `infinite recursion encountered` error message. - -## Let-expressions - -A let-expression allows you to define local variables for an expression. - -> *let-in* = `let` [ *identifier* = *expr* ]... `in` *expr* - -Example: - -```nix -let - x = "foo"; - y = "bar"; -in x + y -``` - -This evaluates to `"foobar"`. - -## Inheriting attributes - -When defining an [attribute set](./values.md#attribute-set) or in a [let-expression](#let-expressions) it is often convenient to copy variables from the surrounding lexical scope (e.g., when you want to propagate attributes). -This can be shortened using the `inherit` keyword. - -Example: - -```nix -let x = 123; in -{ - inherit x; - y = 456; -} -``` - -is equivalent to - -```nix -let x = 123; in -{ - x = x; - y = 456; -} -``` - -and both evaluate to `{ x = 123; y = 456; }`. - -> **Note** -> -> This works because `x` is added to the lexical scope by the `let` construct. - -It is also possible to inherit attributes from another attribute set. - -Example: - -In this fragment from `all-packages.nix`, - -```nix -graphviz = (import ../tools/graphics/graphviz) { - inherit fetchurl stdenv libpng libjpeg expat x11 yacc; - inherit (xorg) libXaw; -}; - -xorg = { - libX11 = ...; - libXaw = ...; - ... -} - -libpng = ...; -libjpg = ...; -... -``` - -the set used in the function call to the function defined in -`../tools/graphics/graphviz` inherits a number of variables from the -surrounding scope (`fetchurl` ... `yacc`), but also inherits `libXaw` -(the X Athena Widgets) from the `xorg` set. - -Summarizing the fragment - -```nix -... -inherit x y z; -inherit (src-set) a b c; -... -``` - -is equivalent to - -```nix -... -x = x; y = y; z = z; -a = src-set.a; b = src-set.b; c = src-set.c; -... -``` - -when used while defining local variables in a let-expression or while -defining a set. - -In a `let` expression, `inherit` can be used to selectively bring specific attributes of a set into scope. For example - - -```nix -let - x = { a = 1; b = 2; }; - inherit (builtins) attrNames; -in -{ - names = attrNames x; -} -``` - -is equivalent to - -```nix -let - x = { a = 1; b = 2; }; -in -{ - names = builtins.attrNames x; -} -``` - -both evaluate to `{ names = [ "a" "b" ]; }`. - -## Functions - -Functions have the following form: - -```nix -pattern: body -``` - -The pattern specifies what the argument of the function must look like, -and binds variables in the body to (parts of) the argument. There are -three kinds of patterns: - - - If a pattern is a single identifier, then the function matches any - argument. Example: - - ```nix - let negate = x: !x; - concat = x: y: x + y; - in if negate true then concat "foo" "bar" else "" - ``` - - Note that `concat` is a function that takes one argument and returns - a function that takes another argument. This allows partial - parameterisation (i.e., only filling some of the arguments of a - function); e.g., - - ```nix - map (concat "foo") [ "bar" "bla" "abc" ] - ``` - - evaluates to `[ "foobar" "foobla" "fooabc" ]`. - - - A *set pattern* of the form `{ name1, name2, …, nameN }` matches a - set containing the listed attributes, and binds the values of those - attributes to variables in the function body. For example, the - function - - ```nix - { x, y, z }: z + y + x - ``` - - can only be called with a set containing exactly the attributes `x`, - `y` and `z`. No other attributes are allowed. If you want to allow - additional arguments, you can use an ellipsis (`...`): - - ```nix - { x, y, z, ... }: z + y + x - ``` - - This works on any set that contains at least the three named - attributes. - - It is possible to provide *default values* for attributes, in - which case they are allowed to be missing. A default value is - specified by writing `name ? e`, where *e* is an arbitrary - expression. For example, - - ```nix - { x, y ? "foo", z ? "bar" }: z + y + x - ``` - - specifies a function that only requires an attribute named `x`, but - optionally accepts `y` and `z`. - - - An `@`-pattern provides a means of referring to the whole value - being matched: - - ```nix - args@{ x, y, z, ... }: z + y + x + args.a - ``` - - but can also be written as: - - ```nix - { x, y, z, ... } @ args: z + y + x + args.a - ``` - - Here `args` is bound to the argument *as passed*, which is further - matched against the pattern `{ x, y, z, ... }`. - The `@`-pattern makes mainly sense with an ellipsis(`...`) as - you can access attribute names as `a`, using `args.a`, which was - given as an additional attribute to the function. - - > **Warning** - > - > `args@` binds the name `args` to the attribute set that is passed to the function. - > In particular, `args` does *not* include any default values specified with `?` in the function's set pattern. - > - > For instance - > - > ```nix - > let - > f = args@{ a ? 23, ... }: [ a args ]; - > in - > f {} - > ``` - > - > is equivalent to - > - > ```nix - > let - > f = args @ { ... }: [ (args.a or 23) args ]; - > in - > f {} - > ``` - > - > and both expressions will evaluate to: - > - > ```nix - > [ 23 {} ] - > ``` - -Note that functions do not have names. If you want to give them a name, -you can bind them to an attribute, e.g., - -```nix -let concat = { x, y }: x + y; -in concat { x = "foo"; y = "bar"; } -``` - -## Conditionals - -Conditionals look like this: - -```nix -if e1 then e2 else e3 -``` - -where *e1* is an expression that should evaluate to a Boolean value -(`true` or `false`). - -## Assertions - -Assertions are generally used to check that certain requirements on or -between features and dependencies hold. They look like this: - -```nix -assert e1; e2 -``` - -where *e1* is an expression that should evaluate to a Boolean value. If -it evaluates to `true`, *e2* is returned; otherwise expression -evaluation is aborted and a backtrace is printed. - -Here is a Nix expression for the Subversion package that shows how -assertions can be used:. - -```nix -{ localServer ? false -, httpServer ? false -, sslSupport ? false -, pythonBindings ? false -, javaSwigBindings ? false -, javahlBindings ? false -, stdenv, fetchurl -, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null -}: - -assert localServer -> db4 != null; â‘ -assert httpServer -> httpd != null && httpd.expat == expat; â‘¡ -assert sslSupport -> openssl != null && (httpServer -> httpd.openssl == openssl); â‘¢ -assert pythonBindings -> swig != null && swig.pythonSupport; -assert javaSwigBindings -> swig != null && swig.javaSupport; -assert javahlBindings -> j2sdk != null; - -stdenv.mkDerivation { - name = "subversion-1.1.1"; - ... - openssl = if sslSupport then openssl else null; â‘£ - ... -} -``` - -The points of interest are: - -1. This assertion states that if Subversion is to have support for - local repositories, then Berkeley DB is needed. So if the Subversion - function is called with the `localServer` argument set to `true` but - the `db4` argument set to `null`, then the evaluation fails. - - Note that `->` is the [logical - implication](https://en.wikipedia.org/wiki/Truth_table#Logical_implication) - Boolean operation. - -2. This is a more subtle condition: if Subversion is built with Apache - (`httpServer`) support, then the Expat library (an XML library) used - by Subversion should be same as the one used by Apache. This is - because in this configuration Subversion code ends up being linked - with Apache code, and if the Expat libraries do not match, a build- - or runtime link error or incompatibility might occur. - -3. This assertion says that in order for Subversion to have SSL support - (so that it can access `https` URLs), an OpenSSL library must be - passed. Additionally, it says that *if* Apache support is enabled, - then Apache's OpenSSL should match Subversion's. (Note that if - Apache support is not enabled, we don't care about Apache's - OpenSSL.) - -4. The conditional here is not really related to assertions, but is - worth pointing out: it ensures that if SSL support is disabled, then - the Subversion derivation is not dependent on OpenSSL, even if a - non-`null` value was passed. This prevents an unnecessary rebuild of - Subversion if OpenSSL changes. - -## With-expressions - -A *with-expression*, - -```nix -with e1; e2 -``` - -introduces the set *e1* into the lexical scope of the expression *e2*. -For instance, - -```nix -let as = { x = "foo"; y = "bar"; }; -in with as; x + y -``` - -evaluates to `"foobar"` since the `with` adds the `x` and `y` attributes -of `as` to the lexical scope in the expression `x + y`. The most common -use of `with` is in conjunction with the `import` function. E.g., - -```nix -with (import ./definitions.nix); ... -``` - -makes all attributes defined in the file `definitions.nix` available as -if they were defined locally in a `let`-expression. - -The bindings introduced by `with` do not shadow bindings introduced by -other means, e.g. - -```nix -let a = 3; in with { a = 1; }; let a = 4; in with { a = 2; }; ... -``` - -establishes the same scope as - -```nix -let a = 1; in let a = 2; in let a = 3; in let a = 4; in ... -``` - -## Comments - -Comments can be single-line, started with a `#` character, or -inline/multi-line, enclosed within `/* ... */`. diff --git a/doc/manual/src/language/constructs/lookup-path.md b/doc/manual/src/language/constructs/lookup-path.md index e87d2922b..11b9fe88c 100644 --- a/doc/manual/src/language/constructs/lookup-path.md +++ b/doc/manual/src/language/constructs/lookup-path.md @@ -4,9 +4,9 @@ > > *lookup-path* = `<` *identifier* [ `/` *identifier* ]... `>` -A lookup path is an identifier with an optional path suffix that resolves to a [path value](@docroot@/language/values.md#type-path) if the identifier matches a search path entry. +A lookup path is an identifier with an optional path suffix that resolves to a [path value](@docroot@/language/types.md#type-path) if the identifier matches a search path entry. -The value of a lookup path is determined by [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath). +The value of a lookup path is determined by [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath). See [`builtins.findFile`](@docroot@/language/builtins.md#builtins-findFile) for details on lookup path resolution. diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index 75f824a34..8e3f0f791 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -12,12 +12,12 @@ It outputs an attribute set, and produces a [store derivation] as a side effect ### Required -- [`name`]{#attr-name} ([String](@docroot@/language/values.md#type-string)) +- [`name`]{#attr-name} ([String](@docroot@/language/types.md#type-string)) A symbolic name for the derivation. It is added to the [store path] of the corresponding [store derivation] as well as to its [output paths](@docroot@/glossary.md#gloss-output-path). - [store path]: @docroot@/glossary.md#gloss-store-path + [store path]: @docroot@/store/store-path.md > **Example** > @@ -31,7 +31,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect > The store derivation's path will be `/nix/store/-hello.drv`. > The [output](#attr-outputs) paths will be of the form `/nix/store/-hello[-]` -- [`system`]{#attr-system} ([String](@docroot@/language/values.md#type-string)) +- [`system`]{#attr-system} ([String](@docroot@/language/types.md#type-string)) The system type on which the [`builder`](#attr-builder) executable is meant to be run. @@ -64,9 +64,9 @@ It outputs an attribute set, and produces a [store derivation] as a side effect > } > ``` > - > [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) has the value of the [`system` configuration option], and defaults to the system type of the current Nix installation. + > [`builtins.currentSystem`](@docroot@/language/builtins.md#builtins-currentSystem) has the value of the [`system` configuration option], and defaults to the system type of the current Nix installation. -- [`builder`]{#attr-builder} ([Path](@docroot@/language/values.md#type-path) | [String](@docroot@/language/values.md#type-string)) +- [`builder`]{#attr-builder} ([Path](@docroot@/language/types.md#type-path) | [String](@docroot@/language/types.md#type-string)) Path to an executable that will perform the build. @@ -113,7 +113,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect ### Optional -- [`args`]{#attr-args} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) +- [`args`]{#attr-args} ([List](@docroot@/language/types.md#list) of [String](@docroot@/language/types.md#type-string)) Default: `[ ]` @@ -132,7 +132,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect > }; > ``` -- [`outputs`]{#attr-outputs} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) +- [`outputs`]{#attr-outputs} ([List](@docroot@/language/types.md#list) of [String](@docroot@/language/types.md#type-string)) Default: `[ "out" ]` @@ -141,7 +141,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect By default, a derivation produces a single output called `out`. However, derivations can produce multiple outputs. - This allows the associated [store objects](@docroot@/glossary.md#gloss-store-object) and their [closures](@docroot@/glossary.md#gloss-closure) to be copied or garbage-collected separately. + This allows the associated [store objects](@docroot@/store/store-object.md) and their [closures](@docroot@/glossary.md#gloss-closure) to be copied or garbage-collected separately. > **Example** > diff --git a/doc/manual/src/language/identifiers.md b/doc/manual/src/language/identifiers.md new file mode 100644 index 000000000..861ee3e20 --- /dev/null +++ b/doc/manual/src/language/identifiers.md @@ -0,0 +1,51 @@ +# Identifiers + +An *identifier* is an [ASCII](https://en.wikipedia.org/wiki/ASCII) character sequence that: +- Starts with a letter (`a-z`, `A-Z`) or underscore (`_`) +- Can contain any number of: + - Letters (`a-z`, `A-Z`) + - Digits (`0-9`) + - Underscores (`_`) + - Apostrophes (`'`) + - Hyphens (`-`) +- Is not one of the [keywords](#keywords) + +> **Syntax** +> +> *identifier* ~ `[A-Za-z_][A-Za-z0-9_'-]*` + +# Names + +A *name* can be written as an [identifier](#identifier) or a [string literal](./syntax.md#string-literal). + +> **Syntax** +> +> *name* → *identifier* | *string* + +Names are used in [attribute sets](./syntax.md#attrs-literal), [`let` bindings](./syntax.md#let-expressions), and [`inherit`](./syntax.md#inheriting-attributes). +Two names are the same if they represent the same sequence of characters, regardless of whether they are written as identifiers or strings. + +# Keywords + +These keywords are reserved and cannot be used as [identifiers](#identifiers): + +- [`assert`](./syntax.md#assertions) +- [`else`][if] +- [`if`][if] +- [`in`][let] +- [`inherit`](./syntax.md#inheriting-attributes) +- [`let`][let] +- [`or`](./operators.md#attribute-selection) (see note) +- [`rec`](./syntax.md#recursive-sets) +- [`then`][if] +- [`with`](./syntax.md#with-expressions) + +[if]: ./syntax.md#conditionals +[let]: ./syntax.md#let-expressions + +> **Note** +> +> The Nix language evaluator currently allows `or` to be used as a name in some contexts, for backwards compatibility reasons. +> Users are advised not to rely on this. +> +> There are long-standing issues with how `or` is parsed as a name, which can't be resolved without making a breaking change to the language. diff --git a/doc/manual/src/language/import-from-derivation.md b/doc/manual/src/language/import-from-derivation.md index fb12ba51a..e901f5bcf 100644 --- a/doc/manual/src/language/import-from-derivation.md +++ b/doc/manual/src/language/import-from-derivation.md @@ -2,9 +2,9 @@ The value of a Nix expression can depend on the contents of a [store object]. -[store object]: @docroot@/glossary.md#gloss-store-object +[store object]: @docroot@/store/store-object.md -Passing an expression `expr` that evaluates to a [store path](@docroot@/glossary.md#gloss-store-path) to any built-in function which reads from the filesystem constitutes Import From Derivation (IFD): +Passing an expression `expr` that evaluates to a [store path](@docroot@/store/store-path.md) to any built-in function which reads from the filesystem constitutes Import From Derivation (IFD): - [`import`](./builtins.md#builtins-import)` expr` - [`builtins.readFile`](./builtins.md#builtins-readFile)` expr` diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md index a26e43a05..2bfdbb8a0 100644 --- a/doc/manual/src/language/index.md +++ b/doc/manual/src/language/index.md @@ -1,7 +1,13 @@ # Nix Language The Nix language is designed for conveniently creating and composing *derivations* – precise descriptions of how contents of existing files are used to derive new files. -It is: + +> **Tip** +> +> These pages are written as a reference. +> If you are learning Nix, nix.dev has a good [introduction to the Nix language](https://nix.dev/tutorials/nix-language). + +The language is: - *domain-specific* @@ -47,7 +53,7 @@ This is an incomplete overview of language features, by example. - *Basic values* + *Basic values ([primitives](@docroot@/language/types.md#primitives))* @@ -65,7 +71,7 @@ This is an incomplete overview of language features, by example. - A string + A [string](@docroot@/language/types.md#type-string) @@ -88,6 +94,18 @@ This is an incomplete overview of language features, by example. + + + + `# Explanation` + + + + + A [comment](@docroot@/language/syntax.md#comments). + + + @@ -100,7 +118,7 @@ This is an incomplete overview of language features, by example. - String interpolation (expands to `"hello world"`, `"1 2 3"`, `"/nix/store/-bash-/bin/sh"`) + [String interpolation](@docroot@/language/string-interpolation.md) (expands to `"hello world"`, `"1 2 3"`, `"/nix/store/-bash-/bin/sh"`) @@ -112,7 +130,7 @@ This is an incomplete overview of language features, by example. - Booleans + [Booleans](@docroot@/language/types.md#type-boolean) @@ -124,7 +142,7 @@ This is an incomplete overview of language features, by example. - Null value + [Null](@docroot@/language/types.md#type-null) value @@ -136,7 +154,7 @@ This is an incomplete overview of language features, by example. - An integer + An [integer](@docroot@/language/types.md#type-int) @@ -148,7 +166,7 @@ This is an incomplete overview of language features, by example. - A floating point number + A [floating point number](@docroot@/language/types.md#type-float) @@ -160,7 +178,7 @@ This is an incomplete overview of language features, by example. - An absolute path + An absolute [path](@docroot@/language/types.md#type-path) @@ -172,7 +190,7 @@ This is an incomplete overview of language features, by example. - A path relative to the file containing this Nix expression + A [path](@docroot@/language/types.md#type-path) relative to the file containing this Nix expression @@ -184,7 +202,7 @@ This is an incomplete overview of language features, by example. - A home path. Evaluates to the `"/.config"`. + A home [path](@docroot@/language/types.md#type-path). Evaluates to the `"/.config"`. @@ -196,7 +214,7 @@ This is an incomplete overview of language features, by example. - Search path for Nix files. Value determined by [`$NIX_PATH` environment variable](../command-ref/env-common.md#env-NIX_PATH). + A [lookup path](@docroot@/language/constructs/lookup-path.md) for Nix files. Value determined by [`$NIX_PATH` environment variable](../command-ref/env-common.md#env-NIX_PATH). @@ -220,7 +238,7 @@ This is an incomplete overview of language features, by example. - A set with attributes named `x` and `y` + An [attribute set](@docroot@/language/types.md#attribute-set) with attributes named `x` and `y` @@ -244,7 +262,7 @@ This is an incomplete overview of language features, by example. - A recursive set, equivalent to `{ x = "foo"; y = "foobar"; }` + A [recursive set](@docroot@/language/syntax.md#recursive-sets), equivalent to `{ x = "foo"; y = "foobar"; }`. @@ -260,7 +278,7 @@ This is an incomplete overview of language features, by example. - Lists with three elements. + [Lists](@docroot@/language/types.md#list) with three elements. @@ -344,7 +362,7 @@ This is an incomplete overview of language features, by example. - Attribute selection (evaluates to `1`) + [Attribute selection](@docroot@/language/types.md#attribute-set) (evaluates to `1`) @@ -356,7 +374,7 @@ This is an incomplete overview of language features, by example. - Attribute selection with default (evaluates to `3`) + [Attribute selection](@docroot@/language/types.md#attribute-set) with default (evaluates to `3`) @@ -392,7 +410,7 @@ This is an incomplete overview of language features, by example. - Conditional expression + [Conditional expression](@docroot@/language/syntax.md#conditionals). @@ -404,7 +422,7 @@ This is an incomplete overview of language features, by example. - Assertion check (evaluates to `"yes!"`). + [Assertion](@docroot@/language/syntax.md#assertions) check (evaluates to `"yes!"`). @@ -416,7 +434,7 @@ This is an incomplete overview of language features, by example. - Variable definition + Variable definition. See [`let`-expressions](@docroot@/language/syntax.md#let-expressions). @@ -428,14 +446,44 @@ This is an incomplete overview of language features, by example. - Add all attributes from the given set to the scope (evaluates to `1`) + Add all attributes from the given set to the scope (evaluates to `1`). + + See [`with`-expressions](@docroot@/language/syntax.md#with-expressions) for details and shadowing caveats. - *Functions (lambdas)* + `inherit pkgs src;` + + + + + Adds the variables to the current scope (attribute set or `let` binding). + Desugars to `pkgs = pkgs; src = src;`. + See [Inheriting attributes](@docroot@/language/syntax.md#inheriting-attributes). + + + + + + + `inherit (pkgs) lib stdenv;` + + + + + Adds the attributes, from the attribute set in parentheses, to the current scope (attribute set or `let` binding). + Desugars to `lib = pkgs.lib; stdenv = pkgs.stdenv;`. + See [Inheriting attributes](@docroot@/language/syntax.md#inheriting-attributes). + + + + + + + *[Functions](@docroot@/language/syntax.md#functions) (lambdas)* @@ -452,7 +500,7 @@ This is an incomplete overview of language features, by example. - A function that expects an integer and returns it increased by 1 + A [function](@docroot@/language/syntax.md#functions) that expects an integer and returns it increased by 1. @@ -464,7 +512,7 @@ This is an incomplete overview of language features, by example. - Curried function, equivalent to `x: (y: x + y)`. Can be used like a function that takes two arguments and returns their sum. + Curried [function](@docroot@/language/syntax.md#functions), equivalent to `x: (y: x + y)`. Can be used like a function that takes two arguments and returns their sum. @@ -476,7 +524,7 @@ This is an incomplete overview of language features, by example. - A function call (evaluates to 101) + A [function](@docroot@/language/syntax.md#functions) call (evaluates to 101) @@ -488,7 +536,7 @@ This is an incomplete overview of language features, by example. - A function bound to a variable and subsequently called by name (evaluates to 103) + A [function](@docroot@/language/syntax.md#functions) bound to a variable and subsequently called by name (evaluates to 103) @@ -500,7 +548,7 @@ This is an incomplete overview of language features, by example. - A function that expects a set with required attributes `x` and `y` and concatenates them + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attributes `x` and `y` and concatenates them @@ -512,7 +560,7 @@ This is an incomplete overview of language features, by example. - A function that expects a set with required attribute `x` and optional `y`, using `"bar"` as default value for `y` + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attribute `x` and optional `y`, using `"bar"` as default value for `y` @@ -524,7 +572,7 @@ This is an incomplete overview of language features, by example. - A function that expects a set with required attributes `x` and `y` and ignores any other attributes + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attributes `x` and `y` and ignores any other attributes @@ -538,7 +586,7 @@ This is an incomplete overview of language features, by example. - A function that expects a set with required attributes `x` and `y`, and binds the whole set to `args` + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attributes `x` and `y`, and binds the whole set to `args` @@ -562,7 +610,8 @@ This is an incomplete overview of language features, by example. - Load and return Nix expression in given file + Load and return Nix expression in given file. + See [import](@docroot@/language/builtins.md#builtins-import). @@ -574,7 +623,8 @@ This is an incomplete overview of language features, by example. - Apply a function to every element of a list (evaluates to `[ 2 4 6 ]`) + Apply a function to every element of a list (evaluates to `[ 2 4 6 ]`). + See [`map`](@docroot@/language/builtins.md#builtins-map). diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index 6fd66864b..e1c020781 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -26,12 +26,16 @@ | Logical conjunction (`AND`) | *bool* `&&` *bool* | left | 12 | | Logical disjunction (`OR`) | *bool* \|\| *bool* | left | 13 | | [Logical implication] | *bool* `->` *bool* | right | 14 | +| [Pipe operator] (experimental) | *expr* `\|>` *func* | left | 15 | +| [Pipe operator] (experimental) | *func* `<\|` *expr* | right | 15 | -[string]: ./values.md#type-string -[path]: ./values.md#type-path -[number]: ./values.md#type-number -[list]: ./values.md#list -[attribute set]: ./values.md#attribute-set +[string]: ./types.md#type-string +[path]: ./types.md#type-path +[number]: ./types.md#type-float +[list]: ./types.md#list +[attribute set]: ./types.md#attribute-set + + ## Attribute selection @@ -42,12 +46,6 @@ Select the attribute denoted by attribute path *attrpath* from [attribute set] *attrset*. If the attribute doesn’t exist, return the *expr* after `or` if provided, otherwise abort evaluation. -An attribute path is a dot-separated list of [attribute names](./values.md#attribute-set). - -> **Syntax** -> -> *attrpath* = *name* [ `.` *name* ]... - [Attribute selection]: #attribute-selection ## Has attribute @@ -61,7 +59,7 @@ The result is a [Boolean] value. See also: [`builtins.hasAttr`](@docroot@/language/builtins.md#builtins-hasAttr) -[Boolean]: ./values.md#type-boolean +[Boolean]: ./types.md#type-boolean [Has attribute]: #has-attribute @@ -128,8 +126,8 @@ The result is a string. > The file or directory at *path* must exist and is copied to the [store]. > The path appears in the result as the corresponding [store path]. -[store path]: ../glossary.md#gloss-store-path -[store]: ../glossary.md#gloss-store +[store path]: @docroot@/store/store-path.md +[store]: @docroot@/glossary.md#gloss-store [String and path concatenation]: #string-and-path-concatenation @@ -141,7 +139,7 @@ The result is a string. Update [attribute set] *attrset1* with names and values from *attrset2*. -The returned attribute set will have of all the attributes in *attrset1* and *attrset2*. +The returned attribute set will have all of the attributes in *attrset1* and *attrset2*. If an attribute name is present in both, the attribute value from the latter is taken. [Update]: #update @@ -172,7 +170,7 @@ All comparison operators are implemented in terms of `<`, and the following equi - Numbers are type-compatible, see [arithmetic] operators. - Floating point numbers only differ up to a limited precision. -[function]: ./constructs.md#functions +[function]: ./syntax.md#functions [Equality]: #equality @@ -182,3 +180,34 @@ Equivalent to `!`*b1* `||` *b2*. [Logical implication]: #logical-implication +## Pipe operators + +- *a* `|>` *b* is equivalent to *b* *a* +- *a* `<|` *b* is equivalent to *a* *b* + +> **Example** +> +> ``` +> nix-repl> 1 |> builtins.add 2 |> builtins.mul 3 +> 9 +> +> nix-repl> builtins.add 1 <| builtins.mul 2 <| 3 +> 7 +> ``` + +> **Warning** +> +> This syntax is part of an +> [experimental feature](@docroot@/development/experimental-features.md) +> and may change in future releases. +> +> To use this syntax, make sure the +> [`pipe-operators` experimental feature](@docroot@/development/experimental-features.md#xp-feature-pipe-operators) +> is enabled. +> For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): +> +> ``` +> extra-experimental-features = pipe-operators +> ``` + +[Pipe operator]: #pipe-operators diff --git a/doc/manual/src/language/scope.md b/doc/manual/src/language/scope.md new file mode 100644 index 000000000..9373324e2 --- /dev/null +++ b/doc/manual/src/language/scope.md @@ -0,0 +1,28 @@ +# Scoping rules + +A *scope* in the Nix language is a dictionary keyed by [name](./identifiers.md#names), mapping each name to an expression and a *definition type*. +The definition type is either *explicit* or *implicit*. +Each entry in this dictionary is a *definition*. + +Explicit definitions are created by the following expressions: +- [let-expressions](syntax.md#let-expressions) +- [recursive attribute set literals](syntax.md#recursive-sets) (`rec`) +- [function literals](syntax.md#functions) + +Implicit definitions are only created by [with-expressions](./syntax.md#with-expressions). + +Every expression is *enclosed* by a scope. +The outermost expression is enclosed by the [built-in, global scope](./builtins.md), which contains only explicit definitions. +The expressions listed above *extend* their enclosing scope by adding new definitions, or replacing existing ones with the same name. +An explicit definition can replace a definition of any type; an implicit definition can only replace another implicit definition. + +Each of the above expressions defines which of its subexpressions are enclosed by the extended scope. +In all other cases, the same scope that encloses an expression is the enclosing scope for its subexpressions. + +The Nix language is [statically scoped](https://en.wikipedia.org/wiki/Scope_(computer_science)#Lexical_scope); +the value of a variable is determined only by the variable's enclosing scope, and not by the dynamic context in which the variable is evaluated. + +> **Note** +> +> Expressions entered into the [Nix REPL](@docroot@/command-ref/new-cli/nix3-repl.md) are enclosed by a scope that can be extended by command line arguments or previous REPL commands. +> These ways of extending scope are not, strictly speaking, part of the Nix language. diff --git a/doc/manual/src/language/string-context.md b/doc/manual/src/language/string-context.md new file mode 100644 index 000000000..6a3482cfd --- /dev/null +++ b/doc/manual/src/language/string-context.md @@ -0,0 +1,134 @@ +# String context + +> **Note** +> +> This is an advanced topic. +> The Nix language is designed to be used without the programmer consciously dealing with string contexts or even knowing what they are. + +A string in the Nix language is not just a sequence of characters like strings in other languages. +It is actually a pair of a sequence of characters and a *string context*. +The string context is an (unordered) set of *string context elements*. + +The purpose of string contexts is to collect non-string values attached to strings via +[string concatenation](./operators.md#string-concatenation), +[string interpolation](./string-interpolation.md), +and similar operations. +The idea is that a user can combine together values to create a build instructions for derivations without manually keeping track of where they come from. +Then the Nix language implicitly does that bookkeeping to efficiently obtain the closure of derivation inputs. + +> **Note** +> +> String contexts are *not* explicitly manipulated in idiomatic Nix language code. + +String context elements come in different forms: + +- [deriving path]{#string-context-element-derived-path} + + A string context element of this type is a [deriving path](@docroot@/glossary.md#gloss-deriving-path). + They can be either of type [constant](#string-context-constant) or [output](#string-context-output), which correspond to the types of deriving paths. + + - [Constant string context elements]{#string-context-constant} + + > **Example** + > + > [`builtins.storePath`] creates a string with a single constant string context element: + > + > ```nix + > builtins.getContext (builtins.storePath "/nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10") + > ``` + > evaluates to + > ```nix + > { + > "/nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10" = { + > path = true; + > }; + > } + > ``` + + [deriving path]: @docroot@/glossary.md#gloss-deriving-path + [store path]: @docroot@/glossary.md#gloss-store-path + [`builtins.storePath`]: ./builtins.md#builtins-storePath + + - [Output string context elements]{#string-context-output} + + > **Example** + > + > The behavior of string contexts are best demonstrated with a built-in function that is still experimental: [`builtins.outputOf`]. + > This example will *not* work with stable Nix! + > + > ```nix + > builtins.getContext + > (builtins.outputOf + > (builtins.storePath "/nix/store/fvchh9cvcr7kdla6n860hshchsba305w-hello-2.12.drv") + > "out") + > ``` + > evaluates to + > ```nix + > { + > "/nix/store/fvchh9cvcr7kdla6n860hshchsba305w-hello-2.12.drv" = { + > outputs = [ "out" ]; + > }; + > } + > ``` + + [`builtins.outputOf`]: ./builtins.md#builtins-outputOf + +- [*derivation deep*]{#string-context-element-derivation-deep} + + *derivation deep* is an advanced feature intended to be used with the + [`exportReferencesGraph` derivation attribute](./advanced-attributes.html#adv-attr-exportReferencesGraph). + A *derivation deep* string context element is a derivation path, and refers to both its outputs and the entire build closure of that derivation: + all its outputs, all the other derivations the given derivation depends on, and all the outputs of those. + + > **Example** + > + > The best way to illustrate *derivation deep* string contexts is with [`builtins.addDrvOutputDependencies`]. + > Take a regular constant string context element pointing to a derivation, and transform it into a "Derivation deep" string context element. + > + > ```nix + > builtins.getContext + > (builtins.addDrvOutputDependencies + > (builtins.storePath "/nix/store/fvchh9cvcr7kdla6n860hshchsba305w-hello-2.12.drv")) + > ``` + > evaluates to + > ```nix + > { + > "/nix/store/fvchh9cvcr7kdla6n860hshchsba305w-hello-2.12.drv" = { + > allOutputs = true; + > }; + > } + > ``` + + [`builtins.addDrvOutputDependencies`]: ./builtins.md#builtins-addDrvOutputDependencies + [`builtins.unsafeDiscardOutputDependency`]: ./builtins.md#builtins-unsafeDiscardOutputDependency + +## Inspecting string contexts + +Most basically, [`builtins.hasContext`] will tell whether a string has a non-empty context. + +When more granular information is needed, [`builtins.getContext`] can be used. +It creates an [attribute set] representing the string context, which can be inspected as usual. + +[`builtins.hasContext`]: ./builtins.md#builtins-hasContext +[`builtins.getContext`]: ./builtins.md#builtins-getContext +[attribute set]: ./types.md#attribute-set + +## Clearing string contexts + +[`buitins.unsafeDiscardStringContext`](./builtins.md#builtins-unsafeDiscardStringContext) will make a copy of a string, but with an empty string context. +The returned string can be used in more ways, e.g. by operators that require the string context to be empty. +The requirement to explicitly discard the string context in such use cases helps ensure that string context elements are not lost by mistake. +The "unsafe" marker is only there to remind that Nix normally guarantees that dependencies are tracked, whereas the returned string has lost them. + +## Constructing string contexts + +[`builtins.appendContext`] will create a copy of a string, but with additional string context elements. +The context is specified explicitly by an [attribute set] in the format that [`builtins.hasContext`] produces. +A string with arbitrary contexts can be made like this: + +1. Create a string with the desired string context elements. + (The contents of the string do not matter.) +2. Dump its context with [`builtins.getContext`]. +3. Combine it with a base string and repeated [`builtins.appendContext`] calls. + +[`builtins.appendContext`]: ./builtins.md#builtins-appendContext diff --git a/doc/manual/src/language/string-interpolation.md b/doc/manual/src/language/string-interpolation.md index 7d81c2020..1778bdfa0 100644 --- a/doc/manual/src/language/string-interpolation.md +++ b/doc/manual/src/language/string-interpolation.md @@ -4,9 +4,9 @@ String interpolation is a language feature where a [string], [path], or [attribu Such a construct is called *interpolated string*, and the expression inside is an [interpolated expression](#interpolated-expression). -[string]: ./values.md#type-string -[path]: ./values.md#type-path -[attribute set]: ./values.md#attribute-set +[string]: ./types.md#type-string +[path]: ./types.md#type-path +[attribute set]: ./types.md#attribute-set ## Examples @@ -20,7 +20,7 @@ Rather than writing (where `freetype` is a [derivation]), you can instead write -[derivation]: ../glossary.md#gloss-derivation +[derivation]: @docroot@/glossary.md#gloss-derivation ```nix "--with-freetype2-library=${freetype}/lib" @@ -43,6 +43,47 @@ configureFlags = " Note that Nix expressions and strings can be arbitrarily nested; in this case the outer string contains various interpolated expressions that themselves contain strings (e.g., `"-thread"`), some of which in turn contain interpolated expressions (e.g., `${mesa}`). +To write a literal `${` in an regular string, escape it with a backslash (`\`). + +> **Example** +> +> ```nix +> "echo \${PATH}" +> ``` +> +> "echo ${PATH}" + +To write a literal `${` in an indented string, escape it with two single quotes (`''`). + +> **Example** +> +> ```nix +> '' +> echo ''${PATH} +> '' +> ``` +> +> "echo ${PATH}\n" + +`$${` can be written literally in any string. + +> **Example** +> +> In Make, `$` in file names or recipes is represented as `$$`, see [GNU `make`: Basics of Variable Reference](https://www.gnu.org/software/make/manual/html_node/Reference.html#Basics-of-Variable-References). +> This can be expressed directly in the Nix language strings: +> +> ```nix +> '' +> MAKEVAR = Hello +> all: +> @export BASHVAR=world; echo $(MAKEVAR) $${BASHVAR} +> '' +> ``` +> +> "MAKEVAR = Hello\nall:\n\t@export BASHVAR=world; echo $(MAKEVAR) $\${BASHVAR}\n" + +See the [documentation on strings][string] for details. + ### Path Rather than writing @@ -107,9 +148,9 @@ An expression that is interpolated must evaluate to one of the following: A string interpolates to itself. -A path in an interpolated expression is first copied into the Nix store, and the resulting string is the [store path] of the newly created [store object](../glossary.md#gloss-store-object). +A path in an interpolated expression is first copied into the Nix store, and the resulting string is the [store path] of the newly created [store object](@docroot@/store/store-object.md). -[store path]: ../glossary.md#gloss-store-path +[store path]: @docroot@/store/store-path.md > **Example** > diff --git a/doc/manual/src/language/syntax.md b/doc/manual/src/language/syntax.md new file mode 100644 index 000000000..6108bacd6 --- /dev/null +++ b/doc/manual/src/language/syntax.md @@ -0,0 +1,866 @@ +# Language Constructs + +This section covers syntax and semantics of the Nix language. + +## Basic Literals + +### String {#string-literal} + + *Strings* can be written in three ways. + + The most common way is to enclose the string between double quotes, e.g., `"foo bar"`. + Strings can span multiple lines. + The results of other expressions can be included into a string by enclosing them in `${ }`, a feature known as [string interpolation]. + + [string interpolation]: ./string-interpolation.md + + The following must be escaped to represent them within a string, by prefixing with a backslash (`\`): + + - Double quote (`"`) + + > **Example** + > + > ```nix + > "\"" + > ``` + > + > "\"" + + - Backslash (`\`) + + > **Example** + > + > ```nix + > "\\" + > ``` + > + > "\\" + + - Dollar sign followed by an opening curly bracket (`${`) – "dollar-curly" + + > **Example** + > + > ```nix + > "\${" + > ``` + > + > "\${" + + The newline, carriage return, and tab characters can be written as `\n`, `\r` and `\t`, respectively. + + A "double-dollar-curly" (`$${`) can be written literally. + + > **Example** + > + > ```nix + > "$${" + > ``` + > + > "$\${" + + String values are output on the terminal with Nix-specific escaping. + Strings written to files will contain the characters encoded by the escaping. + + The second way to write string literals is as an *indented string*, which is enclosed between pairs of *double single-quotes* (`''`), like so: + + ```nix + '' + This is the first line. + This is the second line. + This is the third line. + '' + ``` + + This kind of string literal intelligently strips indentation from + the start of each line. To be precise, it strips from each line a + number of spaces equal to the minimal indentation of the string as a + whole (disregarding the indentation of empty lines). For instance, + the first and second line are indented two spaces, while the third + line is indented four spaces. Thus, two spaces are stripped from + each line, so the resulting string is + + ```nix + "This is the first line.\nThis is the second line.\n This is the third line.\n" + ``` + + > **Note** + > + > Whitespace and newline following the opening `''` is ignored if there is no non-whitespace text on the initial line. + + > **Warning** + > + > Prefixed tab characters are not stripped. + > + > > **Example** + > > + > > The following indented string is prefixed with tabs: + > > + > > '' + > > all: + > > @echo hello + > > '' + > > + > > "\tall:\n\t\t@echo hello\n" + + Indented strings support [string interpolation]. + + The following must be escaped to represent them in an indented string: + + - `$` is escaped by prefixing it with two single quotes (`''`) + + > **Example** + > + > ```nix + > '' + > ''$ + > '' + > ``` + > + > "$\n" + + - `''` is escaped by prefixing it with one single quote (`'`) + + > **Example** + > + > ```nix + > '' + > ''' + > '' + > ``` + > + > "''\n" + + These special characters are escaped as follows: + - Linefeed (`\n`): `''\n` + - Carriage return (`\r`): `''\r` + - Tab (`\t`): `''\t` + + `''\` escapes any other character. + + A "double-dollar-curly" (`$${`) can be written literally. + + > **Example** + > + > ```nix + > '' + > $${ + > '' + > ``` + > + > "$\${\n" + + Indented strings are primarily useful in that they allow multi-line + string literals to follow the indentation of the enclosing Nix + expression, and that less escaping is typically necessary for + strings representing languages such as shell scripts and + configuration files because `''` is much less common than `"`. + Example: + + ```nix + stdenv.mkDerivation { + ... + postInstall = + '' + mkdir $out/bin $out/etc + cp foo $out/bin + echo "Hello World" > $out/etc/foo.conf + ${if enableBar then "cp bar $out/bin" else ""} + ''; + ... + } + ``` + + Finally, as a convenience, *URIs* as defined in appendix B of + [RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as + is*, without quotes. For instance, the string + `"http://example.org/foo.tar.bz2"` can also be written as + `http://example.org/foo.tar.bz2`. + +### Number {#number-literal} + + + + Numbers, which can be *integers* (like `123`) or *floating point* + (like `123.43` or `.27e13`). + + See [arithmetic] and [comparison] operators for semantics. + + [arithmetic]: ./operators.md#arithmetic + [comparison]: ./operators.md#comparison + +### Path {#path-literal} + + *Paths* can be expressed by path literals such as `./builder.sh`. + + A path literal must contain at least one slash to be recognised as such. + For instance, `builder.sh` is not a path: + it's parsed as an expression that selects the attribute `sh` from the variable `builder`. + + Path literals are resolved relative to their [base directory](@docroot@/glossary.md#gloss-base-directory). + Path literals may also refer to absolute paths by starting with a slash. + + > **Note** + > + > Absolute paths make expressions less portable. + > In the case where a function translates a path literal into an absolute path string for a configuration file, it is recommended to write a string literal instead. + > This avoids some confusion about whether files at that location will be used during evaluation. + > It also avoids unintentional situations where some function might try to copy everything at the location into the store. + + If the first component of a path is a `~`, it is interpreted such that the rest of the path were relative to the user's home directory. + For example, `~/foo` would be equivalent to `/home/edolstra/foo` for a user whose home directory is `/home/edolstra`. + Path literals that start with `~` are not allowed in [pure](@docroot@/command-ref/conf-file.md#conf-pure-eval) evaluation. + + Path literals can also include [string interpolation], besides being [interpolated into other expressions]. + + [interpolated into other expressions]: ./string-interpolation.md#interpolated-expressions + + At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path. + + `a.${foo}/b.${bar}` is a syntactically valid number division operation. + `./a.${foo}/b.${bar}` is a path. + + [Lookup path](./constructs/lookup-path.md) literals such as `` also resolve to path values. + +## List {#list-literal} + +Lists are formed by enclosing a whitespace-separated list of values +between square brackets. For example, + +```nix +[ 123 ./foo.nix "abc" (f { x = y; }) ] +``` + +defines a list of four elements, the last being the result of a call to +the function `f`. Note that function calls have to be enclosed in +parentheses. If they had been omitted, e.g., + +```nix +[ 123 ./foo.nix "abc" f { x = y; } ] +``` + +the result would be a list of five elements, the fourth one being a +function and the fifth being a set. + +Note that lists are only lazy in values, and they are strict in length. + +Elements in a list can be accessed using [`builtins.elemAt`](./builtins.md#builtins-elemAt). + +## Attribute Set {#attrs-literal} + +An attribute set is a collection of name-value-pairs called *attributes*. + +Attribute sets are written enclosed in curly brackets (`{ }`). +Attribute names and attribute values are separated by an equal sign (`=`). +Each value can be an arbitrary expression, terminated by a semicolon (`;`) + +An attribute name is a string without context, and is denoted by a [name] (an [identifier](./identifiers.md#identifiers) or [string literal](#string-literal)). + +[name]: ./identifiers.md#names + +> **Syntax** +> +> *attrset* → `{` { *name* `=` *expr* `;` } `}` + +Attributes can appear in any order. +An attribute name may only occur once in each attribute set. + +> **Example** +> +> This defines an attribute set with attributes named: +> - `x` with the value `123`, an integer +> - `text` with the value `"Hello"`, a string +> - `y` where the value is the result of applying the function `f` to the attribute set `{ bla = 456; }` +> +> ```nix +> { +> x = 123; +> text = "Hello"; +> y = f { bla = 456; }; +> } +> ``` + +Attributes in nested attribute sets can be written using *attribute paths*. + +> **Syntax** +> +> *attrset* → `{` { *attrpath* `=` *expr* `;` } `}` + +An attribute path is a dot-separated list of [names][name]. + +> **Syntax** +> +> *attrpath* = *name* { `.` *name* } + + + +> **Example** +> +> ```nix +> { a.b.c = 1; a.b.d = 2; } +> ``` +> +> { +> a = { +> b = { +> c = 1; +> d = 2; +> }; +> }; +> } + +Attribute names can also be set implicitly by using the [`inherit` keyword](#inheriting-attributes). + +> **Example** +> +> ```nix +> { inherit (builtins) true; } +> ``` +> +> { true = true; } + +Attributes can be accessed with the [`.` operator](./operators.md#attribute-selection). + +Example: + +```nix +{ a = "Foo"; b = "Bar"; }.a +``` + +This evaluates to `"Foo"`. + +It is possible to provide a default value in an attribute selection using the `or` keyword. + +Example: + +```nix +{ a = "Foo"; b = "Bar"; }.c or "Xyzzy" +``` + +```nix +{ a = "Foo"; b = "Bar"; }.c.d.e.f.g or "Xyzzy" +``` + +will both evaluate to `"Xyzzy"` because there is no `c` attribute in the set. + +You can use arbitrary double-quoted strings as attribute names: + +```nix +{ "$!@#?" = 123; }."$!@#?" +``` + +```nix +let bar = "bar"; in +{ "foo ${bar}" = 123; }."foo ${bar}" +``` + +Both will evaluate to `123`. + +Attribute names support [string interpolation]: + +```nix +let bar = "foo"; in +{ foo = 123; }.${bar} +``` + +```nix +let bar = "foo"; in +{ ${bar} = 123; }.foo +``` + +Both will evaluate to `123`. + +In the special case where an attribute name inside of a set declaration +evaluates to `null` (which is normally an error, as `null` cannot be coerced to +a string), that attribute is simply not added to the set: + +```nix +{ ${if foo then "bar" else null} = true; } +``` + +This will evaluate to `{}` if `foo` evaluates to `false`. + +A set that has a `__functor` attribute whose value is callable (i.e. is +itself a function or a set with a `__functor` attribute whose value is +callable) can be applied as if it were a function, with the set itself +passed in first , e.g., + +```nix +let add = { __functor = self: x: x + self.x; }; + inc = add // { x = 1; }; +in inc 1 +``` + +evaluates to `2`. This can be used to attach metadata to a function +without the caller needing to treat it specially, or to implement a form +of object-oriented programming, for example. + +## Recursive sets + +Recursive sets are like normal [attribute sets](./types.md#attribute-set), but the attributes can refer to each other. + +> *rec-attrset* = `rec {` [ *name* `=` *expr* `;` `]`... `}` + +Example: + +```nix +rec { + x = y; + y = 123; +}.x +``` + +This evaluates to `123`. + +Note that without `rec` the binding `x = y;` would +refer to the variable `y` in the surrounding scope, if one exists, and +would be invalid if no such variable exists. That is, in a normal +(non-recursive) set, attributes are not added to the lexical scope; in a +recursive set, they are. + +Recursive sets of course introduce the danger of infinite recursion. For +example, the expression + +```nix +rec { + x = y; + y = x; +}.x +``` + +will crash with an `infinite recursion encountered` error message. + +## Let-expressions + +A let-expression allows you to define local variables for an expression. + +> *let-in* = `let` [ *identifier* = *expr* ]... `in` *expr* + +Example: + +```nix +let + x = "foo"; + y = "bar"; +in x + y +``` + +This evaluates to `"foobar"`. + +## Inheriting attributes + +When defining an [attribute set](./types.md#attribute-set) or in a [let-expression](#let-expressions) it is often convenient to copy variables from the surrounding lexical scope (e.g., when you want to propagate attributes). +This can be shortened using the `inherit` keyword. + +Example: + +```nix +let x = 123; in +{ + inherit x; + y = 456; +} +``` + +is equivalent to + +```nix +let x = 123; in +{ + x = x; + y = 456; +} +``` + +and both evaluate to `{ x = 123; y = 456; }`. + +> **Note** +> +> This works because `x` is added to the lexical scope by the `let` construct. + +It is also possible to inherit attributes from another attribute set. + +Example: + +In this fragment from `all-packages.nix`, + +```nix +graphviz = (import ../tools/graphics/graphviz) { + inherit fetchurl stdenv libpng libjpeg expat x11 yacc; + inherit (xorg) libXaw; +}; + +xorg = { + libX11 = ...; + libXaw = ...; + ... +} + +libpng = ...; +libjpg = ...; +... +``` + +the set used in the function call to the function defined in +`../tools/graphics/graphviz` inherits a number of variables from the +surrounding scope (`fetchurl` ... `yacc`), but also inherits `libXaw` +(the X Athena Widgets) from the `xorg` set. + +Summarizing the fragment + +```nix +... +inherit x y z; +inherit (src-set) a b c; +... +``` + +is equivalent to + +```nix +... +x = x; y = y; z = z; +a = src-set.a; b = src-set.b; c = src-set.c; +... +``` + +when used while defining local variables in a let-expression or while +defining a set. + +In a `let` expression, `inherit` can be used to selectively bring specific attributes of a set into scope. For example + + +```nix +let + x = { a = 1; b = 2; }; + inherit (builtins) attrNames; +in +{ + names = attrNames x; +} +``` + +is equivalent to + +```nix +let + x = { a = 1; b = 2; }; +in +{ + names = builtins.attrNames x; +} +``` + +both evaluate to `{ names = [ "a" "b" ]; }`. + +## Functions + +Functions have the following form: + +```nix +pattern: body +``` + +The pattern specifies what the argument of the function must look like, +and binds variables in the body to (parts of) the argument. There are +three kinds of patterns: + + - If a pattern is a single identifier, then the function matches any + argument. Example: + + ```nix + let negate = x: !x; + concat = x: y: x + y; + in if negate true then concat "foo" "bar" else "" + ``` + + Note that `concat` is a function that takes one argument and returns + a function that takes another argument. This allows partial + parameterisation (i.e., only filling some of the arguments of a + function); e.g., + + ```nix + map (concat "foo") [ "bar" "bla" "abc" ] + ``` + + evaluates to `[ "foobar" "foobla" "fooabc" ]`. + + - A *set pattern* of the form `{ name1, name2, …, nameN }` matches a + set containing the listed attributes, and binds the values of those + attributes to variables in the function body. For example, the + function + + ```nix + { x, y, z }: z + y + x + ``` + + can only be called with a set containing exactly the attributes `x`, + `y` and `z`. No other attributes are allowed. If you want to allow + additional arguments, you can use an ellipsis (`...`): + + ```nix + { x, y, z, ... }: z + y + x + ``` + + This works on any set that contains at least the three named + attributes. + + It is possible to provide *default values* for attributes, in + which case they are allowed to be missing. A default value is + specified by writing `name ? e`, where *e* is an arbitrary + expression. For example, + + ```nix + { x, y ? "foo", z ? "bar" }: z + y + x + ``` + + specifies a function that only requires an attribute named `x`, but + optionally accepts `y` and `z`. + + - An `@`-pattern provides a means of referring to the whole value + being matched: + + ```nix + args@{ x, y, z, ... }: z + y + x + args.a + ``` + + but can also be written as: + + ```nix + { x, y, z, ... } @ args: z + y + x + args.a + ``` + + Here `args` is bound to the argument *as passed*, which is further + matched against the pattern `{ x, y, z, ... }`. + The `@`-pattern makes mainly sense with an ellipsis(`...`) as + you can access attribute names as `a`, using `args.a`, which was + given as an additional attribute to the function. + + > **Warning** + > + > `args@` binds the name `args` to the attribute set that is passed to the function. + > In particular, `args` does *not* include any default values specified with `?` in the function's set pattern. + > + > For instance + > + > ```nix + > let + > f = args@{ a ? 23, ... }: [ a args ]; + > in + > f {} + > ``` + > + > is equivalent to + > + > ```nix + > let + > f = args @ { ... }: [ (args.a or 23) args ]; + > in + > f {} + > ``` + > + > and both expressions will evaluate to: + > + > ```nix + > [ 23 {} ] + > ``` + +Note that functions do not have names. If you want to give them a name, +you can bind them to an attribute, e.g., + +```nix +let concat = { x, y }: x + y; +in concat { x = "foo"; y = "bar"; } +``` + +## Conditionals + +Conditionals look like this: + +```nix +if e1 then e2 else e3 +``` + +where *e1* is an expression that should evaluate to a Boolean value +(`true` or `false`). + +## Assertions + +Assertions are generally used to check that certain requirements on or +between features and dependencies hold. They look like this: + +```nix +assert e1; e2 +``` + +where *e1* is an expression that should evaluate to a Boolean value. If +it evaluates to `true`, *e2* is returned; otherwise expression +evaluation is aborted and a backtrace is printed. + +Here is a Nix expression for the Subversion package that shows how +assertions can be used:. + +```nix +{ localServer ? false +, httpServer ? false +, sslSupport ? false +, pythonBindings ? false +, javaSwigBindings ? false +, javahlBindings ? false +, stdenv, fetchurl +, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null +}: + +assert localServer -> db4 != null; â‘ +assert httpServer -> httpd != null && httpd.expat == expat; â‘¡ +assert sslSupport -> openssl != null && (httpServer -> httpd.openssl == openssl); â‘¢ +assert pythonBindings -> swig != null && swig.pythonSupport; +assert javaSwigBindings -> swig != null && swig.javaSupport; +assert javahlBindings -> j2sdk != null; + +stdenv.mkDerivation { + name = "subversion-1.1.1"; + ... + openssl = if sslSupport then openssl else null; â‘£ + ... +} +``` + +The points of interest are: + +1. This assertion states that if Subversion is to have support for + local repositories, then Berkeley DB is needed. So if the Subversion + function is called with the `localServer` argument set to `true` but + the `db4` argument set to `null`, then the evaluation fails. + + Note that `->` is the [logical + implication](https://en.wikipedia.org/wiki/Truth_table#Logical_implication) + Boolean operation. + +2. This is a more subtle condition: if Subversion is built with Apache + (`httpServer`) support, then the Expat library (an XML library) used + by Subversion should be same as the one used by Apache. This is + because in this configuration Subversion code ends up being linked + with Apache code, and if the Expat libraries do not match, a build- + or runtime link error or incompatibility might occur. + +3. This assertion says that in order for Subversion to have SSL support + (so that it can access `https` URLs), an OpenSSL library must be + passed. Additionally, it says that *if* Apache support is enabled, + then Apache's OpenSSL should match Subversion's. (Note that if + Apache support is not enabled, we don't care about Apache's + OpenSSL.) + +4. The conditional here is not really related to assertions, but is + worth pointing out: it ensures that if SSL support is disabled, then + the Subversion derivation is not dependent on OpenSSL, even if a + non-`null` value was passed. This prevents an unnecessary rebuild of + Subversion if OpenSSL changes. + +## With-expressions + +A *with-expression*, + +```nix +with e1; e2 +``` + +introduces the set *e1* into the lexical scope of the expression *e2*. +For instance, + +```nix +let as = { x = "foo"; y = "bar"; }; +in with as; x + y +``` + +evaluates to `"foobar"` since the `with` adds the `x` and `y` attributes +of `as` to the lexical scope in the expression `x + y`. The most common +use of `with` is in conjunction with the `import` function. E.g., + +```nix +with (import ./definitions.nix); ... +``` + +makes all attributes defined in the file `definitions.nix` available as +if they were defined locally in a `let`-expression. + +The bindings introduced by `with` do not shadow bindings introduced by +other means, e.g. + +```nix +let a = 3; in with { a = 1; }; let a = 4; in with { a = 2; }; ... +``` + +establishes the same scope as + +```nix +let a = 1; in let a = 2; in let a = 3; in let a = 4; in ... +``` + +Variables coming from outer `with` expressions *are* shadowed: + +```nix +with { a = "outer"; }; +with { a = "inner"; }; +a +``` + +Does evaluate to `"inner"`. + +## Comments + +- Inline comments start with `#` and run until the end of the line. + + > **Example** + > + > ```nix + > # A number + > 2 # Equals 1 + 1 + > ``` + > + > ```console + > 2 + > ``` + +- Block comments start with `/*` and run until the next occurrence of `*/`. + + > **Example** + > + > ```nix + > /* + > Block comments + > can span multiple lines. + > */ "hello" + > ``` + > + > ```console + > "hello" + > ``` + + This means that block comments cannot be nested. + + > **Example** + > + > ```nix + > /* /* nope */ */ 1 + > ``` + > + > ```console + > error: syntax error, unexpected '*' + > + > at «string»:1:15: + > + > 1| /* /* nope */ * + > | ^ + > ``` + + Consider escaping nested comments and unescaping them in post-processing. + + > **Example** + > + > ```nix + > /* /* nested *\/ */ 1 + > ``` + > + > ```console + > 1 + > ``` diff --git a/doc/manual/src/language/types.md b/doc/manual/src/language/types.md new file mode 100644 index 000000000..229756e6b --- /dev/null +++ b/doc/manual/src/language/types.md @@ -0,0 +1,120 @@ +# Data Types + +Every value in the Nix language has one of the following types: + +* [Integer](#type-int) +* [Float](#type-float) +* [Boolean](#type-bool) +* [String](#type-string) +* [Path](#type-path) +* [Null](#type-null) +* [Attribute set](#type-attrs) +* [List](#type-list) +* [Function](#type-function) +* [External](#type-external) + +## Primitives + +### Integer {#type-int} + +An _integer_ in the Nix language is a signed 64-bit integer. + +Non-negative integers can be expressed as [integer literals](syntax.md#number-literal). +Negative integers are created with the [arithmetic negation operator](./operators.md#arithmetic). +The function [`builtins.isInt`](builtins.md#builtins-isInt) can be used to determine if a value is an integer. + +### Float {#type-float} + +A _float_ in the Nix language is a 64-bit [IEEE 754](https://en.wikipedia.org/wiki/IEEE_754) floating-point number. + +Most non-negative floats can be expressed as [float literals](syntax.md#number-literal). +Negative floats are created with the [arithmetic negation operator](./operators.md#arithmetic). +The function [`builtins.isFloat`](builtins.md#builtins-isFloat) can be used to determine if a value is a float. + +### Boolean {#type-bool} + +A _boolean_ in the Nix language is one of _true_ or _false_. + + + +These values are available as attributes of [`builtins`](builtins.md#builtins-builtins) as [`builtins.true`](builtins.md#builtins-true) and [`builtins.false`](builtins.md#builtins-false). +The function [`builtins.isBool`](builtins.md#builtins-isBool) can be used to determine if a value is a boolean. + +### String {#type-string} + +A _string_ in the Nix language is an immutable, finite-length sequence of bytes, along with a [string context](string-context.md). +Nix does not assume or support working natively with character encodings. + +String values without string context can be expressed as [string literals](syntax.md#string-literal). +The function [`builtins.isString`](builtins.md#builtins-isString) can be used to determine if a value is a string. + +### Path {#type-path} + +A _path_ in the Nix language is an immutable, finite-length sequence of bytes starting with `/`, representing a POSIX-style, canonical file system path. +Path values are distinct from string values, even if they contain the same sequence of bytes. +Operations that produce paths will simplify the result as the standard C function [`realpath`] would, except that there is no symbolic link resolution. + +[`realpath`]: https://pubs.opengroup.org/onlinepubs/9699919799/functions/realpath.html + +Paths are suitable for referring to local files, and are often preferable over strings. +- Path values do not contain trailing or duplicate slashes, `.`, or `..`. +- Relative path literals are automatically resolved relative to their [base directory]. +- Tooling can recognize path literals and provide additional features, such as autocompletion, refactoring automation and jump-to-file. + +[base directory]: @docroot@/glossary.md#gloss-base-directory + +A file is not required to exist at a given path in order for that path value to be valid, but a path that is converted to a string with [string interpolation] or [string-and-path concatenation] must resolve to a readable file or directory which will be copied into the Nix store. +For instance, evaluating `"${./foo.txt}"` will cause `foo.txt` from the same directory to be copied into the Nix store and result in the string `"/nix/store/-foo.txt"`. +Operations such as [`import`] can also expect a path to resolve to a readable file or directory. + +[string interpolation]: string-interpolation.md#interpolated-expression +[string-and-path concatenation]: operators.md#string-and-path-concatenation +[`import`]: builtins.md#builtins-import + +> **Note** +> +> The Nix language assumes that all input files will remain _unchanged_ while evaluating a Nix expression. +> For example, assume you used a file path in an interpolated string during a `nix repl` session. +> Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new [store path], since Nix might not re-read the file contents. +> Use `:r` to reset the repl as needed. + +[store path]: @docroot@/store/store-path.md + +Path values can be expressed as [path literals](syntax.md#path-literal). +The function [`builtins.isPath`](builtins.md#builtins-isPath) can be used to determine if a value is a path. + +### Null {#type-null} + +There is a single value of type _null_ in the Nix language. + + + +This value is available as an attribute on the [`builtins`](builtins.md#builtins-builtins) attribute set as [`builtins.null`](builtins.md#builtins-null). + +## Compound values + +### Attribute set {#type-attrs} + + + +An attribute set can be constructed with an [attribute set literal](syntax.md#attrs-literal). +The function [`builtins.isAttrs`](builtins.md#builtins-isAttrs) can be used to determine if a value is an attribute set. + +### List {#type-list} + + + +A list can be constructed with a [list literal](syntax.md#list-literal). +The function [`builtins.isList`](builtins.md#builtins-isList) can be used to determine if a value is a list. + +## Function {#type-function} + + + +A function can be constructed with a [function expression](syntax.md#functions). +The function [`builtins.isFunction`](builtins.md#builtins-isFunction) can be used to determine if a value is a function. + +## External {#type-external} + +An _external_ value is an opaque value created by a Nix [plugin](../command-ref/conf-file.md#conf-plugin-files). +Such a value can be substituted in Nix expressions but only created and used by plugin code. diff --git a/doc/manual/src/language/values.md b/doc/manual/src/language/values.md deleted file mode 100644 index 74ffc7070..000000000 --- a/doc/manual/src/language/values.md +++ /dev/null @@ -1,269 +0,0 @@ -# Data Types - -## Primitives - -- String - - *Strings* can be written in three ways. - - The most common way is to enclose the string between double quotes, - e.g., `"foo bar"`. Strings can span multiple lines. The special - characters `"` and `\` and the character sequence `${` must be - escaped by prefixing them with a backslash (`\`). Newlines, carriage - returns and tabs can be written as `\n`, `\r` and `\t`, - respectively. - - You can include the results of other expressions into a string by enclosing them in `${ }`, a feature known as [string interpolation]. - - [string interpolation]: ./string-interpolation.md - - The second way to write string literals is as an *indented string*, - which is enclosed between pairs of *double single-quotes*, like so: - - ```nix - '' - This is the first line. - This is the second line. - This is the third line. - '' - ``` - - This kind of string literal intelligently strips indentation from - the start of each line. To be precise, it strips from each line a - number of spaces equal to the minimal indentation of the string as a - whole (disregarding the indentation of empty lines). For instance, - the first and second line are indented two spaces, while the third - line is indented four spaces. Thus, two spaces are stripped from - each line, so the resulting string is - - ```nix - "This is the first line.\nThis is the second line.\n This is the third line.\n" - ``` - - Note that the whitespace and newline following the opening `''` is - ignored if there is no non-whitespace text on the initial line. - - Indented strings support [string interpolation]. - - Since `${` and `''` have special meaning in indented strings, you - need a way to quote them. `$` can be escaped by prefixing it with - `''` (that is, two single quotes), i.e., `''$`. `''` can be escaped - by prefixing it with `'`, i.e., `'''`. `$` removes any special - meaning from the following `$`. Linefeed, carriage-return and tab - characters can be written as `''\n`, `''\r`, `''\t`, and `''\` - escapes any other character. - - Indented strings are primarily useful in that they allow multi-line - string literals to follow the indentation of the enclosing Nix - expression, and that less escaping is typically necessary for - strings representing languages such as shell scripts and - configuration files because `''` is much less common than `"`. - Example: - - ```nix - stdenv.mkDerivation { - ... - postInstall = - '' - mkdir $out/bin $out/etc - cp foo $out/bin - echo "Hello World" > $out/etc/foo.conf - ${if enableBar then "cp bar $out/bin" else ""} - ''; - ... - } - ``` - - Finally, as a convenience, *URIs* as defined in appendix B of - [RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as - is*, without quotes. For instance, the string - `"http://example.org/foo.tar.bz2"` can also be written as - `http://example.org/foo.tar.bz2`. - -- Number - - Numbers, which can be *integers* (like `123`) or *floating point* - (like `123.43` or `.27e13`). - - See [arithmetic] and [comparison] operators for semantics. - - [arithmetic]: ./operators.md#arithmetic - [comparison]: ./operators.md#comparison - -- Path - - *Paths*, e.g., `/bin/sh` or `./builder.sh`. A path must contain at - least one slash to be recognised as such. For instance, `builder.sh` - is not a path: it's parsed as an expression that selects the - attribute `sh` from the variable `builder`. If the file name is - relative, i.e., if it does not begin with a slash, it is made - absolute at parse time relative to the directory of the Nix - expression that contained it. For instance, if a Nix expression in - `/foo/bar/bla.nix` refers to `../xyzzy/fnord.nix`, the absolute path - is `/foo/xyzzy/fnord.nix`. - - If the first component of a path is a `~`, it is interpreted as if - the rest of the path were relative to the user's home directory. - e.g. `~/foo` would be equivalent to `/home/edolstra/foo` for a user - whose home directory is `/home/edolstra`. - - For instance, evaluating `"${./foo.txt}"` will cause `foo.txt` in the current directory to be copied into the Nix store and result in the string `"/nix/store/-foo.txt"`. - - Note that the Nix language assumes that all input files will remain _unchanged_ while evaluating a Nix expression. - For example, assume you used a file path in an interpolated string during a `nix repl` session. - Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new [store path], since Nix might not re-read the file contents. - - [store path]: ../glossary.md#gloss-store-path - - Paths can include [string interpolation] and can themselves be [interpolated in other expressions]. - - [interpolated in other expressions]: ./string-interpolation.md#interpolated-expressions - - At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path. - - `a.${foo}/b.${bar}` is a syntactically valid division operation. - `./a.${foo}/b.${bar}` is a path. - - [Lookup paths](./constructs/lookup-path.md) such as `` resolve to path values. - -- Boolean - - *Booleans* with values `true` and `false`. - -- Null - - The null value, denoted as `null`. - -## List - -Lists are formed by enclosing a whitespace-separated list of values -between square brackets. For example, - -```nix -[ 123 ./foo.nix "abc" (f { x = y; }) ] -``` - -defines a list of four elements, the last being the result of a call to -the function `f`. Note that function calls have to be enclosed in -parentheses. If they had been omitted, e.g., - -```nix -[ 123 ./foo.nix "abc" f { x = y; } ] -``` - -the result would be a list of five elements, the fourth one being a -function and the fifth being a set. - -Note that lists are only lazy in values, and they are strict in length. - -Elements in a list can be accessed using [`builtins.elemAt`](./builtins.md#builtins-elemAt). - -## Attribute Set - -An attribute set is a collection of name-value-pairs (called *attributes*) enclosed in curly brackets (`{ }`). - -An attribute name can be an identifier or a [string](#string). -An identifier must start with a letter (`a-z`, `A-Z`) or underscore (`_`), and can otherwise contain letters (`a-z`, `A-Z`), numbers (`0-9`), underscores (`_`), apostrophes (`'`), or dashes (`-`). - -> **Syntax** -> -> *name* = *identifier* | *string* \ -> *identifier* ~ `[a-zA-Z_][a-zA-Z0-9_'-]*` - -Names and values are separated by an equal sign (`=`). -Each value is an arbitrary expression terminated by a semicolon (`;`). - -> **Syntax** -> -> *attrset* = `{` [ *name* `=` *expr* `;` ]... `}` - -Attributes can appear in any order. -An attribute name may only occur once. - -Example: - -```nix -{ - x = 123; - text = "Hello"; - y = f { bla = 456; }; -} -``` - -This defines a set with attributes named `x`, `text`, `y`. - -Attributes can be accessed with the [`.` operator](./operators.md#attribute-selection). - -Example: - -```nix -{ a = "Foo"; b = "Bar"; }.a -``` - -This evaluates to `"Foo"`. - -It is possible to provide a default value in an attribute selection using the `or` keyword. - -Example: - -```nix -{ a = "Foo"; b = "Bar"; }.c or "Xyzzy" -``` - -```nix -{ a = "Foo"; b = "Bar"; }.c.d.e.f.g or "Xyzzy" -``` - -will both evaluate to `"Xyzzy"` because there is no `c` attribute in the set. - -You can use arbitrary double-quoted strings as attribute names: - -```nix -{ "$!@#?" = 123; }."$!@#?" -``` - -```nix -let bar = "bar"; in -{ "foo ${bar}" = 123; }."foo ${bar}" -``` - -Both will evaluate to `123`. - -Attribute names support [string interpolation]: - -```nix -let bar = "foo"; in -{ foo = 123; }.${bar} -``` - -```nix -let bar = "foo"; in -{ ${bar} = 123; }.foo -``` - -Both will evaluate to `123`. - -In the special case where an attribute name inside of a set declaration -evaluates to `null` (which is normally an error, as `null` cannot be coerced to -a string), that attribute is simply not added to the set: - -```nix -{ ${if foo then "bar" else null} = true; } -``` - -This will evaluate to `{}` if `foo` evaluates to `false`. - -A set that has a `__functor` attribute whose value is callable (i.e. is -itself a function or a set with a `__functor` attribute whose value is -callable) can be applied as if it were a function, with the set itself -passed in first , e.g., - -```nix -let add = { __functor = self: x: x + self.x; }; - inc = add // { x = 1; }; -in inc 1 -``` - -evaluates to `2`. This can be used to attach metadata to a function -without the caller needing to treat it specially, or to implement a form -of object-oriented programming, for example. diff --git a/doc/manual/src/language/variables.md b/doc/manual/src/language/variables.md new file mode 100644 index 000000000..af6aff8a2 --- /dev/null +++ b/doc/manual/src/language/variables.md @@ -0,0 +1,10 @@ +# Variables + +A *variable* is an [identifier](identifiers.md) used as an expression. + +> **Syntax** +> +> *expression* → *identifier* + +A variable must have the same name as a definition in the [scope](./scope.md) that encloses it. +The value of a variable is the value of the corresponding expression in the enclosing scope. diff --git a/doc/manual/src/package-management/copy-closure.md b/doc/manual/src/package-management/copy-closure.md deleted file mode 100644 index 14326298b..000000000 --- a/doc/manual/src/package-management/copy-closure.md +++ /dev/null @@ -1,34 +0,0 @@ -# Copying Closures via SSH - -The command `nix-copy-closure` copies a Nix store path along with all -its dependencies to or from another machine via the SSH protocol. It -doesn’t copy store paths that are already present on the target machine. -For example, the following command copies Firefox with all its -dependencies: - - $ nix-copy-closure --to alice@itchy.example.org $(type -p firefox) - -See the [manpage for `nix-copy-closure`](../command-ref/nix-copy-closure.md) for details. - -With `nix-store ---export` and `nix-store --import` you can write the closure of a store -path (that is, the path and all its dependencies) to a file, and then -unpack that file into another Nix store. For example, - - $ nix-store --export $(nix-store --query --requisites $(type -p firefox)) > firefox.closure - -writes the closure of Firefox to a file. You can then copy this file to -another machine and install the closure: - - $ nix-store --import < firefox.closure - -Any store paths in the closure that are already present in the target -store are ignored. It is also possible to pipe the export into another -command, e.g. to copy and install a closure directly to/on another -machine: - - $ nix-store --export $(nix-store --query --requisites $(type -p firefox)) | bzip2 | \ - ssh alice@itchy.example.org "bunzip2 | nix-store --import" - -However, `nix-copy-closure` is generally more efficient because it only -copies paths that are not already present in the target Nix store. diff --git a/doc/manual/src/protocols/derivation-aterm.md b/doc/manual/src/protocols/derivation-aterm.md index e58b602a3..1ba757ae0 100644 --- a/doc/manual/src/protocols/derivation-aterm.md +++ b/doc/manual/src/protocols/derivation-aterm.md @@ -14,6 +14,6 @@ Derivations are serialised in one of the following formats: DrvWithVersion(, ...) ``` - The only `version-string`s that are in use today are for [experimental features](@docroot@/contributing/experimental-features.md): + The only `version-string`s that are in use today are for [experimental features](@docroot@/development/experimental-features.md): - - `"xp-dyn-drv"` for the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. + - `"xp-dyn-drv"` for the [`dynamic-derivations`](@docroot@/development/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. diff --git a/doc/manual/src/protocols/json/derivation.md b/doc/manual/src/protocols/json/derivation.md index 649d543cc..2f85340d6 100644 --- a/doc/manual/src/protocols/json/derivation.md +++ b/doc/manual/src/protocols/json/derivation.md @@ -3,7 +3,7 @@ > **Warning** > > This JSON format is currently -> [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) +> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) > and subject to change. The JSON serialization of a @@ -18,10 +18,30 @@ is a JSON object with the following fields: Information about the output paths of the derivation. This is a JSON object with one member per output, where the key is the output name and the value is a JSON object with these fields: - * `path`: The output path. + * `path`: + The output path, if it is known in advanced. + Otherwise, `null`. + + + * `method`: + For an output which will be [content addresed], a string representing the [method](@docroot@/store/store-object/content-address.md) of content addressing that is chosen. + Valid method strings are: + + - [`flat`](@docroot@/store/store-object/content-address.md#method-flat) + - [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive) + - [`text`](@docroot@/store/store-object/content-address.md#method-text) + - [`git`](@docroot@/store/store-object/content-address.md#method-git) + + Otherwise, `null`. * `hashAlgo`: - For fixed-output derivations, the hashing algorithm (e.g. `sha256`), optionally prefixed by `r:` if `hash` denotes a NAR hash rather than a flat file hash. + For an output which will be [content addresed], the name of the hash algorithm used. + Valid algorithm strings are: + + - `md5` + - `sha1` + - `sha256` + - `sha512` * `hash`: For fixed-output derivations, the expected content hash in base-16. @@ -32,7 +52,8 @@ is a JSON object with the following fields: > "outputs": { > "out": { > "path": "/nix/store/2543j7c6jn75blc3drf4g5vhb1rhdq29-source", - > "hashAlgo": "r:sha256", + > "method": "nar", + > "hashAlgo": "sha256", > "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62" > } > } diff --git a/doc/manual/src/protocols/json/store-object-info.md b/doc/manual/src/protocols/json/store-object-info.md index ba4ab098f..6b4f48437 100644 --- a/doc/manual/src/protocols/json/store-object-info.md +++ b/doc/manual/src/protocols/json/store-object-info.md @@ -3,7 +3,7 @@ > **Warning** > > This JSON format is currently -> [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) +> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) > and subject to change. Info about a [store object]. @@ -24,41 +24,45 @@ Info about a [store object]. An array of [store paths][store path], possibly including this one. -* `ca` (optional): +* `ca`: - Content address of this store object's file system object, used to compute its store path. + If the store object is [content-addressed], + this is the content address of this store object's file system object, used to compute its store path. + Otherwise (i.e. if it is [input-addressed]), this is `null`. -[store path]: @docroot@/glossary.md#gloss-store-path +[store path]: @docroot@/store/store-path.md [file system object]: @docroot@/store/file-system-object.md -[Nix Archive]: @docroot@/glossary.md#gloss-nar +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive ## Impure fields These are not intrinsic properties of the store object. In other words, the same store object residing in different store could have different values for these properties. -* `deriver` (optional): +* `deriver`: - The path to the [derivation] from which this store object is produced. + If known, the path to the [derivation] from which this store object was produced. + Otherwise `null`. [derivation]: @docroot@/glossary.md#gloss-store-derivation * `registrationTime` (optional): - When this derivation was added to the store. + If known, when this derivation was added to the store. + Otherwise `null`. -* `ultimate` (optional): +* `ultimate`: Whether this store object is trusted because we built it ourselves, rather than substituted a build product from elsewhere. -* `signatures` (optional): +* `signatures`: Signatures claiming that this store object is what it claims to be. Not relevant for [content-addressed] store objects, but useful for [input-addressed] store objects. - [content-addressed]: @docroot@/glossary.md#gloss-content-addressed-store-object - [input-addressed]: @docroot@/glossary.md#gloss-input-addressed-store-object +[content-addressed]: @docroot@/store/store-object/content-address.md +[input-addressed]: @docroot@/glossary.md#gloss-input-addressed-store-object ### `.narinfo` extra fields @@ -83,7 +87,7 @@ This information is not intrinsic to the store object, but about how it is store ## Computed closure fields -These fields are not stored at all, but computed by traverising the other other fields across all the store objects in a [closure]. +These fields are not stored at all, but computed by traversing the other fields across all the store objects in a [closure]. * `closureSize`: diff --git a/doc/manual/src/protocols/nix-archive.md b/doc/manual/src/protocols/nix-archive.md new file mode 100644 index 000000000..640b527f1 --- /dev/null +++ b/doc/manual/src/protocols/nix-archive.md @@ -0,0 +1,43 @@ +# Nix Archive (NAR) format + +This is the complete specification of the [Nix Archive] format. +The Nix Archive format closely follows the abstract specification of a [file system object] tree, +because it is designed to serialize exactly that data structure. + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#nix-archive +[file system object]: @docroot@/store/file-system-object.md + +The format of this specification is close to [Extended Backus–Naur form](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form), with the exception of the `str(..)` function / parameterized rule, which length-prefixes and pads strings. +This makes the resulting binary format easier to parse. + +Regular users do *not* need to know this information. +But for those interested in exactly how Nix works, e.g. if they are reimplementing it, this information can be useful. + +```ebnf +nar = str("nix-archive-1"), nar-obj; + +nar-obj = str("("), nar-obj-inner, str(")"); + +nar-obj-inner + = str("type"), str("regular") regular + | str("type"), str("symlink") symlink + | str("type"), str("directory") directory + ; + +regular = [ str("executable"), str("") ], str("contents"), str(contents); + +symlink = str("target"), str(target); + +(* side condition: directory entries must be ordered by their names *) +directory = { directory-entry }; + +directory-entry = str("entry"), str("("), str("name"), str(name), str("node"), nar-obj, str(")"); +``` + +The `str` function / parameterized rule is defined as follows: + +- `str(s)` = `int(|s|), pad(s);` + +- `int(n)` = the 64-bit little endian representation of the number `n` + +- `pad(s)` = the byte sequence `s`, padded with 0s to a multiple of 8 byte diff --git a/doc/manual/src/protocols/store-path.md b/doc/manual/src/protocols/store-path.md index 565c4fa75..52352d358 100644 --- a/doc/manual/src/protocols/store-path.md +++ b/doc/manual/src/protocols/store-path.md @@ -1,12 +1,14 @@ # Complete Store Path Calculation -This is the complete specification for how store paths are calculated. +This is the complete specification for how [store path]s are calculated. The format of this specification is close to [Extended Backus–Naur form](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form), but must deviate for a few things such as hash functions which we treat as bidirectional for specification purposes. Regular users do *not* need to know this information --- store paths can be treated as black boxes computed from the properties of the store objects they refer to. But for those interested in exactly how Nix works, e.g. if they are reimplementing it, this information can be useful. +[store path](@docroot@/store/store-path.md) + ## Store path proper ```ebnf @@ -34,18 +36,23 @@ where - `type` = one of: - ```ebnf - | "text" ( ":" store-path )* + | "text" { ":" store-path } ``` - for encoded derivations written to the store. + This is for the + ["Text"](@docroot@/store/store-object/content-address.md#method-text) + method of content addressing store objects. The optional trailing store paths are the references of the store object. - ```ebnf - | "source" ( ":" store-path )* + | "source" { ":" store-path } [ ":self" ] ``` - For paths copied to the store and hashed via a [Nix Archive (NAR)] and [SHA-256][sha-256]. - Just like in the text case, we can have the store objects referenced by their paths. + This is for the + ["Nix Archive"](@docroot@/store/store-object/content-address.md#method-nix-archive) + method of content addressing store objects, + if the hash algorithm is [SHA-256]. + Just like in the "Text" case, we can have the store objects referenced by their paths. Additionally, we can have an optional `:self` label to denote self reference. - ```ebnf @@ -53,8 +60,12 @@ where ``` For either the outputs built from derivations, - paths copied to the store hashed that area single file hashed directly, or the via a hash algorithm other than [SHA-256][sha-256]. - (in that case "source" is used; this is only necessary for compatibility). + or content-addressed store objects that are not using one of the two above cases. + To be explicit about the latter, that is currently these methods: + + - ["Flat"](@docroot@/store/store-object/content-address.md#method-flat) + - ["Git"](@docroot@/store/store-object/content-address.md#method-git) + - ["Nix Archive"](@docroot@/store/store-object/content-address.md#method-nix-archive) if the hash algorithm is not [SHA-256]. `id` is the name of the output (usually, "out"). For content-addressed store objects, `id`, is always "out". @@ -113,8 +124,8 @@ where Note that `id` = `"out"`, regardless of the name part of the store path. Also note that NAR + SHA-256 must not use this case, and instead must use the `type` = `"source:" ...` case. -[Nix Archive (NAR)]: @docroot@/glossary.md#gloss-NAR -[sha-256]: https://en.m.wikipedia.org/wiki/SHA-256 +[Nix Archive (NAR)]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive +[SHA-256]: https://en.m.wikipedia.org/wiki/SHA-256 ### Historical Note diff --git a/doc/manual/src/protocols/tarball-fetcher.md b/doc/manual/src/protocols/tarball-fetcher.md index 274fa6d63..5cff05d66 100644 --- a/doc/manual/src/protocols/tarball-fetcher.md +++ b/doc/manual/src/protocols/tarball-fetcher.md @@ -22,7 +22,7 @@ Link: ; rel="immutable" *flakeref* must be a tarball flakeref. It can contain the tarball flake attributes `narHash`, `rev`, `revCount` and `lastModified`. If `narHash` is included, its -value must be the NAR hash of the unpacked tarball (as computed via +value must be the [NAR hash][Nix Archive] of the unpacked tarball (as computed via `nix hash path`). Nix checks the contents of the returned tarball against the `narHash` attribute. The `rev` and `revCount` attributes are useful when the tarball flake is a mirror of a fetcher type that @@ -40,3 +40,31 @@ Link: ///archive/.tar.gz +``` + +> **Example** +> +> +> ```nix +> # flake.nix +> { +> inputs = { +> foo.url = "https://gitea.example.org/some-person/some-flake/archive/main.tar.gz"; +> bar.url = "https://gitea.example.org/some-other-person/other-flake/archive/442793d9ec0584f6a6e82fa253850c8085bb150a.tar.gz"; +> qux = { +> url = "https://forgejo.example.org/another-person/some-non-flake-repo/archive/development.tar.gz"; +> flake = false; +> }; +> }; +> outputs = { foo, bar, qux }: { /* ... */ }; +> } +``` + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive diff --git a/doc/manual/src/quick-start.md b/doc/manual/src/quick-start.md index 75853ced7..9eb7a3265 100644 --- a/doc/manual/src/quick-start.md +++ b/doc/manual/src/quick-start.md @@ -34,7 +34,7 @@ For more in-depth information you are kindly referred to subsequent chapters. lolcat: command not found ``` -1. Search for more packages on to try them out. +1. Search for more packages on [search.nixos.org](https://search.nixos.org/) to try them out. 1. Free up storage space: diff --git a/doc/manual/src/release-notes/index.md b/doc/manual/src/release-notes/index.md index cc805e631..d4e6292a6 100644 --- a/doc/manual/src/release-notes/index.md +++ b/doc/manual/src/release-notes/index.md @@ -1,12 +1,13 @@ # Nix Release Notes +The Nix release cycle is calendar-based as follows: + Nix has a release cycle of roughly 6 weeks. Notable changes and additions are announced in the release notes for each version. -Bugfixes can be backported on request to previous Nix releases. -We typically backport only as far back as the Nix version used in the latest NixOS release, which is announced in the [NixOS release notes](https://nixos.org/manual/nixos/stable/release-notes.html#ch-release-notes). - -Backports never skip releases. -If a feature is backported to version `x.y`, it must also be available in version `x.(y+1)`. -This ensures that upgrading from an older version with backports is still safe and no backported functionality will go missing. +The supported Nix versions are: +- The latest release +- The version used in the stable NixOS release, which is announced in the [NixOS release notes](https://nixos.org/manual/nixos/stable/release-notes.html#ch-release-notes). +Bugfixes and security issues are backported to every supported version. +Patch releases are published as needed. diff --git a/doc/manual/src/release-notes/rl-2.15.md b/doc/manual/src/release-notes/rl-2.15.md index 133121999..e7e52631b 100644 --- a/doc/manual/src/release-notes/rl-2.15.md +++ b/doc/manual/src/release-notes/rl-2.15.md @@ -11,7 +11,7 @@ As the choice of hash formats is no longer binary, the `--base16` flag is also added to explicitly specify the Base16 format, which is still the default. -* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](../glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents. +* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](@docroot@/glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents. The new `^` syntax for store paths introduced in Nix 2.13 allows explicitly referencing output paths of a derivation. Using this is better and more clear than relying on the now-removed `.drv` special handling. diff --git a/doc/manual/src/release-notes/rl-2.18.md b/doc/manual/src/release-notes/rl-2.18.md index 4bbc52b50..eb26fc9e7 100644 --- a/doc/manual/src/release-notes/rl-2.18.md +++ b/doc/manual/src/release-notes/rl-2.18.md @@ -13,7 +13,7 @@ - The `discard-references` feature has been stabilized. This means that the - [unsafeDiscardReferences](@docroot@/contributing/experimental-features.md#xp-feature-discard-references) + [unsafeDiscardReferences](@docroot@/development/experimental-features.md#xp-feature-discard-references) attribute is no longer guarded by an experimental flag and can be used freely. @@ -21,7 +21,7 @@ This only affects `nix-build --json` when "building" non-derivation things like fetched sources, which is a no-op. - A new builtin [`outputOf`](@docroot@/language/builtins.md#builtins-outputOf) has been added. - It is part of the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. + It is part of the [`dynamic-derivations`](@docroot@/development/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. - Flake follow paths at depths greater than 2 are now handled correctly, preventing "follows a non-existent input" errors. diff --git a/doc/manual/src/release-notes/rl-2.19.md b/doc/manual/src/release-notes/rl-2.19.md index ba6eb9c64..e2e2f85cc 100644 --- a/doc/manual/src/release-notes/rl-2.19.md +++ b/doc/manual/src/release-notes/rl-2.19.md @@ -17,8 +17,8 @@ - `nix-shell` shebang lines now support single-quoted arguments. -- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). - This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). +- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/development/experimental-features.md#xp-fetch-tree). + This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/development/experimental-features.md#xp-fetch-tree). - The interface for creating and updating lock files has been overhauled: @@ -33,7 +33,7 @@ - The flake-specific flags `--recreate-lock-file` and `--update-input` have been removed from all commands operating on installables. They are superceded by `nix flake update`. -- Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). +- Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches). - [`nix path-info --json`](@docroot@/command-ref/new-cli/nix3-path-info.md) (experimental) now returns a JSON map rather than JSON list. diff --git a/doc/manual/src/release-notes/rl-2.20.md b/doc/manual/src/release-notes/rl-2.20.md index 8ede168a4..eb724f600 100644 --- a/doc/manual/src/release-notes/rl-2.20.md +++ b/doc/manual/src/release-notes/rl-2.20.md @@ -200,3 +200,9 @@ while performing various operations (including `nix develop`, `nix flake update`, and so on). With several fixes to Nix's signal handlers, Nix commands will now exit quickly after Ctrl-C is pressed. + +- `nix copy` to a `ssh-ng` store now needs `--substitute-on-destination` (a.k.a. `-s`) + in order to substitute paths on the remote store instead of copying them. + The behavior is consistent with `nix copy` to a different kind of remote store. + Previously this behavior was controlled by the + `builders-use-substitutes` setting and `--substitute-on-destination` was ignored. diff --git a/doc/manual/src/release-notes/rl-2.21.md b/doc/manual/src/release-notes/rl-2.21.md new file mode 100644 index 000000000..75114f117 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.21.md @@ -0,0 +1,302 @@ +# Release 2.21.0 (2024-03-11) + +- Fix a fixed-output derivation sandbox escape (CVE-2024-27297) + + Cooperating Nix derivations could send file descriptors to files in the Nix + store to each other via Unix domain sockets in the abstract namespace. This + allowed one derivation to modify the output of the other derivation, after Nix + has registered the path as "valid" and immutable in the Nix database. + In particular, this allowed the output of fixed-output derivations to be + modified from their expected content. + + This isn't the case any more. + +- CLI options `--arg-from-file` and `--arg-from-stdin` [#10122](https://github.com/NixOS/nix/pull/10122) + + The new CLI option `--arg-from-file` *name* *path* passes the contents + of file *path* as a string value via the function argument *name* to a + Nix expression. Similarly, the new option `--arg-from-stdin` *name* + reads the contents of the string from standard input. + +- Concise error printing in `nix repl` [#9928](https://github.com/NixOS/nix/pull/9928) + + Previously, if an element of a list or attribute set threw an error while + evaluating, `nix repl` would print the entire error (including source location + information) inline. This output was clumsy and difficult to parse: + + ``` + nix-repl> { err = builtins.throw "uh oh!"; } + { err = «error: + … while calling the 'throw' builtin + at «string»:1:9: + 1| { err = builtins.throw "uh oh!"; } + | ^ + + error: uh oh!»; } + ``` + + Now, only the error message is displayed, making the output much more readable. + ``` + nix-repl> { err = builtins.throw "uh oh!"; } + { err = «error: uh oh!»; } + ``` + + However, if the whole expression being evaluated throws an error, source + locations and (if applicable) a stack trace are printed, just like you'd expect: + + ``` + nix-repl> builtins.throw "uh oh!" + error: + … while calling the 'throw' builtin + at «string»:1:1: + 1| builtins.throw "uh oh!" + | ^ + + error: uh oh! + ``` + +- `--debugger` can now access bindings from `let` expressions [#8827](https://github.com/NixOS/nix/issues/8827) [#9918](https://github.com/NixOS/nix/pull/9918) + + Breakpoints and errors in the bindings of a `let` expression can now access + those bindings in the debugger. Previously, only the body of `let` expressions + could access those bindings. + +- Enter the `--debugger` when `builtins.trace` is called if `debugger-on-trace` is set [#9914](https://github.com/NixOS/nix/pull/9914) + + If the `debugger-on-trace` option is set and `--debugger` is given, + `builtins.trace` calls will behave similarly to `builtins.break` and will enter + the debug REPL. This is useful for determining where warnings are being emitted + from. + +- Debugger prints source position information [#9913](https://github.com/NixOS/nix/pull/9913) + + The `--debugger` now prints source location information, instead of the + pointers of source location information. Before: + + ``` + nix-repl> :bt + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + 0x600001522598 + ``` + + After: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + /nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27 + + 131| + 132| bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs; + | ^ + 133| in + ``` + +- The `--debugger` will start more reliably in `let` expressions and function calls [#6649](https://github.com/NixOS/nix/issues/6649) [#9917](https://github.com/NixOS/nix/pull/9917) + + Previously, if you attempted to evaluate this file with the debugger: + + ```nix + let + a = builtins.trace "before inner break" ( + builtins.break "hello" + ); + b = builtins.trace "before outer break" ( + builtins.break a + ); + in + b + ``` + + Nix would correctly enter the debugger at `builtins.break a`, but if you asked + it to `:continue`, it would skip over the `builtins.break "hello"` expression + entirely. + + Now, Nix will correctly enter the debugger at both breakpoints. + +- Nested debuggers are no longer supported [#9920](https://github.com/NixOS/nix/pull/9920) + + Previously, evaluating an expression that throws an error in the debugger would + enter a second, nested debugger: + + ``` + nix-repl> builtins.throw "what" + error: what + + + Starting REPL to allow you to inspect the current state of the evaluator. + + Welcome to Nix 2.18.1. Type :? for help. + + nix-repl> + ``` + + Now, it just prints the error message like `nix repl`: + + ``` + nix-repl> builtins.throw "what" + error: + … while calling the 'throw' builtin + at «string»:1:1: + 1| builtins.throw "what" + | ^ + + error: what + ``` + +- Consistent order of function arguments in printed expressions [#9874](https://github.com/NixOS/nix/pull/9874) + + Function arguments are now printed in lexicographic order rather than the internal, creation-time based symbol order. + +- Fix duplicate attribute error positions for `inherit` [#9874](https://github.com/NixOS/nix/pull/9874) + + When an `inherit` caused a duplicate attribute error the position of the error was not reported correctly, placing the error with the inherit itself or at the start of the bindings block instead of the offending attribute name. + +- `inherit (x) ...` evaluates `x` only once [#9847](https://github.com/NixOS/nix/pull/9847) + + `inherit (x) a b ...` now evaluates the expression `x` only once for all inherited attributes rather than once for each inherited attribute. + This does not usually have a measurable impact, but side-effects (such as `builtins.trace`) would be duplicated and expensive expressions (such as derivations) could cause a measurable slowdown. + +- Store paths are allowed to start with `.` [#912](https://github.com/NixOS/nix/issues/912) [#9091](https://github.com/NixOS/nix/pull/9091) [#9095](https://github.com/NixOS/nix/pull/9095) [#9120](https://github.com/NixOS/nix/pull/9120) [#9121](https://github.com/NixOS/nix/pull/9121) [#9122](https://github.com/NixOS/nix/pull/9122) [#9130](https://github.com/NixOS/nix/pull/9130) [#9219](https://github.com/NixOS/nix/pull/9219) [#9224](https://github.com/NixOS/nix/pull/9224) [#9867](https://github.com/NixOS/nix/pull/9867) + + Leading periods were allowed by accident in Nix 2.4. The Nix team has considered this to be a bug, but this behavior has since been relied on by users, leading to unnecessary difficulties. + From now on, leading periods are supported. The names `.` and `..` are disallowed, as well as those starting with `.-` or `..-`. + + Nix versions that denied leading periods are documented [in the issue](https://github.com/NixOS/nix/issues/912#issuecomment-1919583286). + +- `nix repl` pretty-prints values [#9931](https://github.com/NixOS/nix/pull/9931) + + `nix repl` will now pretty-print values: + + ``` + { + attrs = { + a = { + b = { + c = { }; + }; + }; + }; + list = [ 1 ]; + list' = [ + 1 + 2 + 3 + ]; + } + ``` + +- Introduction of `--regex` and `--all` in `nix profile remove` and `nix profile upgrade` [#10166](https://github.com/NixOS/nix/pull/10166) + + Previously the command-line arguments for `nix profile remove` and `nix profile upgrade` matched the package entries using regular expression. + For instance: + + ``` + nix profile remove '.*vim.*' + ``` + + This would remove all packages that contain `vim` in their name. + + In most cases, only singular package names were used to remove and upgrade packages. Mixing this with regular expressions sometimes lead to unintended behavior. For instance, `python3.1` could match `python311`. + + To avoid unintended behavior, the arguments are now only matching exact names. + + Matching using regular expressions is still possible by using the new `--regex` flag: + + ``` + nix profile remove --regex '.*vim.*' + ``` + + One of the most useful cases for using regular expressions was to upgrade all packages. This was previously accomplished by: + + ``` + nix profile upgrade '.*' + ``` + + With the introduction of the `--all` flag, this now becomes more straightforward: + + ``` + nix profile upgrade --all + ``` + +- Visual clutter in `--debugger` is reduced [#9919](https://github.com/NixOS/nix/pull/9919) + + Before: + ``` + info: breakpoint reached + + + Starting REPL to allow you to inspect the current state of the evaluator. + + Welcome to Nix 2.20.0pre20231222_dirty. Type :? for help. + + nix-repl> :continue + error: uh oh + + + Starting REPL to allow you to inspect the current state of the evaluator. + + Welcome to Nix 2.20.0pre20231222_dirty. Type :? for help. + + nix-repl> + ``` + + After: + + ``` + info: breakpoint reached + + Nix 2.20.0pre20231222_dirty debugger + Type :? for help. + nix-repl> :continue + error: uh oh + + nix-repl> + ``` + +- Cycle detection in `nix repl` is simpler and more reliable [#8672](https://github.com/NixOS/nix/issues/8672) [#9926](https://github.com/NixOS/nix/pull/9926) + + The cycle detection in `nix repl`, `nix eval`, `builtins.trace`, and everywhere + else values are printed is now simpler and matches the cycle detection in + `nix-instantiate --eval` output. + + Before: + + ``` + nix eval --expr 'let self = { inherit self; }; in self' + { self = { self = «repeated»; }; } + ``` + + After: + + ``` + { self = «repeated»; } + ``` + +- In the debugger, `while evaluating the attribute` errors now include position information [#9915](https://github.com/NixOS/nix/pull/9915) + + Before: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + 0x600001522598 + ``` + + After: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + /nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27 + + 131| + 132| bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs; + | ^ + 133| in + ``` + +- Stack size is increased on macOS [#9860](https://github.com/NixOS/nix/pull/9860) + + Previously, Nix would set the stack size to 64MiB on Linux, but would leave the + stack size set to the default (approximately 8KiB) on macOS. Now, the stack + size is correctly set to 64MiB on macOS as well, which should reduce stack + overflow segfaults in deeply-recursive Nix expressions. + diff --git a/doc/manual/src/release-notes/rl-2.22.md b/doc/manual/src/release-notes/rl-2.22.md new file mode 100644 index 000000000..c78d3d692 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.22.md @@ -0,0 +1,21 @@ +# Release 2.22.0 (2024-04-23) + +### Significant changes + +- Remove experimental repl-flake [#10103](https://github.com/NixOS/nix/issues/10103) [#10299](https://github.com/NixOS/nix/pull/10299) + + The `repl-flake` experimental feature has been removed. The `nix repl` command now works like the rest of the new CLI in that `nix repl {path}` now tries to load a flake at `{path}` (or fails if the `flakes` experimental feature isn't enabled). + +### Other changes + +- `nix eval` prints derivations as `.drv` paths [#10200](https://github.com/NixOS/nix/pull/10200) + + `nix eval` will now print derivations as their `.drv` paths, rather than as + attribute sets. This makes commands like `nix eval nixpkgs#bash` terminate + instead of infinitely looping into recursive self-referential attributes: + + ```ShellSession + $ nix eval nixpkgs#bash + «derivation /nix/store/m32cbgbd598f4w299g0hwyv7gbw6rqcg-bash-5.2p26.drv» + ``` + diff --git a/doc/manual/src/release-notes/rl-2.23.md b/doc/manual/src/release-notes/rl-2.23.md new file mode 100644 index 000000000..ac842fdc0 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.23.md @@ -0,0 +1,102 @@ +# Release 2.23.0 (2024-06-03) + +- New builtin: `builtins.warn` [#306026](https://github.com/NixOS/nix/issues/306026) [#10592](https://github.com/NixOS/nix/pull/10592) + + `builtins.warn` behaves like `builtins.trace "warning: ${msg}"`, has an accurate log level, and is controlled by the options + [`debugger-on-trace`](@docroot@/command-ref/conf-file.md#conf-debugger-on-trace), + [`debugger-on-warn`](@docroot@/command-ref/conf-file.md#conf-debugger-on-warn) and + [`abort-on-warn`](@docroot@/command-ref/conf-file.md#conf-abort-on-warn). + +- Make `nix build --keep-going` consistent with `nix-build --keep-going` + + This means that if e.g. multiple fixed-output derivations fail to + build, all hash mismatches are displayed. + +- Modify `nix derivation {add,show}` JSON format [#9866](https://github.com/NixOS/nix/issues/9866) [#10722](https://github.com/NixOS/nix/pull/10722) + + The JSON format for derivations has been slightly revised to better conform to our [JSON guidelines](@docroot@/development/cli-guideline.md#returning-future-proof-json). + In particular, the hash algorithm and content addressing method of content-addresed derivation outputs are now separated into two fields `hashAlgo` and `method`, + rather than one field with an arcane `:`-separated format. + + This JSON format is only used by the experimental `nix derivation` family of commands, at this time. + Future revisions are expected as the JSON format is still not entirely in compliance even after these changes. + +- Warn on unknown settings anywhere in the command line [#10701](https://github.com/NixOS/nix/pull/10701) + + All `nix` commands will now properly warn when an unknown option is specified anywhere in the command line. + + Before: + + ```console + $ nix-instantiate --option foobar baz --expr '{}' + warning: unknown setting 'foobar' + $ nix-instantiate '{}' --option foobar baz --expr + $ nix eval --expr '{}' --option foobar baz + { } + ``` + + After: + + ```console + $ nix-instantiate --option foobar baz --expr '{}' + warning: unknown setting 'foobar' + $ nix-instantiate '{}' --option foobar baz --expr + warning: unknown setting 'foobar' + $ nix eval --expr '{}' --option foobar baz + warning: unknown setting 'foobar' + { } + ``` + +- `nix env shell` is the new `nix shell`, and `nix shell` remains an accepted alias [#10504](https://github.com/NixOS/nix/issues/10504) [#10807](https://github.com/NixOS/nix/pull/10807) + + This is part of an effort to bring more structure to the CLI subcommands. + + `nix env` will be about the process environment. + Future commands may include `nix env run` and `nix env print-env`. + + It is also somewhat analogous to the [planned](https://github.com/NixOS/nix/issues/10504) `nix dev shell` (currently `nix develop`), which is less about environment variables, and more about running a development shell, which is a more powerful command, but also requires more setup. + +- Flake operations that expect derivations now print the failing value and its type [#10778](https://github.com/NixOS/nix/pull/10778) + + In errors like `flake output attribute 'nixosConfigurations.yuki.config' is not a derivation or path`, the message now includes the failing value and type. + + Before: + + ``` + error: flake output attribute 'nixosConfigurations.yuki.config' is not a derivation or path + ```` + + After: + + ``` + error: expected flake output attribute 'nixosConfigurations.yuki.config' to be a derivation or path but found a set: { appstream = «thunk»; assertions = «thunk»; boot = { bcache = «thunk»; binfmt = «thunk»; binfmtMiscRegistrations = «thunk»; blacklistedKernelModules = «thunk»; bootMount = «thunk»; bootspec = «thunk»; cleanTmpDir = «thunk»; consoleLogLevel = «thunk»; «43 attributes elided» }; «48 attributes elided» } + ``` + +- `fetchTree` now fetches Git repositories shallowly by default [#10028](https://github.com/NixOS/nix/pull/10028) + + `builtins.fetchTree` now clones Git repositories shallowly by default, which reduces network traffic and disk usage significantly in many cases. + + Previously, the default behavior was to clone the full history of a specific tag or branch (e.g. `ref`) and only afterwards extract the files of one specific revision. + + From now on, the `ref` and `allRefs` arguments will be ignored, except if shallow cloning is disabled by setting `shallow = false`. + + The defaults for `builtins.fetchGit` remain unchanged. Here, shallow cloning has to be enabled manually by passing `shallow = true`. + +- Store object info JSON format now uses `null` rather than omitting fields [#9995](https://github.com/NixOS/nix/pull/9995) + + The [store object info JSON format](@docroot@/protocols/json/store-object-info.md), used for e.g. `nix path-info`, no longer omits fields to indicate absent information, but instead includes the fields with a `null` value. + For example, `"ca": null` is used to to indicate a store object that isn't content-addressed rather than omitting the `ca` field entirely. + This makes records of this sort more self-describing, and easier to consume programmatically. + + We will follow this design principle going forward; + the [JSON guidelines](@docroot@/development/json-guideline.md) in the contributing section have been updated accordingly. + +- Large path warnings [#10661](https://github.com/NixOS/nix/pull/10661) + + Nix can now warn when evaluation of a Nix expression causes a large + path to be copied to the Nix store. The threshold for this warning can + be configured using [the `warn-large-path-threshold` + setting](@docroot@/command-ref/conf-file.md#warn-large-path-threshold), + e.g. `--warn-large-path-threshold 100M` will warn about paths larger + than 100 MiB. + diff --git a/doc/manual/src/release-notes/rl-2.24.md b/doc/manual/src/release-notes/rl-2.24.md new file mode 100644 index 000000000..5bcc1d79c --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.24.md @@ -0,0 +1,324 @@ +# Release 2.24.0 (2024-07-31) + +### Significant changes + +- Harden user sandboxing + + The build directory has been hardened against interference with the outside world by nesting it inside another directory owned by (and only readable by) the daemon user. + + This is a low severity security fix, [CVE-2024-38531](https://www.cve.org/CVERecord?id=CVE-2024-38531). + + Credit: [**@alois31**](https://github.com/alois31), [**Linus Heckemann (@lheckemann)**](https://github.com/lheckemann) + Co-authors: [**@edolstra**](https://github.com/edolstra) + +- `nix-shell ` looks for `shell.nix` [#496](https://github.com/NixOS/nix/issues/496) [#2279](https://github.com/NixOS/nix/issues/2279) [#4529](https://github.com/NixOS/nix/issues/4529) [#5431](https://github.com/NixOS/nix/issues/5431) [#11053](https://github.com/NixOS/nix/issues/11053) [#11057](https://github.com/NixOS/nix/pull/11057) + + `nix-shell $x` now looks for `$x/shell.nix` when `$x` resolves to a directory. + + Although this might be seen as a breaking change, its primarily interactive usage makes it a minor issue. + This adjustment addresses a commonly reported problem. + + This also applies to `nix-shell` shebang scripts. Consider the following example: + + ```shell + #!/usr/bin/env nix-shell + #!nix-shell -i bash + ``` + + This will now load `shell.nix` from the script's directory, if it exists; `default.nix` otherwise. + + The old behavior can be opted into by setting the option [`nix-shell-always-looks-for-shell-nix`](@docroot@/command-ref/conf-file.md#conf-nix-shell-always-looks-for-shell-nix) to `false`. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + +- `nix-repl`'s `:doc` shows documentation comments [#3904](https://github.com/NixOS/nix/issues/3904) [#10771](https://github.com/NixOS/nix/issues/10771) [#1652](https://github.com/NixOS/nix/pull/1652) [#9054](https://github.com/NixOS/nix/pull/9054) [#11072](https://github.com/NixOS/nix/pull/11072) + + `nix repl` has a `:doc` command that previously only rendered documentation for internally defined functions. + This feature has been extended to also render function documentation comments, in accordance with [RFC 145]. + + Example: + + ``` + nix-repl> :doc lib.toFunction + Function toFunction + … defined at /home/user/h/nixpkgs/lib/trivial.nix:1072:5 + + Turns any non-callable values into constant functions. Returns + callable values as is. + + Inputs + + v + + : Any value + + Examples + + :::{.example} + + ## lib.trivial.toFunction usage example + + | nix-repl> lib.toFunction 1 2 + | 1 + | + | nix-repl> lib.toFunction (x: x + 1) 2 + | 3 + + ::: + ``` + + Known limitations: + - It does not render documentation for "formals", such as `{ /** the value to return */ x, ... }: x`. + - Some extensions to markdown are not yet supported, as you can see in the example above. + + We'd like to acknowledge [Yingchi Long (@inclyc)](https://github.com/inclyc) for proposing a proof of concept for this functionality in [#9054](https://github.com/NixOS/nix/pull/9054), as well as [@sternenseemann](https://github.com/sternenseemann) and [Johannes Kirschbauer (@hsjobeki)](https://github.com/hsjobeki) for their contributions, proposals, and their work on [RFC 145]. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + + [RFC 145]: https://github.com/NixOS/rfcs/pull/145 + +### Other changes + +- Solve `cached failure of attribute X` [#9165](https://github.com/NixOS/nix/issues/9165) [#10513](https://github.com/NixOS/nix/issues/10513) [#10564](https://github.com/NixOS/nix/pull/10564) + + This eliminates all "cached failure of attribute X" messages by forcing evaluation of the original value when needed to show the exception to the user. This enhancement improves error reporting by providing the underlying message and stack trace. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Run the flake regressions test suite [#10603](https://github.com/NixOS/nix/pull/10603) + + This update introduces a GitHub action to run a subset of the [flake regressions test suite](https://github.com/NixOS/flake-regressions), which includes 259 flakes with their expected evaluation results. Currently, the action runs the first 25 flakes due to the full test suite's extensive runtime. A manually triggered action may be implemented later to run the entire test suite. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Support unit prefixes in configuration settings [#10668](https://github.com/NixOS/nix/pull/10668) + + Configuration settings in Nix now support unit prefixes, allowing for more intuitive and readable configurations. For example, you can now specify [`--min-free 1G`](@docroot@/command-ref/opt-common.md#opt-min-free) to set the minimum free space to 1 gigabyte. + + This enhancement was extracted from [#7851](https://github.com/NixOS/nix/pull/7851) and is also useful for PR [#10661](https://github.com/NixOS/nix/pull/10661). + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- `nix build`: show all FOD errors with `--keep-going` [#10734](https://github.com/NixOS/nix/pull/10734) + + The [`nix build`](@docroot@/command-ref/new-cli/nix3-build.md) command has been updated to improve the behavior of the [`--keep-going`] flag. Now, when `--keep-going` is used, all hash-mismatch errors of failing fixed-output derivations (FODs) are displayed, similar to the behavior for other build failures. This enhancement ensures that all relevant build errors are shown, making it easier for users to update multiple derivations at once or to diagnose and fix issues. + + Author: [**Jörg Thalheim (@Mic92)**](https://github.com/Mic92), [**Maximilian Bosch (@Ma27)**](https://github.com/Ma27) + + [`--keep-going`](@docroot@/command-ref/opt-common.md#opt-keep-going) + +- Build with Meson [#2503](https://github.com/NixOS/nix/issues/2503) [#10378](https://github.com/NixOS/nix/pull/10378) [#10855](https://github.com/NixOS/nix/pull/10855) [#10904](https://github.com/NixOS/nix/pull/10904) [#10908](https://github.com/NixOS/nix/pull/10908) [#10914](https://github.com/NixOS/nix/pull/10914) [#10933](https://github.com/NixOS/nix/pull/10933) [#10936](https://github.com/NixOS/nix/pull/10936) [#10954](https://github.com/NixOS/nix/pull/10954) [#10955](https://github.com/NixOS/nix/pull/10955) [#10963](https://github.com/NixOS/nix/pull/10963) [#10967](https://github.com/NixOS/nix/pull/10967) [#10973](https://github.com/NixOS/nix/pull/10973) [#11034](https://github.com/NixOS/nix/pull/11034) [#11054](https://github.com/NixOS/nix/pull/11054) [#11055](https://github.com/NixOS/nix/pull/11055) [#11060](https://github.com/NixOS/nix/pull/11060) [#11064](https://github.com/NixOS/nix/pull/11064) [#11155](https://github.com/NixOS/nix/pull/11155) + + These changes aim to replace the use of autotools and `make` with Meson for building various components of Nix. Additionally, each library is built in its own derivation, leveraging Meson's "subprojects" feature to allow a single development shell for building all libraries while also supporting separate builds. This approach aims to improve productivity and build modularity, compared to both make and a monolithic Meson-based derivation. + + Special thanks to everyone who has contributed to the Meson port, particularly [**@p01arst0rm**](https://github.com/p01arst0rm) and [**@Qyriad**](https://github.com/Qyriad). + + Authors: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314), [**Tom Bereknyei**](https://github.com/tomberek), [**Théophane Hufschmitt (@thufschmitt)**](https://github.com/thufschmitt), [**Valentin Gagarin (@fricklerhandwerk)**](https://github.com/fricklerhandwerk), [**Robert Hensing (@roberth)**](https://github.com/roberth) + Co-authors: [**@p01arst0rm**](https://github.com/p01arst0rm), [**@Qyriad**](https://github.com/Qyriad) + +- Evaluation cache: fix cache regressions [#10570](https://github.com/NixOS/nix/issues/10570) [#11086](https://github.com/NixOS/nix/pull/11086) + + This update addresses two bugs in the evaluation cache system: + + 1. Regression in #10570: The evaluation cache was not being persisted in `nix develop`. + 2. Nix could sometimes try to commit the evaluation cache SQLite transaction without there being an active transaction, resulting in non-error errors being printed. + + Author: [**Lexi Mattick (@kognise)**](https://github.com/kognise) + +- Introduce `libnixflake` [#9063](https://github.com/NixOS/nix/pull/9063) + + A new library, `libnixflake`, has been introduced to better separate the Flakes layer within Nix. This change refactors the codebase to encapsulate Flakes-specific functionality within its own library. + + See the commits in the pull request for detailed changes, with the only significant code modifications happening in the initial commit. + + This change was alluded to in [RFC 134](https://github.com/nixos/rfcs/blob/master/rfcs/0134-nix-store-layer.md) and is a step towards a more modular and maintainable codebase. + + Author: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314) + +- CLI options `--arg-from-file` and `--arg-from-stdin` [#9913](https://github.com/NixOS/nix/pull/9913) + +- The `--debugger` now prints source location information, instead of the + pointers of source location information. Before: + + ``` + nix-repl> :bt + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + 0x600001522598 + ``` + + After: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + /nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27 + + 131| + 132| bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs; + | ^ + 133| in + ``` + +- Stop vendoring `toml11` + + We don't apply any patches to it, and vendoring it locks users into + bugs (it hasn't been updated since its introduction in late 2021). + + Author: [**Winter (@winterqt)**](https://github.com/winterqt) + +- Rename hash format `base32` to `nix32` [#8678](https://github.com/NixOS/nix/pull/8678) + + Hash format `base32` was renamed to `nix32` since it used a special nix-specific character set for + [Base32](https://en.wikipedia.org/wiki/Base32). + + **Deprecation**: Use `nix32` instead of `base32` as `toHashFormat` + + For the builtin `convertHash`, the `toHashFormat` parameter now accepts the same hash formats as the `--to`/`--from` + parameters of the `nix hash conert` command: `"base16"`, `"nix32"`, `"base64"`, and `"sri"`. The former `"base32"` value + remains as a deprecated alias for `"nix32"`. Please convert your code from: + + ```nix + builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base32";} + ``` + + to + + ```nix + builtins.convertHash { inherit hash hashAlgo; toHashFormat = "nix32";} + ``` + +- Add `pipe-operators` experimental feature [#11131](https://github.com/NixOS/nix/pull/11131) + + This is a draft implementation of [RFC 0148](https://github.com/NixOS/rfcs/pull/148). + + The `pipe-operators` experimental feature adds [`<|` and `|>` operators][pipe operators] to the Nix language. + *a* `|>` *b* is equivalent to the function application *b* *a*, and + *a* `<|` *b* is equivalent to the function application *a* *b*. + + For example: + + ``` + nix-repl> 1 |> builtins.add 2 |> builtins.mul 3 + 9 + + nix-repl> builtins.add 1 <| builtins.mul 2 <| 3 + 7 + ``` + + `<|` and `|>` are right and left associative, respectively, and have lower precedence than any other operator. + These properties may change in future releases. + + See [the RFC](https://github.com/NixOS/rfcs/pull/148) for more examples and rationale. + + [pipe operators]: @docroot@/language/operators.md#pipe-operators + +- `nix-shell` shebang uses relative path [#4232](https://github.com/NixOS/nix/issues/4232) [#5088](https://github.com/NixOS/nix/pull/5088) [#11058](https://github.com/NixOS/nix/pull/11058) + + + Relative [path](@docroot@/language/types.md#type-path) literals in `nix-shell` shebang scripts' options are now resolved relative to the [script's location](@docroot@/glossary.md?highlight=base%20directory#gloss-base-directory). + Previously they were resolved relative to the current working directory. + + For example, consider the following script in `~/myproject/say-hi`: + + ```shell + #!/usr/bin/env nix-shell + #!nix-shell --expr 'import ./shell.nix' + #!nix-shell --arg toolset './greeting-tools.nix' + #!nix-shell -i bash + hello + ``` + + Older versions of `nix-shell` would resolve `shell.nix` relative to the current working directory, such as the user's home directory in this example: + + ```console + [hostname:~]$ ./myproject/say-hi + error: + … while calling the 'import' builtin + at «string»:1:2: + 1| (import ./shell.nix) + | ^ + + error: path '/home/user/shell.nix' does not exist + ``` + + Since this release, `nix-shell` resolves `shell.nix` relative to the script's location, and `~/myproject/shell.nix` is used. + + ```console + $ ./myproject/say-hi + Hello, world! + ``` + + **Opt-out** + + This is technically a breaking change, so we have added an option so you can adapt independently of your Nix update. + The old behavior can be opted into by setting the option [`nix-shell-shebang-arguments-relative-to-script`](@docroot@/command-ref/conf-file.md#conf-nix-shell-shebang-arguments-relative-to-script) to `false`. + This option will be removed in a future release. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + +- Improve handling of tarballs that don't consist of a single top-level directory [#11195](https://github.com/NixOS/nix/pull/11195) + + In previous Nix releases, the tarball fetcher (used by `builtins.fetchTarball`) erroneously merged top-level directories into a single directory, and silently discarded top-level files that are not directories. This is no longer the case. The new behaviour is that *only* if the tarball consists of a single directory, the top-level path component of the files in the tarball is removed (similar to `tar`'s `--strip-components=1`). + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Improve handling of tarballs that don't consist of a single top-level directory [#11195](https://github.com/NixOS/nix/pull/11195) + + In previous Nix releases, the tarball fetcher (used by `builtins.fetchTarball`) erroneously merged top-level directories into a single directory, and silently discarded top-level files that are not directories. This is no longer the case. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Setting to warn about large paths [#10778](https://github.com/NixOS/nix/pull/10778) + + Nix can now warn when evaluation of a Nix expression causes a large + path to be copied to the Nix store. The threshold for this warning can + be configured using the `warn-large-path-threshold` setting, + e.g. `--warn-large-path-threshold 100M`. + + +# Contributors + +This release was made possible by the following 43 contributors: + +- Andreas Rammhold [**(@andir)**](https://github.com/andir) +- Andrew Marshall [**(@amarshall)**](https://github.com/amarshall) +- Brian McKenna [**(@puffnfresh)**](https://github.com/puffnfresh) +- Cameron [**(@SkamDart)**](https://github.com/SkamDart) +- Cole Helbling [**(@cole-h)**](https://github.com/cole-h) +- Corbin Simpson [**(@MostAwesomeDude)**](https://github.com/MostAwesomeDude) +- Eelco Dolstra [**(@edolstra)**](https://github.com/edolstra) +- Emily [**(@emilazy)**](https://github.com/emilazy) +- Enno Richter [**(@elohmeier)**](https://github.com/elohmeier) +- Farid Zakaria [**(@fzakaria)**](https://github.com/fzakaria) +- HaeNoe [**(@haenoe)**](https://github.com/haenoe) +- Hamir Mahal [**(@hamirmahal)**](https://github.com/hamirmahal) +- Harmen [**(@alicebob)**](https://github.com/alicebob) +- Ivan Trubach [**(@tie)**](https://github.com/tie) +- Jared Baur [**(@jmbaur)**](https://github.com/jmbaur) +- John Ericson [**(@Ericson2314)**](https://github.com/Ericson2314) +- Jonathan De Troye [**(@detroyejr)**](https://github.com/detroyejr) +- Jörg Thalheim [**(@Mic92)**](https://github.com/Mic92) +- Klemens Nanni [**(@klemensn)**](https://github.com/klemensn) +- Las Safin [**(@L-as)**](https://github.com/L-as) +- Lexi Mattick [**(@kognise)**](https://github.com/kognise) +- Matthew Bauer [**(@matthewbauer)**](https://github.com/matthewbauer) +- Max “Goldstein†Siling [**(@GoldsteinE)**](https://github.com/GoldsteinE) +- Mingye Wang [**(@Artoria2e5)**](https://github.com/Artoria2e5) +- Philip Taron [**(@philiptaron)**](https://github.com/philiptaron) +- Pierre Bourdon [**(@delroth)**](https://github.com/delroth) +- Pino Toscano [**(@pinotree)**](https://github.com/pinotree) +- RTUnreal [**(@RTUnreal)**](https://github.com/RTUnreal) +- Robert Hensing [**(@roberth)**](https://github.com/roberth) +- Romain Neil [**(@romain-neil)**](https://github.com/romain-neil) +- Ryan Hendrickson [**(@rhendric)**](https://github.com/rhendric) +- Sergei Trofimovich [**(@trofi)**](https://github.com/trofi) +- Shogo Takata [**(@pineapplehunter)**](https://github.com/pineapplehunter) +- Siddhant Kumar [**(@siddhantk232)**](https://github.com/siddhantk232) +- Silvan Mosberger [**(@infinisil)**](https://github.com/infinisil) +- Théophane Hufschmitt [**(@thufschmitt)**](https://github.com/thufschmitt) +- Valentin Gagarin [**(@fricklerhandwerk)**](https://github.com/fricklerhandwerk) +- Winter [**(@winterqt)**](https://github.com/winterqt) +- jade [**(@lf-)**](https://github.com/lf-) +- kirillrdy [**(@kirillrdy)**](https://github.com/kirillrdy) +- pennae [**(@pennae)**](https://github.com/pennae) +- poweredbypie [**(@poweredbypie)**](https://github.com/poweredbypie) +- tomberek [**(@tomberek)**](https://github.com/tomberek) diff --git a/doc/manual/src/release-notes/rl-2.4.md b/doc/manual/src/release-notes/rl-2.4.md index 8b566fc7b..1201e53b6 100644 --- a/doc/manual/src/release-notes/rl-2.4.md +++ b/doc/manual/src/release-notes/rl-2.4.md @@ -23,7 +23,7 @@ more than 2800 commits from 195 contributors since release 2.3. * The **`nix` command** has seen a lot of work and is now almost at feature parity with the old command-line interface (the `nix-*` commands). It aims to be [more modern, consistent and pleasant to - use](../contributing/cli-guideline.md) than the old CLI. It is still + use](../development/cli-guideline.md) than the old CLI. It is still marked as experimental but its interface should not change much anymore in future releases. diff --git a/doc/manual/src/store/file-system-object/content-address.md b/doc/manual/src/store/file-system-object/content-address.md new file mode 100644 index 000000000..410d7fb7c --- /dev/null +++ b/doc/manual/src/store/file-system-object/content-address.md @@ -0,0 +1,85 @@ +# Content-Addressing File System Objects + +For many operations, Nix needs to calculate [a content addresses](@docroot@/glossary.md#gloss-content-address) of [a file system object][file system object]. +Usually this is needed as part of +[content addressing store objects](../store-object/content-address.md), +since store objects always have a root file system object. +But some command-line utilities also just work on "raw" file system objects, not part of any store object. + +Every content addressing scheme Nix uses ultimately involves feeding data into a [hash function](https://en.wikipedia.org/wiki/Hash_function), and getting back an opaque fixed-size digest which is deemed a content address. +The various *methods* of content addressing thus differ in how abstract data (in this case, a file system object and its descendents) are fed into the hash function. + +## Serialising File System Objects { #serial } + +The simplest method is to serialise the entire file system object tree into a single binary string, and then hash that binary string, yielding the content address. +In this section we describe the currently-supported methods of serialising file system objects. + +### Flat { #serial-flat } + +A single file object can just be hashed by its contents. +This is not enough information to encode the fact that the file system object is a file, +but if we *already* know that the FSO is a single non-executable file by other means, it is sufficient. + +Because the hashed data is just the raw file, as is, this choice is good for compatibility with other systems. +For example, Unix commands like `sha256sum` or `sha1sum` will produce hashes for single files that match this. + +### Nix Archive (NAR) { #serial-nix-archive } + +For the other cases of [file system objects][file system object], especially directories with arbitrary descendents, we need a more complex serialisation format. +Examples of such serialisations are the ZIP and TAR file formats. +However, for our purposes these formats have two problems: + +- They do not have a canonical serialisation, meaning that given an FSO, there can +be many different serialisations. + For instance, TAR files can have variable amounts of padding between archive members; + and some archive formats leave the order of directory entries undefined. + This would be bad because we use serialisation to compute cryptographic hashes over file system objects, and for those hashes to be useful as a content address or for integrity checking, uniqueness is crucial. + Otherwise, correct hashes would report false mismatches, and the store would fail to find the content. + +- They store more information than we have in our notion of FSOs, such as time stamps. + This can cause FSOs that Nix should consider equal to hash to different values on different machines, just because the dates differ. + +- As a practical consideration, the TAR format is the only truly universal format in the Unix environment. + It has many problems, such as an inability to deal with long file names and files larger than 2^33 bytes. + Current implementations such as GNU Tar work around these limitations in various ways. + +For these reasons, Nix has its very own archive format—the Nix Archive (NAR) format, +which is carefully designed to avoid the problems described above. + +The exact specification of the Nix Archive format is in `protocols/nix-archive.md` + +## Content addressing File System Objects beyond a single serialisation pass + +Serialising the entire tree and then hashing that binary string is not the only option for content addressing, however. +Another technique is that of a [Merkle graph](https://en.wikipedia.org/wiki/Merkle_tree), where previously computed hashes are included in subsequent byte strings to be hashed. + +In particular, the Merkle graphs can match the original graph structure of file system objects: +we can first hash (serialised) child file system objects, and then hash parent objects using the hashes of their children in the serialisation (to be hashed) of the parent file system objects. + +Currently, there is one such Merkle DAG content addressing method supported. + +### Git ([experimental][xp-feature-git-hashing]) { #git } + +> **Warning** +> +> This method is part of the [`git-hashing`][xp-feature-git-hashing] experimental feature. + +Git's file system model is very close to Nix's, and so Git's content addressing method is a pretty good fit. +Just as with regular Git, files and symlinks are hashed as git "blobs", and directories are hashed as git "trees". + +However, one difference between Nix's and Git's file system model needs special treatment. +Plain files, executable files, and symlinks are not differentiated as distinctly addressable objects, but by their context: by the directory entry that refers to them. +That means so long as the root object is a directory, there is no problem: +every non-directory object is owned by a parent directory, and the entry that refers to it provides the missing information. +However, if the root object is not a directory, then we have no way of knowing which one of an executable file, non-executable file, or symlink it is supposed to be. + +In response to this, we have decided to treat a bare file as non-executable file. +This is similar to do what we do with [flat serialisation](#serial-flat), which also lacks this information. +To avoid an address collision, attempts to hash a bare executable file or symlink will result in an error (just as would happen for flat serialisation also). +Thus, Git can encode some, but not all of Nix's "File System Objects", and this sort of content-addressing is likewise partial. + +In the future, we may support a Git-like hash for such file system objects, or we may adopt another Merkle DAG format which is capable of representing all Nix file system objects. + +[file system object]: ../file-system-object.md +[store object]: ../store-object.md +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/doc/manual/src/store/store-object/content-address.md b/doc/manual/src/store/store-object/content-address.md new file mode 100644 index 000000000..02dce2836 --- /dev/null +++ b/doc/manual/src/store/store-object/content-address.md @@ -0,0 +1,95 @@ +# Content-Addressing Store Objects + +Just [like][fso-ca] [File System Objects][File System Object], +[Store Objects][Store Object] can also be [content-addressed](@docroot@/glossary.md#gloss-content-addressed), +unless they are [input-addressed](@docroot@/glossary.md#gloss-input-addressed-store-object). + +For store objects, the content address we produce will take the form of a [Store Path] rather than regular hash. +In particular, the content-addressing scheme will ensure that the digest of the store path is solely computed from the + +- file system object graph (the root one and its children, if it has any) +- references +- [store directory](../store-path.md#store-directory) +- name + +of the store object, and not any other information, which would not be an intrinsic property of that store object. + +For the full specification of the algorithms involved, see the [specification of store path digests][sp-spec]. + +[File System Object]: ../file-system-object.md +[Store Object]: ../store-object.md +[Store Path]: ../store-path.md + +## Content addressing each part of a store object + +### File System Objects + +With all currently supported store object content addressing methods, the file system object is always [content-addressed][fso-ca] first, and then that hash is incorporated into content address computation for the store object. + +### References + +With all currently supported store object content addressing methods, +other objects are referred to by their regular (string-encoded-) [store paths][Store Path]. + +Self-references however cannot be referred to by their path, because we are in the midst of describing how to compute that path! + +> The alternative would require finding as hash function fixed point, i.e. the solution to an equation in the form +> ``` +> digest = hash(..... || digest || ....) +> ``` +> which is computationally infeasible. +> As far as we know, this is equivalent to finding a hash collision. + +Instead we just have a "has self reference" boolean, which will end up affecting the digest. + +### Name and Store Directory + +These two items affect the digest in a way that is standard for store path digest computations and not specific to content-addressing. +Consult the [specification of store path digests][sp-spec] for further details. + +## Content addressing Methods + +For historical reasons, we don't support all features in all combinations. +Each currently supported method of content addressing chooses a single method of file system object hashing, and may offer some restrictions on references. +The names and store directories are unrestricted however. + +### Flat { #method-flat } + +This uses the corresponding [Flat](../file-system-object/content-address.md#serial-flat) method of file system object content addressing. + +References are not supported: store objects with flat hashing *and* references can not be created. + +### Text { #method-text } + +This also uses the corresponding [Flat](../file-system-object/content-address.md#serial-flat) method of file system object content addressing. + +References to other store objects are supported, but self references are not. + +This is the only store-object content-addressing method that is not named identically with a corresponding file system object method. +It is somewhat obscure, mainly used for "drv files" +(derivations serialized as store objects in their ["ATerm" file format](@docroot@/protocols/derivation-aterm.md)). +Prefer another method if possible. + +### Nix Archive { #method-nix-archive } + +This uses the corresponding [Nix Archive](../file-system-object/content-address.md#serial-nix-archive) method of file system object content addressing. + +References (to other store objects and self references alike) are supported so long as the hash algorithm is SHA-256, but not (neither kind) otherwise. + +### Git { #method-git } + +> **Warning** +> +> This method is part of the [`git-hashing`][xp-feature-git-hashing] experimental feature. + +This uses the corresponding [Git](../file-system-object/content-address.md#serial-git) method of file system object content addressing. + +References are not supported. + +Only SHA-1 is supported at this time. +If [SHA-256-based Git](https://git-scm.com/docs/hash-function-transition) +becomes more widespread, this restriction will be revisited. + +[fso-ca]: ../file-system-object/content-address.md +[sp-spec]: @docroot@/protocols/store-path.md +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/doc/manual/src/store/store-path.md b/doc/manual/src/store/store-path.md index b5ad0c654..beec2389b 100644 --- a/doc/manual/src/store/store-path.md +++ b/doc/manual/src/store/store-path.md @@ -1,5 +1,11 @@ # Store Path +> **Example** +> +> `/nix/store/a040m110amc4h71lds2jmr8qrkj2jhxd-git-2.38.1` +> +> A rendered store path + Nix implements references to [store objects](./index.md#store-object) as *store paths*. Think of a store path as an [opaque], [unique identifier]: @@ -37,6 +43,10 @@ A store path is rendered to a file system path as the concatenation of > store directory digest name > ``` +Exactly how the digest is calculated depends on the type of store path. +Store path digests are *supposed* to be opaque, and so for most operations, it is not necessary to know the details. +That said, the manual has a full [specification of store path digests](@docroot@/protocols/store-path.md). + ## Store Directory Every [Nix store](./index.md) has a store directory. @@ -46,7 +56,7 @@ But if the store has a file system representation, the store directory contains [file system objects]: ./file-system-object.md -This means a store path is not just derived from the referenced store object itself, but depends on the store the store object is in. +This means a store path is not just derived from the referenced store object itself, but depends on the store that the store object is in. > **Note** > diff --git a/flake.lock b/flake.lock index bb2e400c0..2ac413a69 100644 --- a/flake.lock +++ b/flake.lock @@ -3,11 +3,11 @@ "flake-compat": { "flake": false, "locked": { - "lastModified": 1673956053, - "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", "owner": "edolstra", "repo": "flake-compat", - "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", "type": "github" }, "original": { @@ -16,38 +16,100 @@ "type": "github" } }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1719994518, + "narHash": "sha256-pQMhCCHyQGRzdfAkdJ4cIWiw+JNuWsTX7f0ZYSyz0VY=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "9227223f6d922fee3c7b190b2cc238a99527bbb7", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "git-hooks-nix": { + "inputs": { + "flake-compat": [], + "gitignore": [], + "nixpkgs": [ + "nixpkgs" + ], + "nixpkgs-stable": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1721042469, + "narHash": "sha256-6FPUl7HVtvRHCCBQne7Ylp4p+dpP3P/OYuzjztZ4s70=", + "owner": "cachix", + "repo": "git-hooks.nix", + "rev": "f451c19376071a90d8c58ab1a953c6e9840527fd", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "git-hooks.nix", + "type": "github" + } + }, "libgit2": { "flake": false, "locked": { - "lastModified": 1697646580, - "narHash": "sha256-oX4Z3S9WtJlwvj0uH9HlYcWv+x1hqp8mhXl7HsLu2f0=", + "lastModified": 1715853528, + "narHash": "sha256-J2rCxTecyLbbDdsyBWn9w7r3pbKRMkI9E7RvRgAqBdY=", "owner": "libgit2", "repo": "libgit2", - "rev": "45fd9ed7ae1a9b74b957ef4f337bc3c8b3df01b5", + "rev": "36f7e21ad757a3dacc58cf7944329da6bc1d6e96", "type": "github" }, "original": { "owner": "libgit2", + "ref": "v1.8.1", "repo": "libgit2", "type": "github" } }, "nixpkgs": { "locked": { - "lastModified": 1709083642, - "narHash": "sha256-7kkJQd4rZ+vFrzWu8sTRtta5D1kBG0LSRYAfhtmMlSo=", + "lastModified": 1721548954, + "narHash": "sha256-7cCC8+Tdq1+3OPyc3+gVo9dzUNkNIQfwSDJ2HSi2u3o=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "b550fe4b4776908ac2a861124307045f8e717c8e", + "rev": "63d37ccd2d178d54e7fb691d7ec76000740ea24a", "type": "github" }, "original": { "owner": "NixOS", - "ref": "release-23.11", + "ref": "nixos-24.05", "repo": "nixpkgs", "type": "github" } }, + "nixpkgs-23-11": { + "locked": { + "lastModified": 1717159533, + "narHash": "sha256-oamiKNfr2MS6yH64rUn99mIZjc45nGJlj9eGth/3Xuw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446", + "type": "github" + } + }, "nixpkgs-regression": { "locked": { "lastModified": 1643052045, @@ -67,8 +129,11 @@ "root": { "inputs": { "flake-compat": "flake-compat", + "flake-parts": "flake-parts", + "git-hooks-nix": "git-hooks-nix", "libgit2": "libgit2", "nixpkgs": "nixpkgs", + "nixpkgs-23-11": "nixpkgs-23-11", "nixpkgs-regression": "nixpkgs-regression" } } diff --git a/flake.nix b/flake.nix index 42aaace67..9e8592e3a 100644 --- a/flake.nix +++ b/flake.nix @@ -1,18 +1,28 @@ { description = "The purely functional package manager"; - # TODO switch to nixos-23.11-small - # https://nixpk.gs/pr-tracker.html?pr=291954 - inputs.nixpkgs.url = "github:NixOS/nixpkgs/release-23.11"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; + inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446"; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; - inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; }; + inputs.libgit2 = { url = "github:libgit2/libgit2/v1.8.1"; flake = false; }; + + # dev tooling + inputs.flake-parts.url = "github:hercules-ci/flake-parts"; + inputs.git-hooks-nix.url = "github:cachix/git-hooks.nix"; + # work around https://github.com/NixOS/nix/issues/7730 + inputs.flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; + inputs.git-hooks-nix.inputs.nixpkgs.follows = "nixpkgs"; + inputs.git-hooks-nix.inputs.nixpkgs-stable.follows = "nixpkgs"; + # work around 7730 and https://github.com/NixOS/nix/issues/7807 + inputs.git-hooks-nix.inputs.flake-compat.follows = ""; + inputs.git-hooks-nix.inputs.gitignore.follows = ""; + + outputs = inputs@{ self, nixpkgs, nixpkgs-regression, libgit2, ... }: - outputs = { self, nixpkgs, nixpkgs-regression, libgit2, ... }: let inherit (nixpkgs) lib; - inherit (lib) fileset; officialRelease = false; @@ -31,14 +41,9 @@ crossSystems = [ "armv6l-unknown-linux-gnueabihf" "armv7l-unknown-linux-gnueabihf" - "x86_64-unknown-freebsd13" + "riscv64-unknown-linux-gnu" "x86_64-unknown-netbsd" - ]; - - # Nix doesn't yet build on this platform, so we put it in a - # separate list. We just use this for `devShells` and - # `nixpkgsFor`, which this depends on. - shellCrossSystems = crossSystems ++ [ + "x86_64-unknown-freebsd" "x86_64-w64-mingw32" ]; @@ -50,6 +55,18 @@ "stdenv" ]; + /** + `flatMapAttrs attrs f` applies `f` to each attribute in `attrs` and + merges the results into a single attribute set. + + This can be nested to form a build matrix where all the attributes + generated by the innermost `f` are returned as is. + (Provided that the names are unique.) + + See https://nixos.org/manual/nixpkgs/stable/index.html#function-library-lib.attrsets.concatMapAttrs + */ + flatMapAttrs = attrs: f: lib.concatMapAttrs f attrs; + forAllSystems = lib.genAttrs systems; forAllCrossSystems = lib.genAttrs crossSystems; @@ -63,6 +80,17 @@ }) stdenvs); + + # We don't apply flake-parts to the whole flake so that non-development attributes + # load without fetching any development inputs. + devFlake = inputs.flake-parts.lib.mkFlake { inherit inputs; } { + imports = [ ./maintainers/flake-module.nix ]; + systems = lib.subtractLists crossSystems systems; + perSystem = { system, ... }: { + _module.args.pkgs = nixpkgsFor.${system}.native; + }; + }; + # Memoize nixpkgs for different platforms for efficiency. nixpkgsFor = forAllSystems (system: let @@ -84,31 +112,9 @@ in { inherit stdenvs native; static = native.pkgsStatic; - cross = lib.genAttrs shellCrossSystems (crossSystem: make-pkgs crossSystem "stdenv"); + cross = forAllCrossSystems (crossSystem: make-pkgs crossSystem "stdenv"); }); - installScriptFor = tarballs: - nixpkgsFor.x86_64-linux.native.callPackage ./scripts/installer.nix { - inherit tarballs; - }; - - testNixVersions = pkgs: client: daemon: - pkgs.callPackage ./package.nix { - pname = - "nix-tests" - + lib.optionalString - (lib.versionAtLeast daemon.version "2.4pre20211005" && - lib.versionAtLeast client.version "2.4pre20211005") - "-${client.version}-against-${daemon.version}"; - - inherit fileset; - - test-client = client; - test-daemon = daemon; - - doBuild = false; - }; - binaryTarball = nix: pkgs: pkgs.callPackage ./scripts/binary-tarball.nix { inherit nix; }; @@ -120,244 +126,130 @@ { nixStable = prev.nix; - default-busybox-sandbox-shell = final.busybox.override { - useMusl = true; - enableStatic = true; - enableMinimal = true; - extraConfig = '' - CONFIG_FEATURE_FANCY_ECHO y - CONFIG_FEATURE_SH_MATH y - CONFIG_FEATURE_SH_MATH_64 y + # A new scope, so that we can use `callPackage` to inject our own interdependencies + # without "polluting" the top level "`pkgs`" attrset. + # This also has the benefit of providing us with a distinct set of packages + # we can iterate over. + nixComponents = lib.makeScope final.nixDependencies.newScope (import ./packaging/components.nix); - CONFIG_ASH y - CONFIG_ASH_OPTIMIZE_FOR_SIZE y - - CONFIG_ASH_ALIAS y - CONFIG_ASH_BASH_COMPAT y - CONFIG_ASH_CMDCMD y - CONFIG_ASH_ECHO y - CONFIG_ASH_GETOPTS y - CONFIG_ASH_INTERNAL_GLOB y - CONFIG_ASH_JOB_CONTROL y - CONFIG_ASH_PRINTF y - CONFIG_ASH_TEST y - ''; - }; - - libgit2-nix = final.libgit2.overrideAttrs (attrs: { - src = libgit2; - version = libgit2.lastModifiedDate; - cmakeFlags = attrs.cmakeFlags or [] - ++ [ "-DUSE_SSH=exec" ]; + # The dependencies are in their own scope, so that they don't have to be + # in Nixpkgs top level `pkgs` or `nixComponents`. + nixDependencies = lib.makeScope final.newScope (import ./packaging/dependencies.nix { + inherit inputs stdenv versionSuffix; + pkgs = final; }); - boehmgc-nix = (final.boehmgc.override { - enableLargeConfig = true; - }).overrideAttrs(o: { - patches = (o.patches or []) ++ [ - ./dep-patches/boehmgc-coroutine-sp-fallback.diff + nix = final.nixComponents.nix; - # https://github.com/ivmai/bdwgc/pull/586 - ./dep-patches/boehmgc-traceable_allocator-public.diff - ]; - }); - - changelog-d-nix = final.buildPackages.callPackage ./misc/changelog-d.nix { }; - - nix = - let - officialRelease = false; - versionSuffix = - if officialRelease - then "" - else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}"; - - in final.callPackage ./package.nix { - inherit - fileset - stdenv - versionSuffix - ; - officialRelease = false; - boehmgc = final.boehmgc-nix; - libgit2 = final.libgit2-nix; - busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell; - } // { - # this is a proper separate downstream package, but put - # here also for back compat reasons. - perl-bindings = final.nix-perl-bindings; - }; - - nix-perl-bindings = final.callPackage ./perl { - inherit fileset stdenv; + nix_noTests = final.nix.override { + doInstallCheck = false; + doCheck = false; }; + # See https://github.com/NixOS/nixpkgs/pull/214409 + # Remove when fixed in this flake's nixpkgs + pre-commit = + if prev.stdenv.hostPlatform.system == "i686-linux" + then (prev.pre-commit.override (o: { dotnet-sdk = ""; })).overridePythonAttrs (o: { doCheck = false; }) + else prev.pre-commit; + }; in { # A Nixpkgs overlay that overrides the 'nix' and - # 'nix.perl-bindings' packages. + # 'nix-perl-bindings' packages. overlays.default = overlayFor (p: p.stdenv); - hydraJobs = { - - # Binary package for various platforms. - build = forAllSystems (system: self.packages.${system}.nix); - - shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation); - - buildStatic = lib.genAttrs linux64BitSystems (system: self.packages.${system}.nix-static); - - buildCross = forAllCrossSystems (crossSystem: - lib.genAttrs ["x86_64-linux"] (system: self.packages.${system}."nix-${crossSystem}")); - - buildNoGc = forAllSystems (system: - self.packages.${system}.nix.override { enableGC = false; } - ); - - buildNoTests = forAllSystems (system: - self.packages.${system}.nix.override { - doCheck = false; - doInstallCheck = false; - installUnitTests = false; - } - ); - - # Toggles some settings for better coverage. Windows needs these - # library combinations, and Debian build Nix with GNU readline too. - buildReadlineNoMarkdown = forAllSystems (system: - self.packages.${system}.nix.override { - enableMarkdown = false; - readlineFlavor = "readline"; - } - ); - - # Perl bindings for various platforms. - perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nix.perl-bindings); - - # Binary tarball for various platforms, containing a Nix store - # with the closure of 'nix' package, and the second half of - # the installation script. - binaryTarball = forAllSystems (system: binaryTarball nixpkgsFor.${system}.native.nix nixpkgsFor.${system}.native); - - binaryTarballCross = lib.genAttrs ["x86_64-linux"] (system: - forAllCrossSystems (crossSystem: - binaryTarball - self.packages.${system}."nix-${crossSystem}" - nixpkgsFor.${system}.cross.${crossSystem})); - - # The first half of the installation script. This is uploaded - # to https://nixos.org/nix/install. It downloads the binary - # tarball for the user's system and calls the second half of the - # installation script. - installerScript = installScriptFor [ - # Native - self.hydraJobs.binaryTarball."x86_64-linux" - self.hydraJobs.binaryTarball."i686-linux" - self.hydraJobs.binaryTarball."aarch64-linux" - self.hydraJobs.binaryTarball."x86_64-darwin" - self.hydraJobs.binaryTarball."aarch64-darwin" - # Cross - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" - ]; - installerScriptForGHA = installScriptFor [ - # Native - self.hydraJobs.binaryTarball."x86_64-linux" - self.hydraJobs.binaryTarball."x86_64-darwin" - # Cross - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" - ]; - - # docker image with Nix inside - dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); - - # Line coverage analysis. - coverage = nixpkgsFor.x86_64-linux.native.nix.override { - pname = "nix-coverage"; - withCoverageChecks = true; - }; - - # API docs for Nix's unstable internal C++ interfaces. - internal-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ./package.nix { - inherit fileset; - doBuild = false; - enableInternalAPIDocs = true; - }; - - # System tests. - tests = import ./tests/nixos { inherit lib nixpkgs nixpkgsFor; } // { - - # Make sure that nix-env still produces the exact same result - # on a particular version of Nixpkgs. - evalNixpkgs = - let - inherit (nixpkgsFor.x86_64-linux.native) runCommand nix; - in - runCommand "eval-nixos" { buildInputs = [ nix ]; } - '' - type -p nix-env - # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593. - time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages - [[ $(sha1sum < packages | cut -c1-40) = ff451c521e61e4fe72bdbe2d0ca5d1809affa733 ]] - mkdir $out - ''; - - nixpkgsLibTests = - forAllSystems (system: - import (nixpkgs + "/lib/tests/release.nix") - { pkgs = nixpkgsFor.${system}.native; - nixVersions = [ self.packages.${system}.nix ]; - } - ); - }; - - metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" { - pkgs = nixpkgsFor.x86_64-linux.native; - nixpkgs = nixpkgs-regression; - }; - - installTests = forAllSystems (system: - let pkgs = nixpkgsFor.${system}.native; in - pkgs.runCommand "install-tests" { - againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix; - againstCurrentUnstable = - # FIXME: temporarily disable this on macOS because of #3605. - if system == "x86_64-linux" - then testNixVersions pkgs pkgs.nix pkgs.nixUnstable - else null; - # Disabled because the latest stable version doesn't handle - # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work - # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable; - } "touch $out"); - - installerTests = import ./tests/installer { - binaryTarballs = self.hydraJobs.binaryTarball; - inherit nixpkgsFor; - }; - + hydraJobs = import ./packaging/hydra.nix { + inherit + inputs + binaryTarball + forAllCrossSystems + forAllSystems + lib + linux64BitSystems + nixpkgsFor + self + ; }; checks = forAllSystems (system: { binaryTarball = self.hydraJobs.binaryTarball.${system}; - perlBindings = self.hydraJobs.perlBindings.${system}; installTests = self.hydraJobs.installTests.${system}; nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system}; rl-next = let pkgs = nixpkgsFor.${system}.native; in pkgs.buildPackages.runCommand "test-rl-next-release-notes" { } '' - LANG=C.UTF-8 ${pkgs.changelog-d-nix}/bin/changelog-d ${./doc/manual/rl-next} >$out + LANG=C.UTF-8 ${pkgs.changelog-d}/bin/changelog-d ${./doc/manual/rl-next} >$out ''; + repl-completion = nixpkgsFor.${system}.native.callPackage ./tests/repl-completion.nix { }; } // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) { dockerImage = self.hydraJobs.dockerImage.${system}; - }); + } // (lib.optionalAttrs (!(builtins.elem system linux32BitSystems))) { + # Some perl dependencies are broken on i686-linux. + # Since the support is only best-effort there, disable the perl + # bindings - packages = forAllSystems (system: rec { - inherit (nixpkgsFor.${system}.native) nix changelog-d-nix; - default = nix; - } // (lib.optionalAttrs (builtins.elem system linux64BitSystems) { - nix-static = nixpkgsFor.${system}.static.nix; + # Temporarily disabled because GitHub Actions OOM issues. Once + # the old build system is gone and we are back to one build + # system, we should reenable this. + #perlBindings = self.hydraJobs.perlBindings.${system}; + } + # Add "passthru" tests + // flatMapAttrs ({ + "" = nixpkgsFor.${system}.native; + } // lib.optionalAttrs (! nixpkgsFor.${system}.native.stdenv.hostPlatform.isDarwin) { + # TODO: enable static builds for darwin, blocked on: + # https://github.com/NixOS/nixpkgs/issues/320448 + "static-" = nixpkgsFor.${system}.static; + }) + (nixpkgsPrefix: nixpkgs: + flatMapAttrs nixpkgs.nixComponents + (pkgName: pkg: + flatMapAttrs pkg.tests or {} + (testName: test: { + "${nixpkgsPrefix}${pkgName}-${testName}" = test; + }) + ) + ) + // devFlake.checks.${system} or {} + ); + + packages = forAllSystems (system: + { # Here we put attributes that map 1:1 into packages., ie + # for which we don't apply the full build matrix such as cross or static. + inherit (nixpkgsFor.${system}.native) + changelog-d; + default = self.packages.${system}.nix; + nix-internal-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-internal-api-docs; + nix-external-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-external-api-docs; + } + # We need to flatten recursive attribute sets of derivations to pass `flake check`. + // flatMapAttrs + { # Components we'll iterate over in the upcoming lambda + "nix" = { }; + # Temporarily disabled because GitHub Actions OOM issues. Once + # the old build system is gone and we are back to one build + # system, we should reenable these. + #"nix-util" = { }; + #"nix-store" = { }; + #"nix-fetchers" = { }; + } + (pkgName: {}: { + # These attributes go right into `packages.`. + "${pkgName}" = nixpkgsFor.${system}.native.nixComponents.${pkgName}; + "${pkgName}-static" = nixpkgsFor.${system}.static.nixComponents.${pkgName}; + } + // flatMapAttrs (lib.genAttrs crossSystems (_: { })) (crossSystem: {}: { + # These attributes go right into `packages.`. + "${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName}; + }) + // flatMapAttrs (lib.genAttrs stdenvs (_: { })) (stdenvName: {}: { + # These attributes go right into `packages.`. + "${pkgName}-${stdenvName}" = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nixComponents.${pkgName}; + }) + ) + // lib.optionalAttrs (builtins.elem system linux64BitSystems) { dockerImage = let pkgs = nixpkgsFor.${system}.native; @@ -372,21 +264,27 @@ ln -s ${image} $image echo "file binary-dist $image" >> $out/nix-support/hydra-build-products ''; - } // builtins.listToAttrs (map - (crossSystem: { - name = "nix-${crossSystem}"; - value = nixpkgsFor.${system}.cross.${crossSystem}.nix; - }) - crossSystems) - // builtins.listToAttrs (map - (stdenvName: { - name = "nix-${stdenvName}"; - value = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nix; - }) - stdenvs))); + }); devShells = let - makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; forDevShell = true; }).overrideAttrs (attrs: { + makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; forDevShell = true; }).overrideAttrs (attrs: + let + modular = devFlake.getSystem stdenv.buildPlatform.system; + transformFlag = prefix: flag: + assert builtins.isString flag; + let + rest = builtins.substring 2 (builtins.stringLength flag) flag; + in + "-D${prefix}:${rest}"; + havePerl = stdenv.buildPlatform == stdenv.hostPlatform && stdenv.hostPlatform.isUnix; + ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags; + in { + pname = "shell-for-" + attrs.pname; + + # Remove the version suffix to avoid unnecessary attempts to substitute in nix develop + version = lib.fileContents ./.version; + name = attrs.pname; + installFlags = "sysconfdir=$(out)/etc"; shellHook = '' PATH=$prefix/bin:$PATH @@ -397,11 +295,62 @@ XDG_DATA_DIRS+=:$out/share ''; + # We use this shell with the local checkout, not unpackPhase. + src = null; + + env = { + # Needed for Meson to find Boost. + # https://github.com/NixOS/nixpkgs/issues/86131. + BOOST_INCLUDEDIR = "${lib.getDev pkgs.nixDependencies.boost}/include"; + BOOST_LIBRARYDIR = "${lib.getLib pkgs.nixDependencies.boost}/lib"; + # For `make format`, to work without installing pre-commit + _NIX_PRE_COMMIT_HOOKS_CONFIG = + "${(pkgs.formats.yaml { }).generate "pre-commit-config.yaml" modular.pre-commit.settings.rawConfig}"; + }; + + mesonFlags = + map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents.nix-util.mesonFlags) + ++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents.nix-store.mesonFlags) + ++ map (transformFlag "libfetchers") (ignoreCrossFile pkgs.nixComponents.nix-fetchers.mesonFlags) + ++ lib.optionals havePerl (map (transformFlag "perl") (ignoreCrossFile pkgs.nixComponents.nix-perl-bindings.mesonFlags)) + ++ map (transformFlag "libexpr") (ignoreCrossFile pkgs.nixComponents.nix-expr.mesonFlags) + ++ map (transformFlag "libcmd") (ignoreCrossFile pkgs.nixComponents.nix-cmd.mesonFlags) + ; + nativeBuildInputs = attrs.nativeBuildInputs or [] + ++ pkgs.nixComponents.nix-util.nativeBuildInputs + ++ pkgs.nixComponents.nix-store.nativeBuildInputs + ++ pkgs.nixComponents.nix-fetchers.nativeBuildInputs + ++ lib.optionals havePerl pkgs.nixComponents.nix-perl-bindings.nativeBuildInputs + ++ pkgs.nixComponents.nix-internal-api-docs.nativeBuildInputs + ++ pkgs.nixComponents.nix-external-api-docs.nativeBuildInputs + ++ lib.optional + (!stdenv.buildPlatform.canExecute stdenv.hostPlatform + # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479 + && !(stdenv.hostPlatform.isWindows && stdenv.buildPlatform.isDarwin) + && stdenv.hostPlatform.emulatorAvailable pkgs.buildPackages + && lib.meta.availableOn stdenv.buildPlatform (stdenv.hostPlatform.emulator pkgs.buildPackages)) + pkgs.buildPackages.mesonEmulatorHook + ++ [ + pkgs.buildPackages.cmake + pkgs.buildPackages.shellcheck + pkgs.buildPackages.changelog-d + modular.pre-commit.settings.package + (pkgs.writeScriptBin "pre-commit-hooks-install" + modular.pre-commit.settings.installationScript) + ] # TODO: Remove the darwin check once # https://github.com/NixOS/nixpkgs/pull/291814 is available ++ lib.optional (stdenv.cc.isClang && !stdenv.buildPlatform.isDarwin) pkgs.buildPackages.bear ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) pkgs.buildPackages.clang-tools; + + buildInputs = attrs.buildInputs or [] + ++ [ + pkgs.gtest + pkgs.rapidcheck + ] + ++ lib.optional havePerl pkgs.perl + ; }); in forAllSystems (system: @@ -413,8 +362,8 @@ in (makeShells "native" nixpkgsFor.${system}.native) // (lib.optionalAttrs (!nixpkgsFor.${system}.native.stdenv.isDarwin) - (makeShells "static" nixpkgsFor.${system}.static)) // - (lib.genAttrs shellCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv)) // + (makeShells "static" nixpkgsFor.${system}.static) // + (forAllCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv))) // { default = self.devShells.${system}.native-stdenvPackages; } diff --git a/local.mk b/local.mk index 3f3abb9f0..b27c7031e 100644 --- a/local.mk +++ b/local.mk @@ -2,9 +2,14 @@ GLOBAL_CXXFLAGS += -Wno-deprecated-declarations -Werror=switch # Allow switch-enum to be overridden for files that do not support it, usually because of dependency headers. ERROR_SWITCH_ENUM = -Werror=switch-enum -$(foreach i, config.h $(wildcard src/lib*/*.hh), \ +$(foreach i, config.h $(wildcard src/lib*/*.hh) $(filter-out %_internal.h, $(wildcard src/lib*c/*.h)), \ $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) +ifdef HOST_UNIX + $(foreach i, $(wildcard src/lib*/unix/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) +endif + $(GCH): src/libutil/util.hh config.h -GCH_CXXFLAGS = -I src/libutil +GCH_CXXFLAGS = $(INCLUDE_libutil) diff --git a/m4/gcc_bug_80431.m4 b/m4/gcc_bug_80431.m4 index e42f01956..cdc4ddb40 100644 --- a/m4/gcc_bug_80431.m4 +++ b/m4/gcc_bug_80431.m4 @@ -46,11 +46,13 @@ AC_DEFUN([ENSURE_NO_GCC_BUG_80431], ]])], [status_80431=0], [status_80431=$?], - [ - # Assume we're bug-free when cross-compiling - ]) + [status_80431='']) AC_LANG_POP(C++) AS_CASE([$status_80431], + [''],[ + AC_MSG_RESULT(cannot check because cross compiling) + AC_MSG_NOTICE(assume we are bug free) + ], [0],[ AC_MSG_RESULT(yes) ], diff --git a/maintainers/README.md b/maintainers/README.md index fa321c7c0..b92833497 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -30,17 +30,18 @@ We aim to achieve this by improving the contributor experience and attracting mo ## Members - Eelco Dolstra (@edolstra) – Team lead -- Théophane Hufschmitt (@thufschmitt) - Valentin Gagarin (@fricklerhandwerk) - Thomas Bereknyei (@tomberek) - Robert Hensing (@roberth) - John Ericson (@Ericson2314) +The team is on Github as [@NixOS/nix-team](https://github.com/orgs/NixOS/teams/nix-team). + ## Meeting protocol -The team meets twice a week: +The team meets twice a week (times are denoted in the [Europe/Amsterdam](https://en.m.wikipedia.org/wiki/Time_in_the_Netherlands) time zone): -- Discussion meeting: [Fridays 13:00-14:00 CET](https://calendar.google.com/calendar/event?eid=MHNtOGVuNWtrZXNpZHR2bW1sM3QyN2ZjaGNfMjAyMjExMjVUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) +- Discussion meeting: [Wednesday 21:00-22:00 Europe/Amsterdam](https://www.google.com/calendar/event?eid=ZG5rZzNyajRjajducGV2NGY5aGkzYWIwdnJfMjAyNDA1MDhUMTkwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) 1. Triage issues and pull requests from the [No Status](#no-status) column (30 min) 2. Discuss issues and pull requests from the [To discuss](#to-discuss) column (30 min). @@ -49,15 +50,19 @@ The team meets twice a week: - mark it as draft if it is blocked on the contributor - escalate it back to the team by moving it to To discuss, and leaving a comment as to why the issue needs to be discussed again. -- Work meeting: [Mondays 13:00-15:00 CET](https://calendar.google.com/calendar/event?eid=NTM1MG1wNGJnOGpmOTZhYms3bTB1bnY5cWxfMjAyMjExMjFUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) +- Work meeting: [Mondays 14:00-16:00 Europe/Amsterdam](https://www.google.com/calendar/event?eid=Ym52NDdzYnRic2NzcDcybjZiNDhpNzhpa3NfMjAyNDA1MTNUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) 1. Code review on pull requests from [In review](#in-review). 2. Other chores and tasks. Meeting notes are collected on a [collaborative scratchpad](https://pad.lassul.us/Cv7FpYx-Ri-4VjUykQOLAw). -Notes on issues and pull requests are posted as comments and linked from the meeting notes, so they are easy to find from both places. +Notes on issues and pull requests are posted as comments and linked from the meeting notes, so they are can be found from both places. [All meeting notes](https://discourse.nixos.org/search?expanded=true&q=Nix%20team%20meeting%20minutes%20%23%20%23dev%3Anix%20in%3Atitle%20order%3Alatest_topic) are published on Discourse under the [Nix category](https://discourse.nixos.org/c/dev/nix/50). +Team meetings are generally open to anyone interested. +We can make exceptions to discuss sensitive issues, such as security incidents or people matters. +Contact any team member to get a calendar invite for reminders and updates. + ## Project board protocol The team uses a [GitHub project board](https://github.com/orgs/NixOS/projects/19/views/1) for tracking its work. diff --git a/maintainers/data/release-credits-email-to-handle.json b/maintainers/data/release-credits-email-to-handle.json new file mode 100644 index 000000000..cddc1a6e7 --- /dev/null +++ b/maintainers/data/release-credits-email-to-handle.json @@ -0,0 +1,52 @@ +{ + "bogus": "bogus", + "edolstra@gmail.com": "edolstra", + "roberth@users.noreply.github.com": "roberth", + "toscano.pino@tiscali.it": "pinotree", + "valentin@gagarin.work": "fricklerhandwerk", + "mr.trubach@icloud.com": "tie", + "robert@roberthensing.nl": "roberth", + "lix@jade.fyi": "lf-", + "cole.e.helbling@outlook.com": "cole-h", + "joerg@thalheim.io": "Mic92", + "John.Ericson@Obsidian.Systems": "Ericson2314", + "ryan.hendrickson@alum.mit.edu": "rhendric", + "67135060+poweredbypie@users.noreply.github.com": "poweredbypie", + "detroyejr@outlook.com": "detroyejr", + "silvan.mosberger@tweag.io": "infinisil", + "vcs@emily.moe": "emilazy", + "farid.m.zakaria@gmail.com": "fzakaria", + "22859658+RTUnreal@users.noreply.github.com": "RTUnreal", + "me@las.rs": "L-as", + "philip.taron@gmail.com": "philiptaron", + "root@goldstein.rs": "GoldsteinE", + "tomberek@users.noreply.github.com": "tomberek", + "lexi.mattick@neuralink.com": "kognise", + "andrew@johnandrewmarshall.com": "amarshall", + "contact@romain-neil.fr": "romain-neil", + "Mic92@users.noreply.github.com": "Mic92", + "valentin.gagarin@tweag.io": "fricklerhandwerk", + "siddhantk232@gmail.com": "siddhantk232", + "kn@openbsd.org": "klemensn", + "slyich@gmail.com": "trofi", + "theophane.hufschmitt@tweag.io": "thufschmitt", + "alicebob@lijzij.de": "alicebob", + "winter@winter.cafe": "winterqt", + "brian@brianmckenna.org": "puffnfresh", + "git@haenoe.party": "haenoe", + "peshogo@gmail.com": "pineapplehunter", + "poweredbypie@users.noreply.github.com": "poweredbypie", + "arthur200126@gmail.com": "Artoria2e5", + "tomberek@gmail.com": "tomberek", + "jaredbaur@fastmail.com": "jmbaur", + "andreas@rammhold.de": "andir", + "hamirmahal@gmail.com": "hamirmahal", + "git@JohnEricson.me": "Ericson2314", + "8763518+SkamDart@users.noreply.github.com": "SkamDart", + "kirillrdy@gmail.com": "kirillrdy", + "pennae@lix.systems": "pennae", + "delroth@gmail.com": "delroth", + "enno@nerdworks.de": "elohmeier", + "mjbauer95@gmail.com": "matthewbauer", + "MostAwesomeDude@gmail.com": "MostAwesomeDude" +} \ No newline at end of file diff --git a/maintainers/data/release-credits-handle-to-name.json b/maintainers/data/release-credits-handle-to-name.json new file mode 100644 index 000000000..abf9ed05b --- /dev/null +++ b/maintainers/data/release-credits-handle-to-name.json @@ -0,0 +1,45 @@ +{ + "fzakaria": "Farid Zakaria", + "kognise": "Lexi Mattick", + "L-as": "Las Safin", + "haenoe": "HaeNoe", + "andir": "Andreas Rammhold", + "matthewbauer": "Matthew Bauer", + "emilazy": "Emily", + "pineapplehunter": "Shogo Takata", + "RTUnreal": null, + "jmbaur": "Jared Baur", + "Ericson2314": "John Ericson", + "pinotree": "Pino Toscano", + "tie": "Ivan Trubach", + "poweredbypie": null, + "fricklerhandwerk": "Valentin Gagarin", + "Mic92": "J\u00f6rg Thalheim", + "alicebob": "Harmen", + "elohmeier": "Enno Richter", + "delroth": "Pierre Bourdon", + "kirillrdy": null, + "thufschmitt": "Th\u00e9ophane Hufschmitt", + "detroyejr": "Jonathan De Troye", + "klemensn": "Klemens Nanni", + "tomberek": null, + "rhendric": "Ryan Hendrickson", + "philiptaron": "Philip Taron", + "puffnfresh": "Brian McKenna", + "lf-": "jade", + "romain-neil": "Romain Neil", + "hamirmahal": "Hamir Mahal", + "edolstra": "Eelco Dolstra", + "Artoria2e5": "Mingye Wang", + "SkamDart": "Cameron", + "roberth": "Robert Hensing", + "amarshall": "Andrew Marshall", + "trofi": "Sergei Trofimovich", + "cole-h": "Cole Helbling", + "infinisil": "Silvan Mosberger", + "siddhantk232": "Siddhant Kumar", + "winterqt": "Winter", + "GoldsteinE": "Max \u201cGoldstein\u201d Siling", + "pennae": null, + "MostAwesomeDude": "Corbin Simpson" +} \ No newline at end of file diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix new file mode 100644 index 000000000..be91df536 --- /dev/null +++ b/maintainers/flake-module.nix @@ -0,0 +1,677 @@ +{ lib, getSystem, inputs, ... }: + +{ + imports = [ + inputs.git-hooks-nix.flakeModule + ]; + + perSystem = { config, pkgs, ... }: { + + # https://flake.parts/options/pre-commit-hooks-nix.html#options + pre-commit.settings = { + hooks = { + clang-format = { + enable = true; + excludes = [ + # We don't want to format test data + # ''tests/(?!nixos/).*\.nix'' + ''^tests/unit/[^/]*/data/.*$'' + + # Don't format vendored code + ''^doc/manual/redirects\.js$'' + ''^doc/manual/theme/highlight\.js$'' + + # We haven't applied formatting to these files yet + ''^doc/manual/redirects\.js$'' + ''^doc/manual/theme/highlight\.js$'' + ''^precompiled-headers\.h$'' + ''^src/build-remote/build-remote\.cc$'' + ''^src/libcmd/built-path\.cc$'' + ''^src/libcmd/built-path\.hh$'' + ''^src/libcmd/command\.cc$'' + ''^src/libcmd/command\.hh$'' + ''^src/libcmd/common-eval-args\.cc$'' + ''^src/libcmd/common-eval-args\.hh$'' + ''^src/libcmd/editor-for\.cc$'' + ''^src/libcmd/installable-attr-path\.cc$'' + ''^src/libcmd/installable-attr-path\.hh$'' + ''^src/libcmd/installable-derived-path\.cc$'' + ''^src/libcmd/installable-derived-path\.hh$'' + ''^src/libcmd/installable-flake\.cc$'' + ''^src/libcmd/installable-flake\.hh$'' + ''^src/libcmd/installable-value\.cc$'' + ''^src/libcmd/installable-value\.hh$'' + ''^src/libcmd/installables\.cc$'' + ''^src/libcmd/installables\.hh$'' + ''^src/libcmd/legacy\.hh$'' + ''^src/libcmd/markdown\.cc$'' + ''^src/libcmd/misc-store-flags\.cc$'' + ''^src/libcmd/repl-interacter\.cc$'' + ''^src/libcmd/repl-interacter\.hh$'' + ''^src/libcmd/repl\.cc$'' + ''^src/libcmd/repl\.hh$'' + ''^src/libexpr-c/nix_api_expr\.cc$'' + ''^src/libexpr-c/nix_api_external\.cc$'' + ''^src/libexpr/attr-path\.cc$'' + ''^src/libexpr/attr-path\.hh$'' + ''^src/libexpr/attr-set\.cc$'' + ''^src/libexpr/attr-set\.hh$'' + ''^src/libexpr/eval-cache\.cc$'' + ''^src/libexpr/eval-cache\.hh$'' + ''^src/libexpr/eval-error\.cc$'' + ''^src/libexpr/eval-inline\.hh$'' + ''^src/libexpr/eval-settings\.cc$'' + ''^src/libexpr/eval-settings\.hh$'' + ''^src/libexpr/eval\.cc$'' + ''^src/libexpr/eval\.hh$'' + ''^src/libexpr/function-trace\.cc$'' + ''^src/libexpr/gc-small-vector\.hh$'' + ''^src/libexpr/get-drvs\.cc$'' + ''^src/libexpr/get-drvs\.hh$'' + ''^src/libexpr/json-to-value\.cc$'' + ''^src/libexpr/nixexpr\.cc$'' + ''^src/libexpr/nixexpr\.hh$'' + ''^src/libexpr/parser-state\.hh$'' + ''^src/libexpr/pos-table\.hh$'' + ''^src/libexpr/primops\.cc$'' + ''^src/libexpr/primops\.hh$'' + ''^src/libexpr/primops/context\.cc$'' + ''^src/libexpr/primops/fetchClosure\.cc$'' + ''^src/libexpr/primops/fetchMercurial\.cc$'' + ''^src/libexpr/primops/fetchTree\.cc$'' + ''^src/libexpr/primops/fromTOML\.cc$'' + ''^src/libexpr/print-ambiguous\.cc$'' + ''^src/libexpr/print-ambiguous\.hh$'' + ''^src/libexpr/print-options\.hh$'' + ''^src/libexpr/print\.cc$'' + ''^src/libexpr/print\.hh$'' + ''^src/libexpr/search-path\.cc$'' + ''^src/libexpr/symbol-table\.hh$'' + ''^src/libexpr/value-to-json\.cc$'' + ''^src/libexpr/value-to-json\.hh$'' + ''^src/libexpr/value-to-xml\.cc$'' + ''^src/libexpr/value-to-xml\.hh$'' + ''^src/libexpr/value\.hh$'' + ''^src/libexpr/value/context\.cc$'' + ''^src/libexpr/value/context\.hh$'' + ''^src/libfetchers/attrs\.cc$'' + ''^src/libfetchers/cache\.cc$'' + ''^src/libfetchers/cache\.hh$'' + ''^src/libfetchers/fetch-settings\.cc$'' + ''^src/libfetchers/fetch-settings\.hh$'' + ''^src/libfetchers/fetch-to-store\.cc$'' + ''^src/libfetchers/fetchers\.cc$'' + ''^src/libfetchers/fetchers\.hh$'' + ''^src/libfetchers/filtering-source-accessor\.cc$'' + ''^src/libfetchers/filtering-source-accessor\.hh$'' + ''^src/libfetchers/fs-source-accessor\.cc$'' + ''^src/libfetchers/fs-source-accessor\.hh$'' + ''^src/libfetchers/git-utils\.cc$'' + ''^src/libfetchers/git-utils\.hh$'' + ''^src/libfetchers/github\.cc$'' + ''^src/libfetchers/indirect\.cc$'' + ''^src/libfetchers/memory-source-accessor\.cc$'' + ''^src/libfetchers/path\.cc$'' + ''^src/libfetchers/registry\.cc$'' + ''^src/libfetchers/registry\.hh$'' + ''^src/libfetchers/tarball\.cc$'' + ''^src/libfetchers/tarball\.hh$'' + ''^src/libfetchers/git\.cc$'' + ''^src/libfetchers/mercurial\.cc$'' + ''^src/libflake/flake/config\.cc$'' + ''^src/libflake/flake/flake\.cc$'' + ''^src/libflake/flake/flake\.hh$'' + ''^src/libflake/flake/flakeref\.cc$'' + ''^src/libflake/flake/flakeref\.hh$'' + ''^src/libflake/flake/lockfile\.cc$'' + ''^src/libflake/flake/lockfile\.hh$'' + ''^src/libflake/flake/url-name\.cc$'' + ''^src/libmain/common-args\.cc$'' + ''^src/libmain/common-args\.hh$'' + ''^src/libmain/loggers\.cc$'' + ''^src/libmain/loggers\.hh$'' + ''^src/libmain/progress-bar\.cc$'' + ''^src/libmain/shared\.cc$'' + ''^src/libmain/shared\.hh$'' + ''^src/libmain/unix/stack\.cc$'' + ''^src/libstore/binary-cache-store\.cc$'' + ''^src/libstore/binary-cache-store\.hh$'' + ''^src/libstore/build-result\.hh$'' + ''^src/libstore/builtins\.hh$'' + ''^src/libstore/builtins/buildenv\.cc$'' + ''^src/libstore/builtins/buildenv\.hh$'' + ''^src/libstore/common-protocol-impl\.hh$'' + ''^src/libstore/common-protocol\.cc$'' + ''^src/libstore/common-protocol\.hh$'' + ''^src/libstore/common-ssh-store-config\.hh$'' + ''^src/libstore/content-address\.cc$'' + ''^src/libstore/content-address\.hh$'' + ''^src/libstore/daemon\.cc$'' + ''^src/libstore/daemon\.hh$'' + ''^src/libstore/derivations\.cc$'' + ''^src/libstore/derivations\.hh$'' + ''^src/libstore/derived-path-map\.cc$'' + ''^src/libstore/derived-path-map\.hh$'' + ''^src/libstore/derived-path\.cc$'' + ''^src/libstore/derived-path\.hh$'' + ''^src/libstore/downstream-placeholder\.cc$'' + ''^src/libstore/downstream-placeholder\.hh$'' + ''^src/libstore/dummy-store\.cc$'' + ''^src/libstore/export-import\.cc$'' + ''^src/libstore/filetransfer\.cc$'' + ''^src/libstore/filetransfer\.hh$'' + ''^src/libstore/gc-store\.hh$'' + ''^src/libstore/globals\.cc$'' + ''^src/libstore/globals\.hh$'' + ''^src/libstore/http-binary-cache-store\.cc$'' + ''^src/libstore/legacy-ssh-store\.cc$'' + ''^src/libstore/legacy-ssh-store\.hh$'' + ''^src/libstore/length-prefixed-protocol-helper\.hh$'' + ''^src/libstore/linux/personality\.cc$'' + ''^src/libstore/linux/personality\.hh$'' + ''^src/libstore/local-binary-cache-store\.cc$'' + ''^src/libstore/local-fs-store\.cc$'' + ''^src/libstore/local-fs-store\.hh$'' + ''^src/libstore/log-store\.cc$'' + ''^src/libstore/log-store\.hh$'' + ''^src/libstore/machines\.cc$'' + ''^src/libstore/machines\.hh$'' + ''^src/libstore/make-content-addressed\.cc$'' + ''^src/libstore/make-content-addressed\.hh$'' + ''^src/libstore/misc\.cc$'' + ''^src/libstore/names\.cc$'' + ''^src/libstore/names\.hh$'' + ''^src/libstore/nar-accessor\.cc$'' + ''^src/libstore/nar-accessor\.hh$'' + ''^src/libstore/nar-info-disk-cache\.cc$'' + ''^src/libstore/nar-info-disk-cache\.hh$'' + ''^src/libstore/nar-info\.cc$'' + ''^src/libstore/nar-info\.hh$'' + ''^src/libstore/outputs-spec\.cc$'' + ''^src/libstore/outputs-spec\.hh$'' + ''^src/libstore/parsed-derivations\.cc$'' + ''^src/libstore/path-info\.cc$'' + ''^src/libstore/path-info\.hh$'' + ''^src/libstore/path-references\.cc$'' + ''^src/libstore/path-regex\.hh$'' + ''^src/libstore/path-with-outputs\.cc$'' + ''^src/libstore/path\.cc$'' + ''^src/libstore/path\.hh$'' + ''^src/libstore/pathlocks\.cc$'' + ''^src/libstore/pathlocks\.hh$'' + ''^src/libstore/profiles\.cc$'' + ''^src/libstore/profiles\.hh$'' + ''^src/libstore/realisation\.cc$'' + ''^src/libstore/realisation\.hh$'' + ''^src/libstore/remote-fs-accessor\.cc$'' + ''^src/libstore/remote-fs-accessor\.hh$'' + ''^src/libstore/remote-store-connection\.hh$'' + ''^src/libstore/remote-store\.cc$'' + ''^src/libstore/remote-store\.hh$'' + ''^src/libstore/s3-binary-cache-store\.cc$'' + ''^src/libstore/s3\.hh$'' + ''^src/libstore/serve-protocol-impl\.cc$'' + ''^src/libstore/serve-protocol-impl\.hh$'' + ''^src/libstore/serve-protocol\.cc$'' + ''^src/libstore/serve-protocol\.hh$'' + ''^src/libstore/sqlite\.cc$'' + ''^src/libstore/sqlite\.hh$'' + ''^src/libstore/ssh-store\.cc$'' + ''^src/libstore/ssh\.cc$'' + ''^src/libstore/ssh\.hh$'' + ''^src/libstore/store-api\.cc$'' + ''^src/libstore/store-api\.hh$'' + ''^src/libstore/store-dir-config\.hh$'' + ''^src/libstore/build/derivation-goal\.cc$'' + ''^src/libstore/build/derivation-goal\.hh$'' + ''^src/libstore/build/drv-output-substitution-goal\.cc$'' + ''^src/libstore/build/drv-output-substitution-goal\.hh$'' + ''^src/libstore/build/entry-points\.cc$'' + ''^src/libstore/build/goal\.cc$'' + ''^src/libstore/build/goal\.hh$'' + ''^src/libstore/unix/build/hook-instance\.cc$'' + ''^src/libstore/unix/build/local-derivation-goal\.cc$'' + ''^src/libstore/unix/build/local-derivation-goal\.hh$'' + ''^src/libstore/build/substitution-goal\.cc$'' + ''^src/libstore/build/substitution-goal\.hh$'' + ''^src/libstore/build/worker\.cc$'' + ''^src/libstore/build/worker\.hh$'' + ''^src/libstore/builtins/fetchurl\.cc$'' + ''^src/libstore/builtins/unpack-channel\.cc$'' + ''^src/libstore/gc\.cc$'' + ''^src/libstore/local-overlay-store\.cc$'' + ''^src/libstore/local-overlay-store\.hh$'' + ''^src/libstore/local-store\.cc$'' + ''^src/libstore/local-store\.hh$'' + ''^src/libstore/unix/user-lock\.cc$'' + ''^src/libstore/unix/user-lock\.hh$'' + ''^src/libstore/optimise-store\.cc$'' + ''^src/libstore/unix/pathlocks\.cc$'' + ''^src/libstore/posix-fs-canonicalise\.cc$'' + ''^src/libstore/posix-fs-canonicalise\.hh$'' + ''^src/libstore/uds-remote-store\.cc$'' + ''^src/libstore/uds-remote-store\.hh$'' + ''^src/libstore/windows/build\.cc$'' + ''^src/libstore/worker-protocol-impl\.hh$'' + ''^src/libstore/worker-protocol\.cc$'' + ''^src/libstore/worker-protocol\.hh$'' + ''^src/libutil-c/nix_api_util_internal\.h$'' + ''^src/libutil/archive\.cc$'' + ''^src/libutil/archive\.hh$'' + ''^src/libutil/args\.cc$'' + ''^src/libutil/args\.hh$'' + ''^src/libutil/args/root\.hh$'' + ''^src/libutil/callback\.hh$'' + ''^src/libutil/canon-path\.cc$'' + ''^src/libutil/canon-path\.hh$'' + ''^src/libutil/chunked-vector\.hh$'' + ''^src/libutil/closure\.hh$'' + ''^src/libutil/comparator\.hh$'' + ''^src/libutil/compute-levels\.cc$'' + ''^src/libutil/config-impl\.hh$'' + ''^src/libutil/config\.cc$'' + ''^src/libutil/config\.hh$'' + ''^src/libutil/current-process\.cc$'' + ''^src/libutil/current-process\.hh$'' + ''^src/libutil/english\.cc$'' + ''^src/libutil/english\.hh$'' + ''^src/libutil/environment-variables\.cc$'' + ''^src/libutil/error\.cc$'' + ''^src/libutil/error\.hh$'' + ''^src/libutil/exit\.hh$'' + ''^src/libutil/experimental-features\.cc$'' + ''^src/libutil/experimental-features\.hh$'' + ''^src/libutil/file-content-address\.cc$'' + ''^src/libutil/file-content-address\.hh$'' + ''^src/libutil/file-descriptor\.cc$'' + ''^src/libutil/file-descriptor\.hh$'' + ''^src/libutil/file-path-impl\.hh$'' + ''^src/libutil/file-path\.hh$'' + ''^src/libutil/file-system\.cc$'' + ''^src/libutil/file-system\.hh$'' + ''^src/libutil/finally\.hh$'' + ''^src/libutil/fmt\.hh$'' + ''^src/libutil/fs-sink\.cc$'' + ''^src/libutil/fs-sink\.hh$'' + ''^src/libutil/git\.cc$'' + ''^src/libutil/git\.hh$'' + ''^src/libutil/hash\.cc$'' + ''^src/libutil/hash\.hh$'' + ''^src/libutil/hilite\.cc$'' + ''^src/libutil/hilite\.hh$'' + ''^src/libutil/source-accessor\.hh$'' + ''^src/libutil/json-impls\.hh$'' + ''^src/libutil/json-utils\.cc$'' + ''^src/libutil/json-utils\.hh$'' + ''^src/libutil/linux/cgroup\.cc$'' + ''^src/libutil/linux/namespaces\.cc$'' + ''^src/libutil/logging\.cc$'' + ''^src/libutil/logging\.hh$'' + ''^src/libutil/lru-cache\.hh$'' + ''^src/libutil/memory-source-accessor\.cc$'' + ''^src/libutil/memory-source-accessor\.hh$'' + ''^src/libutil/pool\.hh$'' + ''^src/libutil/position\.cc$'' + ''^src/libutil/position\.hh$'' + ''^src/libutil/posix-source-accessor\.cc$'' + ''^src/libutil/posix-source-accessor\.hh$'' + ''^src/libutil/processes\.hh$'' + ''^src/libutil/ref\.hh$'' + ''^src/libutil/references\.cc$'' + ''^src/libutil/references\.hh$'' + ''^src/libutil/regex-combinators\.hh$'' + ''^src/libutil/serialise\.cc$'' + ''^src/libutil/serialise\.hh$'' + ''^src/libutil/signals\.hh$'' + ''^src/libutil/signature/local-keys\.cc$'' + ''^src/libutil/signature/local-keys\.hh$'' + ''^src/libutil/signature/signer\.cc$'' + ''^src/libutil/signature/signer\.hh$'' + ''^src/libutil/source-accessor\.cc$'' + ''^src/libutil/source-accessor\.hh$'' + ''^src/libutil/source-path\.cc$'' + ''^src/libutil/source-path\.hh$'' + ''^src/libutil/split\.hh$'' + ''^src/libutil/suggestions\.cc$'' + ''^src/libutil/suggestions\.hh$'' + ''^src/libutil/sync\.hh$'' + ''^src/libutil/terminal\.cc$'' + ''^src/libutil/terminal\.hh$'' + ''^src/libutil/thread-pool\.cc$'' + ''^src/libutil/thread-pool\.hh$'' + ''^src/libutil/topo-sort\.hh$'' + ''^src/libutil/types\.hh$'' + ''^src/libutil/unix/file-descriptor\.cc$'' + ''^src/libutil/unix/file-path\.cc$'' + ''^src/libutil/unix/monitor-fd\.hh$'' + ''^src/libutil/unix/processes\.cc$'' + ''^src/libutil/unix/signals-impl\.hh$'' + ''^src/libutil/unix/signals\.cc$'' + ''^src/libutil/unix-domain-socket\.cc$'' + ''^src/libutil/unix/users\.cc$'' + ''^src/libutil/url-parts\.hh$'' + ''^src/libutil/url\.cc$'' + ''^src/libutil/url\.hh$'' + ''^src/libutil/users\.cc$'' + ''^src/libutil/users\.hh$'' + ''^src/libutil/util\.cc$'' + ''^src/libutil/util\.hh$'' + ''^src/libutil/variant-wrapper\.hh$'' + ''^src/libutil/windows/environment-variables\.cc$'' + ''^src/libutil/windows/file-descriptor\.cc$'' + ''^src/libutil/windows/file-path\.cc$'' + ''^src/libutil/windows/processes\.cc$'' + ''^src/libutil/windows/users\.cc$'' + ''^src/libutil/windows/windows-error\.cc$'' + ''^src/libutil/windows/windows-error\.hh$'' + ''^src/libutil/xml-writer\.cc$'' + ''^src/libutil/xml-writer\.hh$'' + ''^src/nix-build/nix-build\.cc$'' + ''^src/nix-channel/nix-channel\.cc$'' + ''^src/nix-collect-garbage/nix-collect-garbage\.cc$'' + ''^src/nix-env/buildenv.nix$'' + ''^src/nix-env/nix-env\.cc$'' + ''^src/nix-env/user-env\.cc$'' + ''^src/nix-env/user-env\.hh$'' + ''^src/nix-instantiate/nix-instantiate\.cc$'' + ''^src/nix-store/dotgraph\.cc$'' + ''^src/nix-store/graphml\.cc$'' + ''^src/nix-store/nix-store\.cc$'' + ''^src/nix/add-to-store\.cc$'' + ''^src/nix/app\.cc$'' + ''^src/nix/build\.cc$'' + ''^src/nix/bundle\.cc$'' + ''^src/nix/cat\.cc$'' + ''^src/nix/config-check\.cc$'' + ''^src/nix/config\.cc$'' + ''^src/nix/copy\.cc$'' + ''^src/nix/derivation-add\.cc$'' + ''^src/nix/derivation-show\.cc$'' + ''^src/nix/derivation\.cc$'' + ''^src/nix/develop\.cc$'' + ''^src/nix/diff-closures\.cc$'' + ''^src/nix/dump-path\.cc$'' + ''^src/nix/edit\.cc$'' + ''^src/nix/eval\.cc$'' + ''^src/nix/flake\.cc$'' + ''^src/nix/fmt\.cc$'' + ''^src/nix/hash\.cc$'' + ''^src/nix/log\.cc$'' + ''^src/nix/ls\.cc$'' + ''^src/nix/main\.cc$'' + ''^src/nix/make-content-addressed\.cc$'' + ''^src/nix/nar\.cc$'' + ''^src/nix/optimise-store\.cc$'' + ''^src/nix/path-from-hash-part\.cc$'' + ''^src/nix/path-info\.cc$'' + ''^src/nix/prefetch\.cc$'' + ''^src/nix/profile\.cc$'' + ''^src/nix/realisation\.cc$'' + ''^src/nix/registry\.cc$'' + ''^src/nix/repl\.cc$'' + ''^src/nix/run\.cc$'' + ''^src/nix/run\.hh$'' + ''^src/nix/search\.cc$'' + ''^src/nix/sigs\.cc$'' + ''^src/nix/store-copy-log\.cc$'' + ''^src/nix/store-delete\.cc$'' + ''^src/nix/store-gc\.cc$'' + ''^src/nix/store-info\.cc$'' + ''^src/nix/store-repair\.cc$'' + ''^src/nix/store\.cc$'' + ''^src/nix/unix/daemon\.cc$'' + ''^src/nix/upgrade-nix\.cc$'' + ''^src/nix/verify\.cc$'' + ''^src/nix/why-depends\.cc$'' + + ''^tests/functional/plugins/plugintest\.cc'' + ''^tests/functional/test-libstoreconsumer/main\.cc'' + ''^tests/nixos/ca-fd-leak/sender\.c'' + ''^tests/nixos/ca-fd-leak/smuggler\.c'' + ''^tests/nixos/user-sandboxing/attacker\.c'' + ''^tests/unit/libexpr-support/tests/libexpr\.hh'' + ''^tests/unit/libexpr-support/tests/value/context\.cc'' + ''^tests/unit/libexpr-support/tests/value/context\.hh'' + ''^tests/unit/libexpr/derived-path\.cc'' + ''^tests/unit/libexpr/error_traces\.cc'' + ''^tests/unit/libexpr/eval\.cc'' + ''^tests/unit/libexpr/json\.cc'' + ''^tests/unit/libexpr/main\.cc'' + ''^tests/unit/libexpr/primops\.cc'' + ''^tests/unit/libexpr/search-path\.cc'' + ''^tests/unit/libexpr/trivial\.cc'' + ''^tests/unit/libexpr/value/context\.cc'' + ''^tests/unit/libexpr/value/print\.cc'' + ''^tests/unit/libfetchers/public-key\.cc'' + ''^tests/unit/libflake/flakeref\.cc'' + ''^tests/unit/libflake/url-name\.cc'' + ''^tests/unit/libstore-support/tests/derived-path\.cc'' + ''^tests/unit/libstore-support/tests/derived-path\.hh'' + ''^tests/unit/libstore-support/tests/nix_api_store\.hh'' + ''^tests/unit/libstore-support/tests/outputs-spec\.cc'' + ''^tests/unit/libstore-support/tests/outputs-spec\.hh'' + ''^tests/unit/libstore-support/tests/path\.cc'' + ''^tests/unit/libstore-support/tests/path\.hh'' + ''^tests/unit/libstore-support/tests/protocol\.hh'' + ''^tests/unit/libstore/common-protocol\.cc'' + ''^tests/unit/libstore/content-address\.cc'' + ''^tests/unit/libstore/derivation\.cc'' + ''^tests/unit/libstore/derived-path\.cc'' + ''^tests/unit/libstore/downstream-placeholder\.cc'' + ''^tests/unit/libstore/machines\.cc'' + ''^tests/unit/libstore/nar-info-disk-cache\.cc'' + ''^tests/unit/libstore/nar-info\.cc'' + ''^tests/unit/libstore/outputs-spec\.cc'' + ''^tests/unit/libstore/path-info\.cc'' + ''^tests/unit/libstore/path\.cc'' + ''^tests/unit/libstore/serve-protocol\.cc'' + ''^tests/unit/libstore/worker-protocol\.cc'' + ''^tests/unit/libutil-support/tests/characterization\.hh'' + ''^tests/unit/libutil-support/tests/hash\.cc'' + ''^tests/unit/libutil-support/tests/hash\.hh'' + ''^tests/unit/libutil/args\.cc'' + ''^tests/unit/libutil/canon-path\.cc'' + ''^tests/unit/libutil/chunked-vector\.cc'' + ''^tests/unit/libutil/closure\.cc'' + ''^tests/unit/libutil/compression\.cc'' + ''^tests/unit/libutil/config\.cc'' + ''^tests/unit/libutil/file-content-address\.cc'' + ''^tests/unit/libutil/git\.cc'' + ''^tests/unit/libutil/hash\.cc'' + ''^tests/unit/libutil/hilite\.cc'' + ''^tests/unit/libutil/json-utils\.cc'' + ''^tests/unit/libutil/logging\.cc'' + ''^tests/unit/libutil/lru-cache\.cc'' + ''^tests/unit/libutil/pool\.cc'' + ''^tests/unit/libutil/references\.cc'' + ''^tests/unit/libutil/suggestions\.cc'' + ''^tests/unit/libutil/tests\.cc'' + ''^tests/unit/libutil/url\.cc'' + ''^tests/unit/libutil/xml-writer\.cc'' + ]; + }; + shellcheck = { + enable = true; + excludes = [ + # We haven't linted these files yet + ''^config/install-sh$'' + ''^misc/bash/completion\.sh$'' + ''^misc/fish/completion\.fish$'' + ''^misc/zsh/completion\.zsh$'' + ''^scripts/create-darwin-volume\.sh$'' + ''^scripts/install-darwin-multi-user\.sh$'' + ''^scripts/install-multi-user\.sh$'' + ''^scripts/install-nix-from-closure\.sh$'' + ''^scripts/install-systemd-multi-user\.sh$'' + ''^src/nix/get-env\.sh$'' + ''^tests/functional/build\.sh$'' + ''^tests/functional/ca/build-dry\.sh$'' + ''^tests/functional/ca/build-with-garbage-path\.sh$'' + ''^tests/functional/ca/common\.sh$'' + ''^tests/functional/ca/concurrent-builds\.sh$'' + ''^tests/functional/ca/eval-store\.sh$'' + ''^tests/functional/ca/gc\.sh$'' + ''^tests/functional/ca/import-derivation\.sh$'' + ''^tests/functional/ca/new-build-cmd\.sh$'' + ''^tests/functional/ca/nix-shell\.sh$'' + ''^tests/functional/ca/post-hook\.sh$'' + ''^tests/functional/ca/recursive\.sh$'' + ''^tests/functional/ca/repl\.sh$'' + ''^tests/functional/ca/selfref-gc\.sh$'' + ''^tests/functional/ca/why-depends\.sh$'' + ''^tests/functional/characterisation-test-infra\.sh$'' + ''^tests/functional/check\.sh$'' + ''^tests/functional/common/vars-and-functions\.sh$'' + ''^tests/functional/completions\.sh$'' + ''^tests/functional/compute-levels\.sh$'' + ''^tests/functional/config\.sh$'' + ''^tests/functional/db-migration\.sh$'' + ''^tests/functional/debugger\.sh$'' + ''^tests/functional/dependencies\.builder0\.sh$'' + ''^tests/functional/dependencies\.sh$'' + ''^tests/functional/dump-db\.sh$'' + ''^tests/functional/dyn-drv/build-built-drv\.sh$'' + ''^tests/functional/dyn-drv/common\.sh$'' + ''^tests/functional/dyn-drv/dep-built-drv\.sh$'' + ''^tests/functional/dyn-drv/eval-outputOf\.sh$'' + ''^tests/functional/dyn-drv/old-daemon-error-hack\.sh$'' + ''^tests/functional/dyn-drv/recursive-mod-json\.sh$'' + ''^tests/functional/eval-store\.sh$'' + ''^tests/functional/eval\.sh$'' + ''^tests/functional/export-graph\.sh$'' + ''^tests/functional/export\.sh$'' + ''^tests/functional/extra-sandbox-profile\.sh$'' + ''^tests/functional/fetchClosure\.sh$'' + ''^tests/functional/fetchGit\.sh$'' + ''^tests/functional/fetchGitRefs\.sh$'' + ''^tests/functional/fetchGitSubmodules\.sh$'' + ''^tests/functional/fetchGitVerification\.sh$'' + ''^tests/functional/fetchMercurial\.sh$'' + ''^tests/functional/fetchurl\.sh$'' + ''^tests/functional/fixed\.builder1\.sh$'' + ''^tests/functional/fixed\.builder2\.sh$'' + ''^tests/functional/fixed\.sh$'' + ''^tests/functional/flakes/absolute-paths\.sh$'' + ''^tests/functional/flakes/check\.sh$'' + ''^tests/functional/flakes/common\.sh$'' + ''^tests/functional/flakes/config\.sh$'' + ''^tests/functional/flakes/develop\.sh$'' + ''^tests/functional/flakes/flakes\.sh$'' + ''^tests/functional/flakes/follow-paths\.sh$'' + ''^tests/functional/flakes/prefetch\.sh$'' + ''^tests/functional/flakes/run\.sh$'' + ''^tests/functional/flakes/show\.sh$'' + ''^tests/functional/fmt\.sh$'' + ''^tests/functional/fmt\.simple\.sh$'' + ''^tests/functional/gc-auto\.sh$'' + ''^tests/functional/gc-concurrent\.builder\.sh$'' + ''^tests/functional/gc-concurrent\.sh$'' + ''^tests/functional/gc-concurrent2\.builder\.sh$'' + ''^tests/functional/gc-non-blocking\.sh$'' + ''^tests/functional/gc\.sh$'' + ''^tests/functional/git-hashing/common\.sh$'' + ''^tests/functional/git-hashing/simple\.sh$'' + ''^tests/functional/hash-convert\.sh$'' + ''^tests/functional/help\.sh$'' + ''^tests/functional/impure-derivations\.sh$'' + ''^tests/functional/impure-env\.sh$'' + ''^tests/functional/impure-eval\.sh$'' + ''^tests/functional/install-darwin\.sh$'' + ''^tests/functional/lang\.sh$'' + ''^tests/functional/legacy-ssh-store\.sh$'' + ''^tests/functional/linux-sandbox\.sh$'' + ''^tests/functional/local-overlay-store/add-lower-inner\.sh$'' + ''^tests/functional/local-overlay-store/add-lower\.sh$'' + ''^tests/functional/local-overlay-store/bad-uris\.sh$'' + ''^tests/functional/local-overlay-store/build-inner\.sh$'' + ''^tests/functional/local-overlay-store/build\.sh$'' + ''^tests/functional/local-overlay-store/check-post-init-inner\.sh$'' + ''^tests/functional/local-overlay-store/check-post-init\.sh$'' + ''^tests/functional/local-overlay-store/common\.sh$'' + ''^tests/functional/local-overlay-store/delete-duplicate-inner\.sh$'' + ''^tests/functional/local-overlay-store/delete-duplicate\.sh$'' + ''^tests/functional/local-overlay-store/delete-refs-inner\.sh$'' + ''^tests/functional/local-overlay-store/delete-refs\.sh$'' + ''^tests/functional/local-overlay-store/gc-inner\.sh$'' + ''^tests/functional/local-overlay-store/gc\.sh$'' + ''^tests/functional/local-overlay-store/optimise-inner\.sh$'' + ''^tests/functional/local-overlay-store/optimise\.sh$'' + ''^tests/functional/local-overlay-store/redundant-add-inner\.sh$'' + ''^tests/functional/local-overlay-store/redundant-add\.sh$'' + ''^tests/functional/local-overlay-store/remount\.sh$'' + ''^tests/functional/local-overlay-store/stale-file-handle-inner\.sh$'' + ''^tests/functional/local-overlay-store/stale-file-handle\.sh$'' + ''^tests/functional/local-overlay-store/verify-inner\.sh$'' + ''^tests/functional/local-overlay-store/verify\.sh$'' + ''^tests/functional/logging\.sh$'' + ''^tests/functional/misc\.sh$'' + ''^tests/functional/multiple-outputs\.sh$'' + ''^tests/functional/nar-access\.sh$'' + ''^tests/functional/nested-sandboxing\.sh$'' + ''^tests/functional/nested-sandboxing/command\.sh$'' + ''^tests/functional/nix-build\.sh$'' + ''^tests/functional/nix-channel\.sh$'' + ''^tests/functional/nix-collect-garbage-d\.sh$'' + ''^tests/functional/nix-copy-ssh-common\.sh$'' + ''^tests/functional/nix-copy-ssh-ng\.sh$'' + ''^tests/functional/nix-copy-ssh\.sh$'' + ''^tests/functional/nix-daemon-untrusting\.sh$'' + ''^tests/functional/nix-profile\.sh$'' + ''^tests/functional/nix-shell\.sh$'' + ''^tests/functional/nix_path\.sh$'' + ''^tests/functional/optimise-store\.sh$'' + ''^tests/functional/output-normalization\.sh$'' + ''^tests/functional/parallel\.builder\.sh$'' + ''^tests/functional/parallel\.sh$'' + ''^tests/functional/pass-as-file\.sh$'' + ''^tests/functional/path-from-hash-part\.sh$'' + ''^tests/functional/path-info\.sh$'' + ''^tests/functional/placeholders\.sh$'' + ''^tests/functional/plugins\.sh$'' + ''^tests/functional/post-hook\.sh$'' + ''^tests/functional/pure-eval\.sh$'' + ''^tests/functional/push-to-store-old\.sh$'' + ''^tests/functional/push-to-store\.sh$'' + ''^tests/functional/read-only-store\.sh$'' + ''^tests/functional/readfile-context\.sh$'' + ''^tests/functional/recursive\.sh$'' + ''^tests/functional/referrers\.sh$'' + ''^tests/functional/remote-store\.sh$'' + ''^tests/functional/repair\.sh$'' + ''^tests/functional/restricted\.sh$'' + ''^tests/functional/search\.sh$'' + ''^tests/functional/secure-drv-outputs\.sh$'' + ''^tests/functional/selfref-gc\.sh$'' + ''^tests/functional/shell\.sh$'' + ''^tests/functional/shell\.shebang\.sh$'' + ''^tests/functional/signing\.sh$'' + ''^tests/functional/simple\.builder\.sh$'' + ''^tests/functional/simple\.sh$'' + ''^tests/functional/ssh-relay\.sh$'' + ''^tests/functional/store-info\.sh$'' + ''^tests/functional/structured-attrs\.sh$'' + ''^tests/functional/substitute-with-invalid-ca\.sh$'' + ''^tests/functional/suggestions\.sh$'' + ''^tests/functional/supplementary-groups\.sh$'' + ''^tests/functional/tarball\.sh$'' + ''^tests/functional/test-infra\.sh$'' + ''^tests/functional/test-libstoreconsumer\.sh$'' + ''^tests/functional/timeout\.sh$'' + ''^tests/functional/toString-path\.sh$'' + ''^tests/functional/user-envs-migration\.sh$'' + ''^tests/functional/user-envs-test-case\.sh$'' + ''^tests/functional/user-envs\.builder\.sh$'' + ''^tests/functional/user-envs\.sh$'' + ''^tests/functional/why-depends\.sh$'' + ''^tests/functional/zstd\.sh$'' + ''^tests/unit/libutil/data/git/check-data\.sh$'' + ]; + }; + # TODO: nixfmt, https://github.com/NixOS/nixfmt/issues/153 + }; + }; + }; + + # We'll be pulling from this in the main flake + flake.getSystem = getSystem; +} diff --git a/maintainers/local.mk b/maintainers/local.mk new file mode 100644 index 000000000..88d594d67 --- /dev/null +++ b/maintainers/local.mk @@ -0,0 +1,15 @@ + +.PHONY: format +print-top-help += echo ' format: Format source code' + +# This uses the cached .pre-commit-hooks.yaml file +format: + @if ! type -p pre-commit &>/dev/null; then \ + echo "make format: pre-commit not found. Please use \`nix develop\`."; \ + exit 1; \ + fi; \ + if test -z "$$_NIX_PRE_COMMIT_HOOKS_CONFIG"; then \ + echo "make format: _NIX_PRE_COMMIT_HOOKS_CONFIG not set. Please use \`nix develop\`."; \ + exit 1; \ + fi; \ + pre-commit run --config $$_NIX_PRE_COMMIT_HOOKS_CONFIG --all-files diff --git a/maintainers/release-credits b/maintainers/release-credits new file mode 100755 index 000000000..7a5c87d7d --- /dev/null +++ b/maintainers/release-credits @@ -0,0 +1,184 @@ +#!/usr/bin/env nix +# vim: set filetype=python: +#!nix develop --impure --expr +#!nix `` +#!nix let flake = builtins.getFlake ("git+file://" + toString ../.); +#!nix pkgs = flake.inputs.nixpkgs.legacyPackages.${builtins.currentSystem}; +#!nix in pkgs.mkShell { nativeBuildInputs = [ +#!nix (pkgs.python3.withPackages (ps: with ps; [ requests ])) +#!nix ]; } +#!nix `` --command python3 + +# This script lists out the contributors for a given release. +# It must be run from the root of the Nix repository. + +import os +import sys +import json +import requests + +github_token = os.environ.get("GITHUB_TOKEN") +if not github_token: + print("GITHUB_TOKEN is not set. If you hit the rate limit, set it", file=sys.stderr) + # Might be ok, as we have a cache. + # raise ValueError("GITHUB_TOKEN must be set") + +# 1. Read the current version in .version +version = os.environ.get("VERSION") +if not version: + version = open(".version").read().strip() + +print(f"Generating release credits for Nix {version}", file=sys.stderr) + +# 2. Compute previous version +vcomponents = version.split(".") +if len(vcomponents) >= 2: + prev_version = f"{vcomponents[0]}.{int(vcomponents[1])-1}.0" +else: + raise ValueError(".version must have at least two components") + +# For unreleased versions +endref = "HEAD" +# For older releases +# endref = version + +# 2. Find the merge base between the current version and the previous version +mergeBase = os.popen(f"git merge-base {prev_version} {endref}").read().strip() +print(f"Merge base between {prev_version} and {endref} is {mergeBase}", file=sys.stderr) + +# 3. Find the date of the merge base +mergeBaseDate = os.popen(f"git show -s --format=%ci {mergeBase}").read().strip()[0:10] +print(f"Merge base date is {mergeBaseDate}", file=sys.stderr) + +# 4. Get the commits between the merge base and the current version + +def get_commits(): + raw = os.popen(f"git log --pretty=format:'%H\t%an\t%ae' {mergeBase}..{endref}").read().strip() + lines = raw.split("\n") + return [ { "hash": items[0], "author": items[1], "email": items[2] } + for line in lines + for items in (line.split("\t"),) + ] + +def commits_to_first_commit_by_email(commits): + by_email = dict() + for commit in commits: + email = commit["email"] + if email not in by_email: + by_email[email] = commit + return by_email + + +samples = commits_to_first_commit_by_email(get_commits()) + +# For quick testing, only pick two samples from the dict +# samples = dict(list(samples.items())[:2]) + +# Query the GitHub API to get handle +def get_github_commit(commit): + url = f"https://api.github.com/repos/NixOS/nix/commits/{commit['hash']}" + headers = {'Authorization': f'token {github_token}'} + response = requests.get(url, headers=headers) + response.raise_for_status() + return response.json() + +class Cache: + def __init__(self, filename, require = True): + self.filename = filename + try: + with open(filename, "r") as f: + self.values = json.load(f) + except FileNotFoundError: + if require: + raise + self.values = dict() + def save(self): + with open(self.filename, "w") as f: + json.dump(self.values, f, indent=4) + print(f"Saved cache to {self.filename}", file=sys.stderr) + +# The email to handle cache maps email addresses to either +# - a handle (string) +# - None (if no handle was found) +email_to_handle_cache = Cache("maintainers/data/release-credits-email-to-handle.json") + +handles = set() +emails = dict() + +for sample in samples: + s = samples[sample] + email = s["email"] + if not email in email_to_handle_cache.values: + print(f"Querying GitHub API for {s['hash']}, to get handle for {s['email']}") + ghc = get_github_commit(samples[sample]) + gha = ghc["author"] + if gha and gha["login"]: + handle = gha["login"] + print(f"Handle: {handle}") + email_to_handle_cache.values[email] = handle + else: + print(f"Found no handle for {s['email']}") + email_to_handle_cache.values[email] = None + handle = email_to_handle_cache.values[email] + if handle is not None: + handles.add(handle) + else: + emails[email] = s["author"] + +# print(email_to_handle_cache.values) + +email_to_handle_cache.save() + +handle_to_name_cache = Cache("maintainers/data/release-credits-handle-to-name.json") + +print(f"Found {len(handles)} handles", file=sys.stderr) + +for handle in handles: + if not handle in handle_to_name_cache.values: + print(f"Querying GitHub API for {handle}, to get name", file=sys.stderr) + url = f"https://api.github.com/users/{handle}" + headers = {'Authorization': f'token {github_token}'} + response = requests.get(url, headers=headers) + response.raise_for_status() + user = response.json() + name = user["name"] + print(f"Name: {name}", file=sys.stderr) + handle_to_name_cache.values[handle] = name + +handle_to_name_cache.save() + +entries = list() + +def is_bot(handle): + return handle.endswith("[bot]") + +for handle in handles: + name = handle_to_name_cache.values[handle] + if name is None: + # This way it looks more regular + name = handle + + # Note to bots: I'm only doing this because it would distract humans, and + # therefore delay your creation. I'm sorry; it's for your benefit. + if not is_bot(handle): + entries += [ f"- {name} [**(@{handle})**](https://github.com/{handle})" ] + +def shuffle(entries): + salt = os.urandom(16) + return sorted(entries, key=lambda x: hash((x, salt))) + +# Fair ordering is undecidable +entries = shuffle(entries) + +# For a sanity check, we could sort the entries by handle instead. +# entries = sorted(entries) + +print("") +print(f"This release was made possible by the following {len(entries)} contributors:") +print("") + +for entry in entries: + print(entry) + +for email in emails: + print(f"- {emails[email]}") diff --git a/maintainers/release-notes b/maintainers/release-notes index 2d84485c1..c0c4ee734 100755 --- a/maintainers/release-notes +++ b/maintainers/release-notes @@ -1,5 +1,6 @@ #!/usr/bin/env nix -#!nix shell .#changelog-d-nix --command bash +# vim: set filetype=bash: +#!nix shell .#changelog-d --command bash # --- CONFIGURATION --- @@ -151,6 +152,13 @@ section_title="Release $version_full ($DATE)" echo "# $section_title" echo changelog-d doc/manual/rl-next | sed -e 's/ *$//' + + if ! $IS_PATCH; then + echo + echo "# Contributors" + echo + VERSION=$version_full ./maintainers/release-credits + fi ) | tee -a $file log "Wrote $file" diff --git a/maintainers/release-process.md b/maintainers/release-process.md index da6886ea9..7a2b3c0a7 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -39,6 +39,10 @@ release: * Proof-read / edit / rearrange the release notes if needed. Breaking changes and highlights should go to the top. +* Run `maintainers/release-credits` to make sure the credits script works + and produces a sensible output. Some emails might not automatically map to + a GitHub handle. + * Push. ```console diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 4e2c379f0..731988568 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -11,6 +11,8 @@ use JSON::PP; use LWP::UserAgent; use Net::Amazon::S3; +delete $ENV{'shell'}; # shut up a LWP::UserAgent.pm warning + my $evalId = $ARGV[0] or die "Usage: $0 EVAL-ID\n"; my $releasesBucketName = "nix-releases"; @@ -36,11 +38,11 @@ sub fetch { my $evalUrl = "https://hydra.nixos.org/eval/$evalId"; my $evalInfo = decode_json(fetch($evalUrl, 'application/json')); #print Dumper($evalInfo); -my $flakeUrl = $evalInfo->{flake} or die; -my $flakeInfo = decode_json(`nix flake metadata --json "$flakeUrl"` or die); -my $nixRev = $flakeInfo->{revision} or die; +my $flakeUrl = $evalInfo->{flake}; +my $flakeInfo = decode_json(`nix flake metadata --json "$flakeUrl"` or die) if $flakeUrl; +my $nixRev = ($flakeInfo ? $flakeInfo->{revision} : $evalInfo->{jobsetevalinputs}->{nix}->{revision}) or die; -my $buildInfo = decode_json(fetch("$evalUrl/job/build.x86_64-linux", 'application/json')); +my $buildInfo = decode_json(fetch("$evalUrl/job/build.nix.x86_64-linux", 'application/json')); #print Dumper($buildInfo); my $releaseName = $buildInfo->{nixname}; @@ -83,12 +85,19 @@ my $channelsBucket = $s3_us->bucket($channelsBucketName) or die; sub getStorePath { my ($jobName, $output) = @_; my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json')); - return $buildInfo->{buildoutputs}->{$output or "out"}->{path} or die "cannot get store path for '$jobName'"; + return $buildInfo->{buildoutputs}->{$output or "out"}->{path} // die "cannot get store path for '$jobName'"; } sub copyManual { - my $manual = getStorePath("build.x86_64-linux", "doc"); - print "$manual\n"; + my $manual; + eval { + $manual = getStorePath("build.nix.x86_64-linux", "doc"); + }; + if ($@) { + warn "$@"; + return; + } + print "Manual: $manual\n"; my $manualNar = "$tmpDir/$releaseName-manual.nar.xz"; print "$manualNar\n"; @@ -154,19 +163,34 @@ downloadFile("binaryTarball.x86_64-linux", "1"); downloadFile("binaryTarball.aarch64-linux", "1"); downloadFile("binaryTarball.x86_64-darwin", "1"); downloadFile("binaryTarball.aarch64-darwin", "1"); -downloadFile("binaryTarballCross.x86_64-linux.armv6l-unknown-linux-gnueabihf", "1"); -downloadFile("binaryTarballCross.x86_64-linux.armv7l-unknown-linux-gnueabihf", "1"); +eval { + downloadFile("binaryTarballCross.x86_64-linux.armv6l-unknown-linux-gnueabihf", "1"); +}; +warn "$@" if $@; +eval { + downloadFile("binaryTarballCross.x86_64-linux.armv7l-unknown-linux-gnueabihf", "1"); +}; +warn "$@" if $@; +eval { + downloadFile("binaryTarballCross.x86_64-linux.riscv64-unknown-linux-gnu", "1"); +}; +warn "$@" if $@; downloadFile("installerScript", "1"); # Upload docker images to dockerhub. my $dockerManifest = ""; my $dockerManifestLatest = ""; +my $haveDocker = 0; for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) { my $system = $platforms->[0]; my $dockerPlatform = $platforms->[1]; my $fn = "nix-$version-docker-image-$dockerPlatform.tar.gz"; - downloadFile("dockerImage.$system", "1", $fn); + eval { + downloadFile("dockerImage.$system", "1", $fn); + }; + die "$@" if $@; + $haveDocker = 1; print STDERR "loading docker image for $dockerPlatform...\n"; system("docker load -i $tmpDir/$fn") == 0 or die; @@ -194,31 +218,34 @@ for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) { $dockerManifestLatest .= " --amend $latestTag" } -print STDERR "creating multi-platform docker manifest...\n"; -system("docker manifest rm nixos/nix:$version"); -system("docker manifest create nixos/nix:$version $dockerManifest") == 0 or die; -if ($isLatest) { - print STDERR "creating latest multi-platform docker manifest...\n"; - system("docker manifest rm nixos/nix:latest"); - system("docker manifest create nixos/nix:latest $dockerManifestLatest") == 0 or die; -} +if ($haveDocker) { + print STDERR "creating multi-platform docker manifest...\n"; + system("docker manifest rm nixos/nix:$version"); + system("docker manifest create nixos/nix:$version $dockerManifest") == 0 or die; + if ($isLatest) { + print STDERR "creating latest multi-platform docker manifest...\n"; + system("docker manifest rm nixos/nix:latest"); + system("docker manifest create nixos/nix:latest $dockerManifestLatest") == 0 or die; + } -print STDERR "pushing multi-platform docker manifest...\n"; -system("docker manifest push nixos/nix:$version") == 0 or die; + print STDERR "pushing multi-platform docker manifest...\n"; + system("docker manifest push nixos/nix:$version") == 0 or die; -if ($isLatest) { - print STDERR "pushing latest multi-platform docker manifest...\n"; - system("docker manifest push nixos/nix:latest") == 0 or die; + if ($isLatest) { + print STDERR "pushing latest multi-platform docker manifest...\n"; + system("docker manifest push nixos/nix:latest") == 0 or die; + } } # Upload nix-fallback-paths.nix. write_file("$tmpDir/fallback-paths.nix", "{\n" . - " x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" . - " i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" . - " aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" . - " x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" . - " aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" . + " x86_64-linux = \"" . getStorePath("build.nix.x86_64-linux") . "\";\n" . + " i686-linux = \"" . getStorePath("build.nix.i686-linux") . "\";\n" . + " aarch64-linux = \"" . getStorePath("build.nix.aarch64-linux") . "\";\n" . + " riscv64-linux = \"" . getStorePath("buildCross.nix.riscv64-unknown-linux-gnu.x86_64-linux") . "\";\n" . + " x86_64-darwin = \"" . getStorePath("build.nix.x86_64-darwin") . "\";\n" . + " aarch64-darwin = \"" . getStorePath("build.nix.aarch64-darwin") . "\";\n" . "}\n"); # Upload release files to S3. diff --git a/meson.build b/meson.build new file mode 100644 index 000000000..1554244ab --- /dev/null +++ b/meson.build @@ -0,0 +1,44 @@ +# This is just a stub project to include all the others as subprojects +# for development shell purposes + +project('nix-dev-shell', 'cpp', + version : files('.version'), + subproject_dir : 'src', +) + +# Internal Libraries +subproject('libutil') +subproject('libstore') +subproject('libfetchers') +subproject('libexpr') +subproject('libflake') +subproject('libmain') +subproject('libcmd') + +# Executables +subproject('nix') + +# Docs +subproject('internal-api-docs') +subproject('external-api-docs') + +# External C wrapper libraries +subproject('libutil-c') +subproject('libstore-c') +subproject('libexpr-c') +subproject('libmain-c') + +# Language Bindings +if not meson.is_cross_build() + subproject('perl') +endif + +# Testing +subproject('nix-util-test-support') +subproject('nix-util-tests') +subproject('nix-store-test-support') +subproject('nix-store-tests') +subproject('nix-fetchers-tests') +subproject('nix-expr-test-support') +subproject('nix-expr-tests') +subproject('nix-flake-tests') diff --git a/misc/changelog-d.cabal.nix b/misc/changelog-d.cabal.nix deleted file mode 100644 index 76f9353cd..000000000 --- a/misc/changelog-d.cabal.nix +++ /dev/null @@ -1,31 +0,0 @@ -{ mkDerivation, aeson, base, bytestring, cabal-install-parsers -, Cabal-syntax, containers, directory, filepath, frontmatter -, generic-lens-lite, lib, mtl, optparse-applicative, parsec, pretty -, regex-applicative, text, pkgs -}: -let rev = "f30f6969e9cd8b56242309639d58acea21c99d06"; -in -mkDerivation { - pname = "changelog-d"; - version = "0.1"; - src = pkgs.fetchurl { - name = "changelog-d-${rev}.tar.gz"; - url = "https://codeberg.org/roberth/changelog-d/archive/${rev}.tar.gz"; - hash = "sha256-8a2+i5u7YoszAgd5OIEW0eYUcP8yfhtoOIhLJkylYJ4="; - } // { inherit rev; }; - isLibrary = false; - isExecutable = true; - libraryHaskellDepends = [ - aeson base bytestring cabal-install-parsers Cabal-syntax containers - directory filepath frontmatter generic-lens-lite mtl parsec pretty - regex-applicative text - ]; - executableHaskellDepends = [ - base bytestring Cabal-syntax directory filepath - optparse-applicative - ]; - doHaddock = false; - description = "Concatenate changelog entries into a single one"; - license = lib.licenses.gpl3Plus; - mainProgram = "changelog-d"; -} diff --git a/misc/changelog-d.nix b/misc/changelog-d.nix deleted file mode 100644 index 1b20f4596..000000000 --- a/misc/changelog-d.nix +++ /dev/null @@ -1,31 +0,0 @@ -# Taken temporarily from -{ - callPackage, - lib, - haskell, - haskellPackages, -}: - -let - hsPkg = haskellPackages.callPackage ./changelog-d.cabal.nix { }; - - addCompletions = haskellPackages.generateOptparseApplicativeCompletions ["changelog-d"]; - - haskellModifications = - lib.flip lib.pipe [ - addCompletions - haskell.lib.justStaticExecutables - ]; - - mkDerivationOverrides = finalAttrs: oldAttrs: { - - version = oldAttrs.version + "-git-${lib.strings.substring 0 7 oldAttrs.src.rev}"; - - meta = oldAttrs.meta // { - homepage = "https://codeberg.org/roberth/changelog-d"; - maintainers = [ lib.maintainers.roberth ]; - }; - - }; -in - (haskellModifications hsPkg).overrideAttrs mkDerivationOverrides diff --git a/misc/systemv/nix-daemon b/misc/systemv/nix-daemon index fea537167..e8326f947 100755 --- a/misc/systemv/nix-daemon +++ b/misc/systemv/nix-daemon @@ -34,6 +34,7 @@ else fi # Source function library. +# shellcheck source=/dev/null . /etc/init.d/functions LOCKFILE=/var/lock/subsys/nix-daemon @@ -41,14 +42,20 @@ RUNDIR=/var/run/nix PIDFILE=${RUNDIR}/nix-daemon.pid RETVAL=0 -base=${0##*/} +# https://www.shellcheck.net/wiki/SC3004 +# Check if gettext exists +if ! type gettext > /dev/null 2>&1 +then + # If not, create a dummy function that returns the input verbatim + gettext() { printf '%s' "$1"; } +fi start() { mkdir -p ${RUNDIR} chown ${NIX_DAEMON_USER}:${NIX_DAEMON_USER} ${RUNDIR} - echo -n $"Starting nix daemon... " + printf '%s' "$(gettext 'Starting nix daemon... ')" daemonize -u $NIX_DAEMON_USER -p ${PIDFILE} $NIX_DAEMON_BIN $NIX_DAEMON_OPTS RETVAL=$? @@ -58,7 +65,7 @@ start() { } stop() { - echo -n $"Shutting down nix daemon: " + printf '%s' "$(gettext 'Shutting down nix daemon: ')" killproc -p ${PIDFILE} $NIX_DAEMON_BIN RETVAL=$? [ $RETVAL -eq 0 ] && rm -f ${LOCKFILE} ${PIDFILE} @@ -67,7 +74,7 @@ stop() { } reload() { - echo -n $"Reloading nix daemon... " + printf '%s' "$(gettext 'Reloading nix daemon... ')" killproc -p ${PIDFILE} $NIX_DAEMON_BIN -HUP RETVAL=$? echo @@ -105,7 +112,7 @@ case "$1" in fi ;; *) - echo $"Usage: $0 {start|stop|status|restart|condrestart}" + printf '%s' "$(gettext "Usage: $0 {start|stop|status|restart|condrestart}")" exit 2 ;; esac diff --git a/mk/common-test.sh b/mk/common-test.sh index 2783d293b..817422c40 100644 --- a/mk/common-test.sh +++ b/mk/common-test.sh @@ -1,27 +1,23 @@ +# shellcheck shell=bash + # Remove overall test dir (at most one of the two should match) and # remove file extension. -test_name=$(echo -n "$test" | sed \ - -e "s|^tests/unit/[^/]*/data/||" \ + +test_name=$(echo -n "${test?must be defined by caller (test runner)}" | sed \ + -e "s|^src/[^/]*-test/data/||" \ -e "s|^tests/functional/||" \ -e "s|\.sh$||" \ ) +# shellcheck disable=SC2016 TESTS_ENVIRONMENT=( "TEST_NAME=$test_name" 'NIX_REMOTE=' 'PS4=+(${BASH_SOURCE[0]-$0}:$LINENO) ' ) -: ${BASH:=/usr/bin/env bash} +read -r -a bash <<< "${BASH:-/usr/bin/env bash}" run () { - cd "$(dirname $1)" && env "${TESTS_ENVIRONMENT[@]}" $BASH -x -e -u -o pipefail $(basename $1) -} - -init_test () { - run "$init" 2>/dev/null > /dev/null -} - -run_test_proper () { - run "$test" + cd "$(dirname "$1")" && env "${TESTS_ENVIRONMENT[@]}" "${bash[@]}" -x -e -u -o pipefail "$(basename "$1")" } diff --git a/mk/compilation-database.mk b/mk/compilation-database.mk new file mode 100644 index 000000000..f69dc0de0 --- /dev/null +++ b/mk/compilation-database.mk @@ -0,0 +1,11 @@ +compile-commands-json-files := + +define write-compile-commands + _srcs := $$(sort $$(foreach src, $$($(1)_SOURCES), $$(src))) + + $(1)_COMPILE_COMMANDS_JSON := $$(addprefix $(buildprefix), $$(addsuffix .compile_commands.json, $$(basename $$(_srcs)))) + + compile-commands-json-files += $$($(1)_COMPILE_COMMANDS_JSON) + + clean-files += $$($(1)_COMPILE_COMMANDS_JSON) +endef diff --git a/mk/cxx-big-literal.mk b/mk/cxx-big-literal.mk index 85365df8e..d64a171c8 100644 --- a/mk/cxx-big-literal.mk +++ b/mk/cxx-big-literal.mk @@ -1,5 +1,5 @@ %.gen.hh: % - @echo 'R"foo(' >> $@.tmp + @echo 'R"__NIX_STR(' >> $@.tmp $(trace-gen) cat $< >> $@.tmp - @echo ')foo"' >> $@.tmp + @echo ')__NIX_STR"' >> $@.tmp @mv $@.tmp $@ diff --git a/mk/debug-test.sh b/mk/debug-test.sh index 52482c01e..0dd4406c3 100755 --- a/mk/debug-test.sh +++ b/mk/debug-test.sh @@ -3,12 +3,8 @@ set -eu -o pipefail test=$1 -init=${2-} dir="$(dirname "${BASH_SOURCE[0]}")" source "$dir/common-test.sh" -if [ -n "$init" ]; then - (init_test) -fi -run_test_proper +run "$test" diff --git a/mk/lib.mk b/mk/lib.mk index fe0add1c9..1e7af6ad5 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -68,6 +68,7 @@ include mk/patterns.mk include mk/templates.mk include mk/cxx-big-literal.mk include mk/tests.mk +include mk/compilation-database.mk # Include all sub-Makefiles. @@ -86,17 +87,23 @@ $(foreach script, $(bin-scripts), $(eval $(call install-program-in,$(script),$(b $(foreach script, $(bin-scripts), $(eval programs-list += $(script))) $(foreach script, $(noinst-scripts), $(eval programs-list += $(script))) $(foreach template, $(template-files), $(eval $(call instantiate-template,$(template)))) -install_test_init=tests/functional/init.sh $(foreach test, $(install-tests), \ - $(eval $(call run-test,$(test),$(install_test_init))) \ + $(eval $(call run-test,$(test))) \ $(eval installcheck: $(test).test)) $(foreach test-group, $(install-tests-groups), \ - $(eval $(call run-test-group,$(test-group),$(install_test_init))) \ + $(eval $(call run-test-group,$(test-group))) \ $(eval installcheck: $(test-group).test-group) \ $(foreach test, $($(test-group)-tests), \ - $(eval $(call run-test,$(test),$(install_test_init))) \ + $(eval $(call run-test,$(test))) \ $(eval $(test-group).test-group: $(test).test))) +# Compilation database. +$(foreach lib, $(libraries), $(eval $(call write-compile-commands,$(lib)))) +$(foreach prog, $(programs), $(eval $(call write-compile-commands,$(prog)))) + +compile_commands.json: $(compile-commands-json-files) + @jq --slurp '.' $^ >$@ + # Include makefiles requiring built programs. $(foreach mf, $(makefiles-late), $(eval $(call include-sub-makefile,$(mf)))) diff --git a/mk/patterns.mk b/mk/patterns.mk index c81150260..4caa2039e 100644 --- a/mk/patterns.mk +++ b/mk/patterns.mk @@ -1,11 +1,41 @@ + +# These are the complete command lines we use to compile C and C++ files. +# - $< is the source file. +# - $1 is the object file to create. +CC_CMD=$(CC) -o $1 -c $< $(CPPFLAGS) $(GLOBAL_CFLAGS) $(CFLAGS) $($1_CFLAGS) -MMD -MF $(call filename-to-dep,$1) -MP +CXX_CMD=$(CXX) -o $1 -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($1_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep,$1) -MP + +# We use COMPILE_COMMANDS_JSON_CMD to turn a compilation command (like CC_CMD +# or CXX_CMD above) into a comple_commands.json file. We rely on bash native +# word splitting to define the positional arguments. +# - $< is the source file being compiled. +COMPILE_COMMANDS_JSON_CMD=jq --null-input '{ directory: $$ENV.PWD, file: "$<", arguments: $$ARGS.positional }' --args -- + + $(buildprefix)%.o: %.cc @mkdir -p "$(dir $@)" - $(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cxx) $(call CXX_CMD,$@) $(buildprefix)%.o: %.cpp @mkdir -p "$(dir $@)" - $(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cxx) $(call CXX_CMD,$@) $(buildprefix)%.o: %.c @mkdir -p "$(dir $@)" - $(trace-cc) $(CC) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CFLAGS) $(CFLAGS) $($@_CFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cc) $(call CC_CMD,$@) + +# In the following we need to replace the .compile_commands.json extension in $@ with .o +# to make the object file. This is needed because CC_CMD and CXX_CMD do further expansions +# based on the object file name (i.e. *_CXXFLAGS and filename-to-dep). + +$(buildprefix)%.compile_commands.json: %.cc + @mkdir -p "$(dir $@)" + $(trace-jq) $(COMPILE_COMMANDS_JSON_CMD) $(call CXX_CMD,$(@:.compile_commands.json=.o)) > $@ + +$(buildprefix)%.compile_commands.json: %.cpp + @mkdir -p "$(dir $@)" + $(trace-jq) $(COMPILE_COMMANDS_JSON_CMD) $(call CXX_CMD,$(@:.compile_commands.json=.o)) > $@ + +$(buildprefix)%.compile_commands.json: %.c + @mkdir -p "$(dir $@)" + $(trace-jq) $(COMPILE_COMMANDS_JSON_CMD) $(call CC_CMD,$(@:.compile_commands.json=.o)) > $@ diff --git a/mk/platform.mk b/mk/platform.mk index fe960dedf..22c114a20 100644 --- a/mk/platform.mk +++ b/mk/platform.mk @@ -29,4 +29,8 @@ ifdef HOST_OS HOST_SOLARIS = 1 HOST_UNIX = 1 endif + ifeq ($(HOST_KERNEL), gnu) + HOST_HURD = 1 + HOST_UNIX = 1 + endif endif diff --git a/mk/precompiled-headers.mk b/mk/precompiled-headers.mk index cdd3daecd..f2803eb79 100644 --- a/mk/precompiled-headers.mk +++ b/mk/precompiled-headers.mk @@ -8,7 +8,7 @@ GCH = $(buildprefix)precompiled-headers.h.gch $(GCH): precompiled-headers.h @rm -f $@ @mkdir -p "$(dir $@)" - $(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS) + $(trace-gen) $(CXX) -c -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS) clean-files += $(GCH) diff --git a/mk/run-test.sh b/mk/run-test.sh index da9c5a473..7f9f1d5f8 100755 --- a/mk/run-test.sh +++ b/mk/run-test.sh @@ -8,7 +8,6 @@ yellow="" normal="" test=$1 -init=${2-} dir="$(dirname "${BASH_SOURCE[0]}")" source "$dir/common-test.sh" @@ -22,20 +21,18 @@ if [ -t 1 ]; then fi run_test () { - if [ -n "$init" ]; then - (init_test 2>/dev/null > /dev/null) - fi - log="$(run_test_proper 2>&1)" && status=0 || status=$? + log="$(run "$test" 2>&1)" && status=0 || status=$? } run_test -if [ $status -eq 0 ]; then +if [[ "$status" = 0 ]]; then echo "$post_run_msg [${green}PASS$normal]" -elif [ $status -eq 99 ]; then +elif [[ "$status" = 77 ]]; then echo "$post_run_msg [${yellow}SKIP$normal]" else echo "$post_run_msg [${red}FAIL$normal]" + # shellcheck disable=SC2001 echo "$log" | sed 's/^/ /' exit "$status" fi diff --git a/mk/tests.mk b/mk/tests.mk index bac9b704a..0a10f6d3b 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -12,8 +12,8 @@ endef define run-test - $(eval $(call run-bash,$1.test,$1 $(test-deps),mk/run-test.sh $1 $2)) - $(eval $(call run-bash,$1.test-debug,$1 $(test-deps),mk/debug-test.sh $1 $2)) + $(eval $(call run-bash,$1.test,$1 $(test-deps),mk/run-test.sh $1)) + $(eval $(call run-bash,$1.test-debug,$1 $(test-deps),mk/debug-test.sh $1)) endef diff --git a/mk/tracing.mk b/mk/tracing.mk index 1fc5573d7..09db1e617 100644 --- a/mk/tracing.mk +++ b/mk/tracing.mk @@ -10,6 +10,8 @@ ifeq ($(V), 0) trace-install = @echo " INST " $@; trace-mkdir = @echo " MKDIR " $@; trace-test = @echo " TEST " $@; + trace-sh = @echo " SH " $@; + trace-jq = @echo " JQ " $@; suppress = @ diff --git a/package.nix b/package.nix index 20796a386..a7c8923e8 100644 --- a/package.nix +++ b/package.nix @@ -13,17 +13,16 @@ , curl , editline , readline -, fileset , flex , git , gtest , jq -, doxygen , libarchive , libcpuid , libgit2 , libseccomp , libsodium +, man , lowdown , mdbook , mdbook-linkcheck @@ -33,7 +32,8 @@ , pkg-config , rapidcheck , sqlite -, util-linux +, toml11 +, unixtools , xz , busybox-sandbox-shell ? null @@ -48,10 +48,8 @@ , pname ? "nix" , versionSuffix ? "" -, officialRelease ? false -# Whether to build Nix. Useful to skip for tasks like (a) just -# generating API docs or (b) testing existing pre-built versions of Nix +# Whether to build Nix. Useful to skip for tasks like testing existing pre-built versions of Nix , doBuild ? true # Run the unit tests as part of the build. See `installUnitTests` for an @@ -74,7 +72,10 @@ # sounds so long as evaluation just takes places within short-lived # processes. (When the process exits, the memory is reclaimed; it is # only leaked *within* the process.) -, enableGC ? true +# +# Temporarily disabled on Windows because the `GC_throw_bad_alloc` +# symbol is missing during linking. +, enableGC ? !stdenv.hostPlatform.isWindows # Whether to enable Markdown rendering in the Nix binary. , enableMarkdown ? !stdenv.hostPlatform.isWindows @@ -87,10 +88,6 @@ # - readline , readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline" -# Whether to build the internal API docs, can be done separately from -# everything else. -, enableInternalAPIDocs ? false - # Whether to install unit tests. This is useful when cross compiling # since we cannot run them natively during the build, but can do so # later. @@ -113,6 +110,8 @@ }: let + inherit (lib) fileset; + version = lib.fileContents ./.version + versionSuffix; # selected attributes with defaults, will be used to define some @@ -161,6 +160,8 @@ in { ./m4 # TODO: do we really need README.md? It doesn't seem used in the build. ./README.md + # This could be put behind a conditional + ./maintainers/local.mk # For make, regardless of what we are building ./local.mk ./Makefile @@ -171,17 +172,11 @@ in { ./doc ./misc ./precompiled-headers.h - ./src + (fileset.difference ./src ./src/perl) ./COPYING ./scripts/local.mk - ] ++ lib.optionals buildUnitTests [ + ] ++ lib.optionals enableManual [ ./doc/manual - ] ++ lib.optionals enableInternalAPIDocs [ - ./doc/internal-api - # Source might not be compiled, but still must be available - # for Doxygen to gather comments. - ./src - ./tests/unit ] ++ lib.optionals buildUnitTests [ ./tests/unit ] ++ lib.optionals doInstallCheck [ @@ -195,8 +190,10 @@ in { ++ lib.optional doBuild "dev" # If we are doing just build or just docs, the one thing will use # "out". We only need additional outputs if we are doing both. - ++ lib.optional (doBuild && (enableManual || enableInternalAPIDocs)) "doc" - ++ lib.optional installUnitTests "check"; + ++ lib.optional (doBuild && enableManual) "doc" + ++ lib.optional installUnitTests "check" + ++ lib.optional doCheck "testresults" + ; nativeBuildInputs = [ autoconf-archive @@ -213,14 +210,14 @@ in { git mercurial openssh + man # for testing `nix-* --help` ] ++ lib.optionals (doInstallCheck || enableManual) [ jq # Also for custom mdBook preprocessor. - ] ++ lib.optional stdenv.hostPlatform.isLinux util-linux - ++ lib.optional enableInternalAPIDocs doxygen + ] ++ lib.optional stdenv.hostPlatform.isStatic unixtools.hexdump ; - buildInputs = lib.optionals doBuild [ - boost + buildInputs = lib.optionals doBuild ( + [ brotli bzip2 curl @@ -229,6 +226,7 @@ in { libsodium openssl sqlite + toml11 xz ({ inherit readline editline; }.${readlineFlavor}) ] ++ lib.optionals enableMarkdown [ @@ -240,46 +238,22 @@ in { ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid # There have been issues building these dependencies ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin)) - (aws-sdk-cpp.override { - apis = ["s3" "transfer"]; - customMemoryManagement = false; - }) - ; + aws-sdk-cpp + ); - propagatedBuildInputs = [ + propagatedBuildInputs = lib.optionals doBuild ([ + boost nlohmann_json - ] ++ lib.optional enableGC boehmgc; + ] ++ lib.optional enableGC boehmgc + ); dontBuild = !attrs.doBuild; doCheck = attrs.doCheck; - disallowedReferences = [ boost ]; - - preConfigure = lib.optionalString (doBuild && ! stdenv.hostPlatform.isStatic) ( - '' - # Copy libboost_context so we don't get all of Boost in our closure. - # https://github.com/NixOS/nixpkgs/issues/45462 - mkdir -p $out/lib - cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib - rm -f $out/lib/*.a - '' + lib.optionalString stdenv.hostPlatform.isLinux '' - chmod u+w $out/lib/*.so.* - patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* - '' + lib.optionalString stdenv.hostPlatform.isDarwin '' - for LIB in $out/lib/*.dylib; do - chmod u+w $LIB - install_name_tool -id $LIB $LIB - install_name_tool -delete_rpath ${boost}/lib/ $LIB || true - done - install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib - '' - ); - configureFlags = [ (lib.enableFeature doBuild "build") (lib.enableFeature buildUnitTests "unit-tests") (lib.enableFeature doInstallCheck "functional-tests") - (lib.enableFeature enableInternalAPIDocs "internal-api-docs") (lib.enableFeature enableManual "doc-gen") (lib.enableFeature enableGC "gc") (lib.enableFeature enableMarkdown "markdown") @@ -303,8 +277,11 @@ in { makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1"; - installTargets = lib.optional doBuild "install" - ++ lib.optional enableInternalAPIDocs "internal-api-html"; + preCheck = '' + mkdir $testresults + ''; + + installTargets = lib.optional doBuild "install"; installFlags = "sysconfdir=$(out)/etc"; @@ -319,18 +296,16 @@ in { lib.optionalString stdenv.hostPlatform.isStatic '' mkdir -p $out/nix-support echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products - '' + lib.optionalString stdenv.isDarwin '' - install_name_tool \ - -change ${boost}/lib/libboost_context.dylib \ - $out/lib/libboost_context.dylib \ - $out/lib/libnixutil.dylib '' ) + lib.optionalString enableManual '' mkdir -p ''${!outputDoc}/nix-support echo "doc manual ''${!outputDoc}/share/doc/nix/manual" >> ''${!outputDoc}/nix-support/hydra-build-products - '' + lib.optionalString enableInternalAPIDocs '' - mkdir -p ''${!outputDoc}/nix-support - echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products + ''; + + # So the check output gets links for DLLs in the out output. + preFixup = lib.optionalString (stdenv.hostPlatform.isWindows && builtins.elem "check" finalAttrs.outputs) '' + ln -s "$check/lib/"*.dll "$check/bin" + ln -s "$out/bin/"*.dll "$check/bin" ''; doInstallCheck = attrs.doInstallCheck; @@ -341,20 +316,26 @@ in { # Work around weird bug where it doesn't think there is a Makefile. installCheckPhase = if (!doBuild && doInstallCheck) then '' + runHook preInstallCheck mkdir -p src/nix-channel make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES '' else null; # Needed for tests if we are not doing a build, but testing existing # built Nix. - preInstallCheck = lib.optionalString (! doBuild) '' - mkdir -p src/nix-channel - ''; + preInstallCheck = + lib.optionalString (! doBuild) '' + mkdir -p src/nix-channel + '' + # See https://github.com/NixOS/nix/issues/2523 + # Occurs often in tests since https://github.com/NixOS/nix/pull/9900 + + lib.optionalString stdenv.hostPlatform.isDarwin '' + export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES + ''; separateDebugInfo = !stdenv.hostPlatform.isStatic; - # TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated - # to work with `strictDeps`. + # TODO Always true after https://github.com/NixOS/nixpkgs/issues/318564 strictDeps = !withCoverageChecks; hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; diff --git a/packaging/components.nix b/packaging/components.nix new file mode 100644 index 000000000..870e9ae61 --- /dev/null +++ b/packaging/components.nix @@ -0,0 +1,43 @@ +scope: +let + inherit (scope) callPackage; +in + +# This becomes the pkgs.nixComponents attribute set +{ + nix = callPackage ../package.nix { }; + + nix-util = callPackage ../src/libutil/package.nix { }; + nix-util-c = callPackage ../src/libutil-c/package.nix { }; + nix-util-test-support = callPackage ../tests/unit/libutil-support/package.nix { }; + nix-util-tests = callPackage ../tests/unit/libutil/package.nix { }; + + nix-store = callPackage ../src/libstore/package.nix { }; + nix-store-c = callPackage ../src/libstore-c/package.nix { }; + nix-store-test-support = callPackage ../tests/unit/libstore-support/package.nix { }; + nix-store-tests = callPackage ../tests/unit/libstore/package.nix { }; + + nix-fetchers = callPackage ../src/libfetchers/package.nix { }; + nix-fetchers-tests = callPackage ../tests/unit/libfetchers/package.nix { }; + + nix-expr = callPackage ../src/libexpr/package.nix { }; + nix-expr-c = callPackage ../src/libexpr-c/package.nix { }; + nix-expr-test-support = callPackage ../tests/unit/libexpr-support/package.nix { }; + nix-expr-tests = callPackage ../tests/unit/libexpr/package.nix { }; + + nix-flake = callPackage ../src/libflake/package.nix { }; + nix-flake-tests = callPackage ../tests/unit/libflake/package.nix { }; + + nix-main = callPackage ../src/libmain/package.nix { }; + nix-main-c = callPackage ../src/libmain-c/package.nix { }; + + nix-cmd = callPackage ../src/libcmd/package.nix { }; + + # Will replace `nix` once the old build system is gone. + nix-ng = callPackage ../src/nix/package.nix { }; + + nix-internal-api-docs = callPackage ../src/internal-api-docs/package.nix { }; + nix-external-api-docs = callPackage ../src/external-api-docs/package.nix { }; + + nix-perl-bindings = callPackage ../src/perl/package.nix { }; +} diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix new file mode 100644 index 000000000..e0737593f --- /dev/null +++ b/packaging/dependencies.nix @@ -0,0 +1,165 @@ +# These overrides are applied to the dependencies of the Nix components. + +{ + # Flake inputs; used for sources + inputs, + + # The raw Nixpkgs, not affected by this scope + pkgs, + + stdenv, + versionSuffix, +}: + +let + prevStdenv = stdenv; +in + +let + inherit (pkgs) lib; + + root = ../.; + + stdenv = if prevStdenv.isDarwin && prevStdenv.isx86_64 + then darwinStdenv + else prevStdenv; + + # Fix the following error with the default x86_64-darwin SDK: + # + # error: aligned allocation function of type 'void *(std::size_t, std::align_val_t)' is only available on macOS 10.13 or newer + # + # Despite the use of the 10.13 deployment target here, the aligned + # allocation function Clang uses with this setting actually works + # all the way back to 10.6. + darwinStdenv = pkgs.overrideSDK prevStdenv { darwinMinVersion = "10.13"; }; + + # Nixpkgs implements this by returning a subpath into the fetched Nix sources. + resolvePath = p: p; + + # Indirection for Nixpkgs to override when package.nix files are vendored + filesetToSource = lib.fileset.toSource; + + localSourceLayer = finalAttrs: prevAttrs: + let + workDirPath = + # Ideally we'd pick finalAttrs.workDir, but for now `mkDerivation` has + # the requirement that everything except passthru and meta must be + # serialized by mkDerivation, which doesn't work for this. + prevAttrs.workDir; + + workDirSubpath = lib.path.removePrefix root workDirPath; + sources = assert prevAttrs.fileset._type == "fileset"; prevAttrs.fileset; + src = lib.fileset.toSource { fileset = sources; inherit root; }; + + in + { + sourceRoot = "${src.name}/" + workDirSubpath; + inherit src; + + # Clear what `derivation` can't/shouldn't serialize; see prevAttrs.workDir. + fileset = null; + workDir = null; + }; + + # Work around weird `--as-needed` linker behavior with BSD, see + # https://github.com/mesonbuild/meson/issues/3593 + bsdNoLinkAsNeeded = finalAttrs: prevAttrs: + lib.optionalAttrs stdenv.hostPlatform.isBSD { + mesonFlags = [ (lib.mesonBool "b_asneeded" false) ] ++ prevAttrs.mesonFlags or []; + }; + + miscGoodPractice = finalAttrs: prevAttrs: + { + strictDeps = prevAttrs.strictDeps or true; + enableParallelBuilding = true; + }; + +in +scope: { + inherit stdenv versionSuffix; + version = lib.fileContents ../.version + versionSuffix; + + aws-sdk-cpp = (pkgs.aws-sdk-cpp.override { + apis = [ "s3" "transfer" ]; + customMemoryManagement = false; + }).overrideAttrs { + # only a stripped down version is built, which takes a lot less resources + # to build, so we don't need a "big-parallel" machine. + requiredSystemFeatures = [ ]; + }; + + libseccomp = pkgs.libseccomp.overrideAttrs (_: rec { + version = "2.5.5"; + src = pkgs.fetchurl { + url = "https://github.com/seccomp/libseccomp/releases/download/v${version}/libseccomp-${version}.tar.gz"; + hash = "sha256-JIosik2bmFiqa69ScSw0r+/PnJ6Ut23OAsHJqiX7M3U="; + }; + }); + + boehmgc = pkgs.boehmgc.override { + enableLargeConfig = true; + }; + + # TODO Hack until https://github.com/NixOS/nixpkgs/issues/45462 is fixed. + boost = (pkgs.boost.override { + extraB2Args = [ + "--with-container" + "--with-context" + "--with-coroutine" + ]; + }).overrideAttrs (old: { + # Need to remove `--with-*` to use `--with-libraries=...` + buildPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.buildPhase; + installPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.installPhase; + }); + + libgit2 = pkgs.libgit2.overrideAttrs (attrs: { + src = inputs.libgit2; + version = inputs.libgit2.lastModifiedDate; + cmakeFlags = attrs.cmakeFlags or [] + ++ [ "-DUSE_SSH=exec" ]; + }); + + busybox-sandbox-shell = pkgs.busybox-sandbox-shell or (pkgs.busybox.override { + useMusl = true; + enableStatic = true; + enableMinimal = true; + extraConfig = '' + CONFIG_FEATURE_FANCY_ECHO y + CONFIG_FEATURE_SH_MATH y + CONFIG_FEATURE_SH_MATH_64 y + + CONFIG_ASH y + CONFIG_ASH_OPTIMIZE_FOR_SIZE y + + CONFIG_ASH_ALIAS y + CONFIG_ASH_BASH_COMPAT y + CONFIG_ASH_CMDCMD y + CONFIG_ASH_ECHO y + CONFIG_ASH_GETOPTS y + CONFIG_ASH_INTERNAL_GLOB y + CONFIG_ASH_JOB_CONTROL y + CONFIG_ASH_PRINTF y + CONFIG_ASH_TEST y + ''; + }); + + # TODO change in Nixpkgs, Windows works fine. First commit of + # https://github.com/NixOS/nixpkgs/pull/322977 backported will fix. + toml11 = pkgs.toml11.overrideAttrs (old: { + meta.platforms = lib.platforms.all; + }); + + inherit resolvePath filesetToSource; + + mkMesonDerivation = f: let + exts = [ + miscGoodPractice + bsdNoLinkAsNeeded + localSourceLayer + ]; + in stdenv.mkDerivation + (lib.extends + (lib.foldr lib.composeExtensions (_: _: {}) exts) + f); +} diff --git a/packaging/hydra.nix b/packaging/hydra.nix new file mode 100644 index 000000000..dbe992476 --- /dev/null +++ b/packaging/hydra.nix @@ -0,0 +1,200 @@ +{ inputs +, binaryTarball +, forAllCrossSystems +, forAllSystems +, lib +, linux64BitSystems +, nixpkgsFor +, self +}: +let + inherit (inputs) nixpkgs nixpkgs-regression; + + installScriptFor = tarballs: + nixpkgsFor.x86_64-linux.native.callPackage ../scripts/installer.nix { + inherit tarballs; + }; + + testNixVersions = pkgs: client: daemon: + pkgs.callPackage ../package.nix { + pname = + "nix-tests" + + lib.optionalString + (lib.versionAtLeast daemon.version "2.4pre20211005" && + lib.versionAtLeast client.version "2.4pre20211005") + "-${client.version}-against-${daemon.version}"; + + test-client = client; + test-daemon = daemon; + + doBuild = false; + }; + + # Technically we could just return `pkgs.nixComponents`, but for Hydra it's + # convention to transpose it, and to transpose it efficiently, we need to + # enumerate them manually, so that we don't evaluate unnecessary package sets. + forAllPackages = lib.genAttrs [ + "nix" + "nix-util" + "nix-util-c" + "nix-util-test-support" + "nix-util-tests" + "nix-store" + "nix-store-c" + "nix-store-test-support" + "nix-store-tests" + "nix-fetchers" + "nix-fetchers-tests" + "nix-expr" + "nix-expr-c" + "nix-expr-test-support" + "nix-expr-tests" + "nix-flake" + "nix-flake-tests" + "nix-main" + "nix-main-c" + "nix-cmd" + "nix-ng" + ]; +in +{ + # Binary package for various platforms. + build = forAllPackages (pkgName: + forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.${pkgName})); + + shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation); + + buildStatic = forAllPackages (pkgName: + lib.genAttrs linux64BitSystems (system: nixpkgsFor.${system}.static.nixComponents.${pkgName})); + + buildCross = forAllPackages (pkgName: + forAllCrossSystems (crossSystem: + lib.genAttrs [ "x86_64-linux" ] (system: nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName}))); + + buildNoGc = forAllSystems (system: + self.packages.${system}.nix.override { enableGC = false; } + ); + + buildNoTests = forAllSystems (system: nixpkgsFor.${system}.native.nix_noTests); + + # Toggles some settings for better coverage. Windows needs these + # library combinations, and Debian build Nix with GNU readline too. + buildReadlineNoMarkdown = forAllSystems (system: + self.packages.${system}.nix.override { + enableMarkdown = false; + readlineFlavor = "readline"; + } + ); + + # Perl bindings for various platforms. + perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.nix-perl-bindings); + + # Binary tarball for various platforms, containing a Nix store + # with the closure of 'nix' package, and the second half of + # the installation script. + binaryTarball = forAllSystems (system: binaryTarball nixpkgsFor.${system}.native.nix nixpkgsFor.${system}.native); + + binaryTarballCross = lib.genAttrs [ "x86_64-linux" ] (system: + forAllCrossSystems (crossSystem: + binaryTarball + nixpkgsFor.${system}.cross.${crossSystem}.nix + nixpkgsFor.${system}.cross.${crossSystem})); + + # The first half of the installation script. This is uploaded + # to https://nixos.org/nix/install. It downloads the binary + # tarball for the user's system and calls the second half of the + # installation script. + installerScript = installScriptFor [ + # Native + self.hydraJobs.binaryTarball."x86_64-linux" + self.hydraJobs.binaryTarball."i686-linux" + self.hydraJobs.binaryTarball."aarch64-linux" + self.hydraJobs.binaryTarball."x86_64-darwin" + self.hydraJobs.binaryTarball."aarch64-darwin" + # Cross + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu" + ]; + installerScriptForGHA = installScriptFor [ + # Native + self.hydraJobs.binaryTarball."x86_64-linux" + self.hydraJobs.binaryTarball."aarch64-darwin" + # Cross + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu" + ]; + + # docker image with Nix inside + dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); + + # Line coverage analysis. + coverage = nixpkgsFor.x86_64-linux.native.nix.override { + pname = "nix-coverage"; + withCoverageChecks = true; + }; + + # API docs for Nix's unstable internal C++ interfaces. + internal-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-internal-api-docs; + + # API docs for Nix's C bindings. + external-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-external-api-docs; + + # System tests. + tests = import ../tests/nixos { inherit lib nixpkgs nixpkgsFor self; } // { + + # Make sure that nix-env still produces the exact same result + # on a particular version of Nixpkgs. + evalNixpkgs = + let + inherit (nixpkgsFor.x86_64-linux.native) runCommand nix; + in + runCommand "eval-nixos" { buildInputs = [ nix ]; } + '' + type -p nix-env + # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593. + ( + set -x + time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages + [[ $(sha1sum < packages | cut -c1-40) = e01b031fc9785a572a38be6bc473957e3b6faad7 ]] + ) + mkdir $out + ''; + + nixpkgsLibTests = + forAllSystems (system: + import (nixpkgs + "/lib/tests/test-with-nix.nix") + { + lib = nixpkgsFor.${system}.native.lib; + nix = self.packages.${system}.nix; + pkgs = nixpkgsFor.${system}.native; + } + ); + }; + + metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" { + pkgs = nixpkgsFor.x86_64-linux.native; + nixpkgs = nixpkgs-regression; + }; + + installTests = forAllSystems (system: + let pkgs = nixpkgsFor.${system}.native; in + pkgs.runCommand "install-tests" + { + againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix; + againstCurrentLatest = + # FIXME: temporarily disable this on macOS because of #3605. + if system == "x86_64-linux" + then testNixVersions pkgs pkgs.nix pkgs.nixVersions.latest + else null; + # Disabled because the latest stable version doesn't handle + # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work + # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable; + } "touch $out"); + + installerTests = import ../tests/installer { + binaryTarballs = self.hydraJobs.binaryTarball; + inherit nixpkgsFor; + }; +} diff --git a/perl/.yath.rc b/perl/.yath.rc deleted file mode 100644 index 118bf80c8..000000000 --- a/perl/.yath.rc +++ /dev/null @@ -1,2 +0,0 @@ -[test] --I=rel(lib/Nix) diff --git a/perl/Makefile b/perl/Makefile deleted file mode 100644 index 832668dd1..000000000 --- a/perl/Makefile +++ /dev/null @@ -1,21 +0,0 @@ -makefiles = local.mk - -GLOBAL_CXXFLAGS += -g -Wall -std=c++2a - -# A convenience for concurrent development of Nix and its Perl bindings. -# Not needed in a standalone build of the Perl bindings. -ifneq ("$(wildcard ../src)", "") - GLOBAL_CXXFLAGS += -I ../src -endif - --include Makefile.config - -OPTIMIZE = 1 - -ifeq ($(OPTIMIZE), 1) - GLOBAL_CXXFLAGS += -O3 -else - GLOBAL_CXXFLAGS += -O0 -endif - -include mk/lib.mk diff --git a/perl/Makefile.config.in b/perl/Makefile.config.in deleted file mode 100644 index d856de3ad..000000000 --- a/perl/Makefile.config.in +++ /dev/null @@ -1,18 +0,0 @@ -HOST_OS = @host_os@ -CC = @CC@ -CFLAGS = @CFLAGS@ -CXX = @CXX@ -CXXFLAGS = @CXXFLAGS@ -PACKAGE_NAME = @PACKAGE_NAME@ -PACKAGE_VERSION = @PACKAGE_VERSION@ -SODIUM_LIBS = @SODIUM_LIBS@ -NIX_CFLAGS = @NIX_CFLAGS@ -NIX_LIBS = @NIX_LIBS@ -nixbindir = @nixbindir@ -curl = @curl@ -nixlibexecdir = @nixlibexecdir@ -nixlocalstatedir = @nixlocalstatedir@ -perl = @perl@ -perllibdir = @perllibdir@ -nixstoredir = @nixstoredir@ -nixsysconfdir = @nixsysconfdir@ diff --git a/perl/configure.ac b/perl/configure.ac deleted file mode 100644 index a02cb06c9..000000000 --- a/perl/configure.ac +++ /dev/null @@ -1,84 +0,0 @@ -AC_INIT(nix-perl, m4_esyscmd([bash -c "echo -n $(cat ../.version)$VERSION_SUFFIX"])) -AC_CONFIG_SRCDIR(MANIFEST) -AC_CONFIG_AUX_DIR(../config) - -CFLAGS= -CXXFLAGS= -AC_PROG_CC -AC_PROG_CXX - -AC_CANONICAL_HOST - -# Use 64-bit file system calls so that we can support files > 2 GiB. -AC_SYS_LARGEFILE - -AC_DEFUN([NEED_PROG], -[ -AC_PATH_PROG($1, $2) -if test -z "$$1"; then - AC_MSG_ERROR([$2 is required]) -fi -]) - -NEED_PROG(perl, perl) -NEED_PROG(curl, curl) -NEED_PROG(bzip2, bzip2) -NEED_PROG(xz, xz) - -# Test that Perl has the open/fork feature (Perl 5.8.0 and beyond). -AC_MSG_CHECKING([whether Perl is recent enough]) -if ! $perl -e 'open(FOO, "-|", "true"); while () { print; }; close FOO or die;'; then - AC_MSG_RESULT(no) - AC_MSG_ERROR([Your Perl version is too old. Nix requires Perl 5.8.0 or newer.]) -fi -AC_MSG_RESULT(yes) - - -# Figure out where to install Perl modules. -AC_MSG_CHECKING([for the Perl installation prefix]) -perlversion=$($perl -e 'use Config; print $Config{version};') -perlarchname=$($perl -e 'use Config; print $Config{archname};') -AC_SUBST(perllibdir, [${libdir}/perl5/site_perl/$perlversion/$perlarchname]) -AC_MSG_RESULT($perllibdir) - -# Look for libsodium. -PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"]) - -# Check for the required Perl dependencies (DBI and DBD::SQLite). -perlFlags="-I$perllibdir" - -AC_ARG_WITH(dbi, AC_HELP_STRING([--with-dbi=PATH], - [prefix of the Perl DBI library]), - perlFlags="$perlFlags -I$withval") - -AC_ARG_WITH(dbd-sqlite, AC_HELP_STRING([--with-dbd-sqlite=PATH], - [prefix of the Perl DBD::SQLite library]), - perlFlags="$perlFlags -I$withval") - -AC_MSG_CHECKING([whether DBD::SQLite works]) -if ! $perl $perlFlags -e 'use DBI; use DBD::SQLite;' 2>&5; then - AC_MSG_RESULT(no) - AC_MSG_FAILURE([The Perl modules DBI and/or DBD::SQLite are missing.]) -fi -AC_MSG_RESULT(yes) - -AC_SUBST(perlFlags) - -PKG_CHECK_MODULES([NIX], [nix-store]) - -NEED_PROG([NIX], [nix]) - -# Expand all variables in config.status. -test "$prefix" = NONE && prefix=$ac_default_prefix -test "$exec_prefix" = NONE && exec_prefix='${prefix}' -for name in $ac_subst_vars; do - declare $name="$(eval echo "${!name}")" - declare $name="$(eval echo "${!name}")" - declare $name="$(eval echo "${!name}")" -done - -rm -f Makefile.config -ln -sfn ../mk mk - -AC_CONFIG_FILES([]) -AC_OUTPUT diff --git a/perl/default.nix b/perl/default.nix deleted file mode 100644 index 7103574c9..000000000 --- a/perl/default.nix +++ /dev/null @@ -1,61 +0,0 @@ -{ lib, fileset -, stdenv -, perl, perlPackages -, autoconf-archive, autoreconfHook, pkg-config -, nix, curl, bzip2, xz, boost, libsodium, darwin -}: - -perl.pkgs.toPerlModule (stdenv.mkDerivation (finalAttrs: { - name = "nix-perl-${nix.version}"; - - src = fileset.toSource { - root = ../.; - fileset = fileset.unions ([ - ../.version - ../m4 - ../mk - ./MANIFEST - ./Makefile - ./Makefile.config.in - ./configure.ac - ./lib - ./local.mk - ] ++ lib.optionals finalAttrs.doCheck [ - ./.yath.rc - ./t - ]); - }; - - nativeBuildInputs = - [ autoconf-archive - autoreconfHook - pkg-config - ]; - - buildInputs = - [ nix - curl - bzip2 - xz - perl - boost - ] - ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium - ++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security; - - # `perlPackages.Test2Harness` is marked broken for Darwin - doCheck = !stdenv.isDarwin; - - nativeCheckInputs = [ - perlPackages.Test2Harness - ]; - - configureFlags = [ - "--with-dbi=${perlPackages.DBI}/${perl.libPrefix}" - "--with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix}" - ]; - - enableParallelBuilding = true; - - postUnpack = "sourceRoot=$sourceRoot/perl"; -})) diff --git a/perl/local.mk b/perl/local.mk deleted file mode 100644 index ed4764eb9..000000000 --- a/perl/local.mk +++ /dev/null @@ -1,46 +0,0 @@ -nix_perl_sources := \ - lib/Nix/Store.pm \ - lib/Nix/Manifest.pm \ - lib/Nix/SSH.pm \ - lib/Nix/CopyClosure.pm \ - lib/Nix/Config.pm.in \ - lib/Nix/Utils.pm - -nix_perl_modules := $(nix_perl_sources:.in=) - -$(foreach x, $(nix_perl_modules), $(eval $(call install-data-in, $(x), $(perllibdir)/Nix))) - -lib/Nix/Store.cc: lib/Nix/Store.xs - $(trace-gen) xsubpp $^ -output $@ - -libraries += Store - -Store_DIR := lib/Nix - -Store_SOURCES := $(Store_DIR)/Store.cc - -Store_CXXFLAGS = \ - $(NIX_CFLAGS) \ - -I$(shell perl -e 'use Config; print $$Config{archlibexp};')/CORE \ - -D_FILE_OFFSET_BITS=64 \ - -Wno-unknown-warning-option -Wno-unused-variable -Wno-literal-suffix \ - -Wno-reserved-user-defined-literal -Wno-duplicate-decl-specifier -Wno-pointer-bool-conversion - -Store_LDFLAGS := $(SODIUM_LIBS) $(NIX_LIBS) - -ifdef HOST_CYGWIN - archlib = $(shell perl -E 'use Config; print $$Config{archlib};') - libperl = $(shell perl -E 'use Config; print $$Config{libperl};') - Store_LDFLAGS += $(shell find ${archlib} -name ${libperl}) -endif - -Store_ALLOW_UNDEFINED = 1 - -Store_FORCE_INSTALL = 1 - -Store_INSTALL_DIR = $(perllibdir)/auto/Nix/Store - -clean-files += lib/Nix/Config.pm lib/Nix/Store.cc Makefile.config - -check: all - yath test diff --git a/precompiled-headers.h b/precompiled-headers.h index f52f1cab8..e1a3f8cc0 100644 --- a/precompiled-headers.h +++ b/precompiled-headers.h @@ -42,19 +42,22 @@ #include #include #include -#include -#include -#include #include -#include -#include -#include #include #include #include -#include -#include -#include #include +#ifndef _WIN32 +# include +# include +# include +# include +# include +# include +# include +# include +# include +#endif + #include diff --git a/scripts/bigsur-nixbld-user-migration.sh b/scripts/bigsur-nixbld-user-migration.sh index f1619fd56..0eb312e07 100755 --- a/scripts/bigsur-nixbld-user-migration.sh +++ b/scripts/bigsur-nixbld-user-migration.sh @@ -3,7 +3,7 @@ ((NEW_NIX_FIRST_BUILD_UID=301)) id_available(){ - dscl . list /Users UniqueID | grep -E '\b'$1'\b' >/dev/null + dscl . list /Users UniqueID | grep -E '\b'"$1"'\b' >/dev/null } change_nixbld_names_and_ids(){ @@ -26,18 +26,18 @@ change_nixbld_names_and_ids(){ fi done - if [[ $name == _* ]]; then + if [[ "$name" == _* ]]; then echo " It looks like $name has already been renamed--skipping." else # first 3 are cleanup, it's OK if they aren't here - sudo dscl . delete /Users/$name dsAttrTypeNative:_writers_passwd &>/dev/null || true - sudo dscl . change /Users/$name NFSHomeDirectory "/private/var/empty 1" "/var/empty" &>/dev/null || true + sudo dscl . delete "/Users/$name" dsAttrTypeNative:_writers_passwd &>/dev/null || true + sudo dscl . change "/Users/$name" NFSHomeDirectory "/private/var/empty 1" "/var/empty" &>/dev/null || true # remove existing user from group - sudo dseditgroup -o edit -t user -d $name nixbld || true - sudo dscl . change /Users/$name UniqueID $uid $next_id - sudo dscl . change /Users/$name RecordName $name _$name + sudo dseditgroup -o edit -t user -d "$name" nixbld || true + sudo dscl . change "/Users/$name" UniqueID "$uid" "$next_id" + sudo dscl . change "/Users/$name" RecordName "$name" "_$name" # add renamed user to group - sudo dseditgroup -o edit -t user -a _$name nixbld + sudo dseditgroup -o edit -t user -a "_$name" nixbld echo " $name migrated to _$name (uid: $next_id)" fi done < <(dscl . list /Users UniqueID | grep nixbld | sort -n -k2) diff --git a/scripts/check-hydra-status.sh b/scripts/check-hydra-status.sh deleted file mode 100644 index e62705e94..000000000 --- a/scripts/check-hydra-status.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail -# set -x - - -# mapfile BUILDS_FOR_LATEST_EVAL < <( -# curl -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master/evals | \ -# jq -r '.evals[0].builds[] | @sh') -BUILDS_FOR_LATEST_EVAL=$( -curl -sS -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master/evals | \ - jq -r '.evals[0].builds[]') - -someBuildFailed=0 - -for buildId in $BUILDS_FOR_LATEST_EVAL; do - buildInfo=$(curl --fail -sS -H 'Accept: application/json' "https://hydra.nixos.org/build/$buildId") - - finished=$(echo "$buildInfo" | jq -r '.finished') - - if [[ $finished = 0 ]]; then - continue - fi - - buildStatus=$(echo "$buildInfo" | jq -r '.buildstatus') - - if [[ $buildStatus != 0 ]]; then - someBuildFailed=1 - echo "Job “$(echo "$buildInfo" | jq -r '.job')†failed on hydra: $buildInfo" - fi -done - -exit "$someBuildFailed" diff --git a/scripts/flake-regressions.sh b/scripts/flake-regressions.sh new file mode 100755 index 000000000..d76531134 --- /dev/null +++ b/scripts/flake-regressions.sh @@ -0,0 +1,27 @@ +#! /usr/bin/env bash + +set -e + +echo "Nix version:" +nix --version + +cd flake-regressions + +status=0 + +flakes=$(find tests -mindepth 3 -maxdepth 3 -type d -not -path '*/.*' | sort | head -n25) + +echo "Running flake tests..." + +for flake in $flakes; do + + if ! REGENERATE=0 ./eval-flake.sh "$flake"; then + status=1 + echo "⌠$flake" + else + echo "✅ $flake" + fi + +done + +exit "$status" diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index ad3ee8881..6aee073e3 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -754,7 +754,7 @@ I will: (if it does, I will tell you how to clean them up.) - create local users (see the list above for the users I'll make) - create a local group ($NIX_BUILD_GROUP_NAME) - - install Nix in to $NIX_ROOT + - install Nix in $NIX_ROOT - create a configuration file in /etc/nix - set up the "default profile" by creating some Nix-related files in $ROOT_HOME diff --git a/scripts/install-systemd-multi-user.sh b/scripts/install-systemd-multi-user.sh index 202a9bb54..a62ed7e3a 100755 --- a/scripts/install-systemd-multi-user.sh +++ b/scripts/install-systemd-multi-user.sh @@ -35,7 +35,7 @@ escape_systemd_env() { # Gather all non-empty proxy environment variables into a string create_systemd_proxy_env() { - vars="http_proxy https_proxy ftp_proxy no_proxy HTTP_PROXY HTTPS_PROXY FTP_PROXY NO_PROXY" + vars="http_proxy https_proxy ftp_proxy all_proxy no_proxy HTTP_PROXY HTTPS_PROXY FTP_PROXY ALL_PROXY NO_PROXY" for v in $vars; do if [ "x${!v:-}" != "x" ]; then echo "Environment=${v}=$(escape_systemd_env ${!v})" diff --git a/scripts/install.in b/scripts/install.in index 7d2e52b26..b4e808d8e 100755 --- a/scripts/install.in +++ b/scripts/install.in @@ -50,6 +50,11 @@ case "$(uname -s).$(uname -m)" in path=@tarballPath_armv7l-linux@ system=armv7l-linux ;; + Linux.riscv64) + hash=@tarballHash_riscv64-linux@ + path=@tarballPath_riscv64-linux@ + system=riscv64-linux + ;; Darwin.x86_64) hash=@tarballHash_x86_64-darwin@ path=@tarballPath_x86_64-darwin@ diff --git a/scripts/nix-profile-daemon.fish.in b/scripts/nix-profile-daemon.fish.in index c23aa64f0..346dce5dd 100644 --- a/scripts/nix-profile-daemon.fish.in +++ b/scripts/nix-profile-daemon.fish.in @@ -28,7 +28,7 @@ else end # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. -if test -n "$NIX_SSH_CERT_FILE" +if test -n "$NIX_SSL_CERT_FILE" : # Allow users to override the NIX_SSL_CERT_FILE else if test -e /etc/ssl/certs/ca-certificates.crt # NixOS, Ubuntu, Debian, Gentoo, Arch set --export NIX_SSL_CERT_FILE /etc/ssl/certs/ca-certificates.crt @@ -44,7 +44,7 @@ else if test -e "$NIX_LINK/etc/ca-bundle.crt" # old cacert in Nix profile set --export NIX_SSL_CERT_FILE "$NIX_LINK/etc/ca-bundle.crt" else # Fall back to what is in the nix profiles, favouring whatever is defined last. - for i in $NIX_PROFILES + for i in (string split ' ' $NIX_PROFILES) if test -e "$i/etc/ssl/certs/ca-bundle.crt" set --export NIX_SSL_CERT_FILE "$i/etc/ssl/certs/ca-bundle.crt" end diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index d256b24ed..eb124c0b5 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -1,4 +1,5 @@ # Only execute this file once per shell. +# This file is tested by tests/installer/default.nix. if [ -n "${__ETC_PROFILE_NIX_SOURCED:-}" ]; then return; fi __ETC_PROFILE_NIX_SOURCED=1 @@ -9,11 +10,9 @@ else NIX_LINK_NEW=$HOME/.local/state/nix/profile fi if [ -e "$NIX_LINK_NEW" ]; then - NIX_LINK="$NIX_LINK_NEW" -else - if [ -t 2 ] && [ -e "$NIX_LINK_NEW" ]; then + if [ -t 2 ] && [ -e "$NIX_LINK" ]; then warning="\033[1;35mwarning:\033[0m" - printf "$warning Both %s and legacy %s exist; using the latter.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2 + printf "$warning Both %s and legacy %s exist; using the former.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2 if [ "$(realpath "$NIX_LINK")" = "$(realpath "$NIX_LINK_NEW")" ]; then printf " Since the profiles match, you can safely delete either of them.\n" 1>&2 else @@ -26,6 +25,7 @@ else printf "$warning Profiles do not match. You should manually migrate from %s to %s.\n" "$NIX_LINK" "$NIX_LINK_NEW" 1>&2 fi fi + NIX_LINK="$NIX_LINK_NEW" fi export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK" @@ -69,4 +69,4 @@ else fi export PATH="$NIX_LINK/bin:@localstatedir@/nix/profiles/default/bin:$PATH" -unset NIX_LINK +unset NIX_LINK NIX_LINK_NEW diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index 44bc96e89..e868399b1 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -1,3 +1,4 @@ +# This file is tested by tests/installer/default.nix. if [ -n "$HOME" ] && [ -n "$USER" ]; then # Set up the per-user profile. @@ -9,11 +10,9 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then NIX_LINK_NEW="$HOME/.local/state/nix/profile" fi if [ -e "$NIX_LINK_NEW" ]; then - NIX_LINK="$NIX_LINK_NEW" - else - if [ -t 2 ] && [ -e "$NIX_LINK_NEW" ]; then + if [ -t 2 ] && [ -e "$NIX_LINK" ]; then warning="\033[1;35mwarning:\033[0m" - printf "$warning Both %s and legacy %s exist; using the latter.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2 + printf "$warning Both %s and legacy %s exist; using the former.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2 if [ "$(realpath "$NIX_LINK")" = "$(realpath "$NIX_LINK_NEW")" ]; then printf " Since the profiles match, you can safely delete either of them.\n" 1>&2 else @@ -26,6 +25,7 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then printf "$warning Profiles do not match. You should manually migrate from %s to %s.\n" "$NIX_LINK" "$NIX_LINK_NEW" 1>&2 fi fi + NIX_LINK="$NIX_LINK_NEW" fi # Set up environment. diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 118468477..82ad7d862 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -11,11 +11,13 @@ #include "machines.hh" #include "shared.hh" +#include "plugin.hh" #include "pathlocks.hh" #include "globals.hh" #include "serialise.hh" #include "build-result.hh" #include "store-api.hh" +#include "strings.hh" #include "derivations.hh" #include "local-store.hh" #include "legacy.hh" @@ -37,7 +39,7 @@ static std::string currentLoad; static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot) { - return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true); + return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri.render()), slot), true); } static bool allSupportedLocally(Store & store, const std::set& requiredFeatures) { @@ -135,7 +137,7 @@ static int main_build_remote(int argc, char * * argv) Machine * bestMachine = nullptr; uint64_t bestLoad = 0; for (auto & m : machines) { - debug("considering building on remote machine '%s'", m.storeUri); + debug("considering building on remote machine '%s'", m.storeUri.render()); if (m.enabled && m.systemSupported(neededSystem) && @@ -202,7 +204,7 @@ static int main_build_remote(int argc, char * * argv) else drvstr = ""; - auto error = HintFmt(errorText); + auto error = HintFmt::fromFormatString(errorText); error % drvstr % neededSystem @@ -232,17 +234,16 @@ static int main_build_remote(int argc, char * * argv) lock = -1; try { + storeUri = bestMachine->storeUri.render(); - Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri)); + Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", storeUri)); sshStore = bestMachine->openStore(); sshStore->connect(); - storeUri = bestMachine->storeUri; - } catch (std::exception & e) { auto msg = chomp(drainFD(5, false)); printError("cannot build on '%s': %s%s", - bestMachine->storeUri, e.what(), + storeUri, e.what(), msg.empty() ? "" : ": " + msg); bestMachine->enabled = false; continue; @@ -262,7 +263,20 @@ connected: auto inputs = readStrings(source); auto wantedOutputs = readStrings(source); - AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true); + AutoCloseFD uploadLock; + { + auto setUpdateLock = [&](auto && fileName){ + uploadLock = openLockFile(currentLoad + "/" + escapeUri(fileName) + ".upload-lock", true); + }; + try { + setUpdateLock(storeUri); + } catch (SysError & e) { + if (e.errNo != ENAMETOOLONG) throw; + // Try again hashing the store URL so we have a shorter path + auto h = hashString(HashAlgorithm::MD5, storeUri); + setUpdateLock(h.to_string(HashFormat::Base64, false)); + } + } { Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri)); diff --git a/doc/internal-api/.gitignore b/src/external-api-docs/.gitignore similarity index 100% rename from doc/internal-api/.gitignore rename to src/external-api-docs/.gitignore diff --git a/src/external-api-docs/.version b/src/external-api-docs/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/external-api-docs/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/external-api-docs/README.md b/src/external-api-docs/README.md new file mode 100644 index 000000000..8760ac88b --- /dev/null +++ b/src/external-api-docs/README.md @@ -0,0 +1,121 @@ +# Getting started + +> **Warning** These bindings are **experimental**, which means they can change +> at any time or be removed outright; nevertheless the plan is to provide a +> stable external C API to the Nix language and the Nix store. + +The language library allows evaluating Nix expressions and interacting with Nix +language values. The Nix store API is still rudimentary, and only allows +initialising and connecting to a store for the Nix language evaluator to +interact with. + +Currently there are two ways to interface with the Nix language evaluator +programmatically: + +1. Embedding the evaluator +2. Writing language plug-ins + +Embedding means you link the Nix C libraries in your program and use them from +there. Adding a plug-in means you make a library that gets loaded by the Nix +language evaluator, specified through a configuration option. + +Many of the components and mechanisms involved are not yet documented, therefore +please refer to the [Nix source code](https://github.com/NixOS/nix/) for +details. Additions to in-code documentation and the reference manual are highly +appreciated. + +The following examples, for simplicity, don't include error handling. See the +[Handling errors](@ref errors) section for more information. + +# Embedding the Nix Evaluator{#nix_evaluator_example} + +In this example we programmatically start the Nix language evaluator with a +dummy store (that has no store paths and cannot be written to), and evaluate the +Nix expression `builtins.nixVersion`. + +**main.c:** + +```C +#include +#include +#include +#include +#include +#include + +// NOTE: This example lacks all error handling. Production code must check for +// errors, as some return values will be undefined. + +void my_get_string_cb(const char * start, unsigned int n, void * user_data) +{ + *((char **) user_data) = strdup(start); +} + +int main() +{ + nix_libexpr_init(NULL); + + Store * store = nix_store_open(NULL, "dummy://", NULL); + EvalState * state = nix_state_create(NULL, NULL, store); // empty search path (NIX_PATH) + Value * value = nix_alloc_value(NULL, state); + + nix_expr_eval_from_string(NULL, state, "builtins.nixVersion", ".", value); + nix_value_force(NULL, state, value); + + char * version; + nix_get_string(NULL, value, my_get_string_cb, &version); + printf("Nix version: %s\n", version); + + free(version); + nix_gc_decref(NULL, value); + nix_state_free(state); + nix_store_free(store); + return 0; +} +``` + +**Usage:** + +```ShellSession +$ gcc main.c $(pkg-config nix-expr-c --libs --cflags) -o main +$ ./main +Nix version: 2.17 +``` + +# Writing a Nix language plug-in + +In this example we add a custom primitive operation (_primop_) to `builtins`. It +will increment the argument if it is an integer and throw an error otherwise. + +**plugin.c:** + +```C +#include +#include +#include + +void increment(void* user_data, nix_c_context* ctx, EvalState* state, Value** args, Value* v) { + nix_value_force(NULL, state, args[0]); + if (nix_get_type(NULL, args[0]) == NIX_TYPE_INT) { + nix_init_int(NULL, v, nix_get_int(NULL, args[0]) + 1); + } else { + nix_set_err_msg(ctx, NIX_ERR_UNKNOWN, "First argument should be an integer."); + } +} + +void nix_plugin_entry() { + const char* args[] = {"n", NULL}; + PrimOp *p = nix_alloc_primop(NULL, increment, 1, "increment", args, "Example custom built-in function: increments an integer", NULL); + nix_register_primop(NULL, p); + nix_gc_decref(NULL, p); +} +``` + +**Usage:** + +```ShellSession +$ gcc plugin.c $(pkg-config nix-expr-c --libs --cflags) -shared -o plugin.so +$ nix --plugin-files ./plugin.so repl +nix-repl> builtins.increment 1 +2 +``` diff --git a/src/external-api-docs/doxygen.cfg.in b/src/external-api-docs/doxygen.cfg.in new file mode 100644 index 000000000..1be71d895 --- /dev/null +++ b/src/external-api-docs/doxygen.cfg.in @@ -0,0 +1,58 @@ +# Doxyfile 1.9.5 + +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by +# double-quotes, unless you are using Doxywizard) that should identify the +# project for which the documentation is generated. This name is used in the +# title of most generated pages and in a few other places. +# The default value is: My Project. + +PROJECT_NAME = "Nix" + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. This +# could be handy for archiving the generated documentation or if some version +# control system is used. + +PROJECT_NUMBER = @PROJECT_NUMBER@ + +OUTPUT_DIRECTORY = @OUTPUT_DIRECTORY@ + +# Using the PROJECT_BRIEF tag one can provide an optional one line description +# for a project that appears at the top of each page and should give viewer a +# quick idea about the purpose of the project. Keep the description short. + +PROJECT_BRIEF = "Nix, the purely functional package manager: C API (experimental)" + +# If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output. +# The default value is: YES. + +GENERATE_LATEX = NO + +# The INPUT tag is used to specify the files and/or directories that contain +# documented source files. You may enter file names like myfile.cpp or +# directories like /usr/src/myproject. Separate the files or directories with +# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING +# Note: If this tag is empty the current directory is searched. + +# FIXME Make this list more maintainable somehow. We could maybe generate this +# in the Makefile, but we would need to change how `.in` files are preprocessed +# so they can expand variables despite configure variables. + +INPUT = \ + @src@/src/libutil-c \ + @src@/src/libexpr-c \ + @src@/src/libstore-c \ + @src@/doc/external-api/README.md + +FILE_PATTERNS = nix_api_*.h *.md + +# The INCLUDE_PATH tag can be used to specify one or more directories that +# contain include files that are not input files but should be processed by the +# preprocessor. Note that the INCLUDE_PATH is not recursive, so the setting of +# RECURSIVE has no effect here. +# This tag requires that the tag SEARCH_INCLUDES is set to YES. + +EXCLUDE_PATTERNS = *_internal.h +GENERATE_TREEVIEW = YES +OPTIMIZE_OUTPUT_FOR_C = YES + +USE_MDFILE_AS_MAINPAGE = doc/external-api/README.md diff --git a/src/external-api-docs/meson.build b/src/external-api-docs/meson.build new file mode 100644 index 000000000..62474ffe4 --- /dev/null +++ b/src/external-api-docs/meson.build @@ -0,0 +1,31 @@ +project('nix-external-api-docs', + version : files('.version'), + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +fs = import('fs') + +doxygen_cfg = configure_file( + input : 'doxygen.cfg.in', + output : 'doxygen.cfg', + configuration : { + 'PROJECT_NUMBER': meson.project_version(), + 'OUTPUT_DIRECTORY' : meson.current_build_dir(), + 'src' : fs.parent(fs.parent(meson.project_source_root())), + }, +) + +doxygen = find_program('doxygen', native : true, required : true) + +custom_target( + 'external-api-docs', + command : [ doxygen , doxygen_cfg ], + input : [ + doxygen_cfg, + ], + output : 'html', + install : true, + install_dir : get_option('datadir') / 'doc/nix/external-api', + build_always_stale : true, +) diff --git a/src/external-api-docs/package.nix b/src/external-api-docs/package.nix new file mode 100644 index 000000000..743b3e9b7 --- /dev/null +++ b/src/external-api-docs/package.nix @@ -0,0 +1,59 @@ +{ lib +, mkMesonDerivation + +, meson +, ninja +, doxygen + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-external-api-docs"; + inherit version; + + workDir = ./.; + fileset = + let + cpp = fileset.fileFilter (file: file.hasExt "cc" || file.hasExt "h"); + in + fileset.unions [ + ./.version + ../../.version + ./meson.build + ./doxygen.cfg.in + ./README.md + # Source is not compiled, but still must be available for Doxygen + # to gather comments. + (cpp ../libexpr-c) + (cpp ../libstore-c) + (cpp ../libutil-c) + ]; + + nativeBuildInputs = [ + meson + ninja + doxygen + ]; + + preConfigure = + '' + chmod u+w ./.version + echo ${finalAttrs.version} > ./.version + ''; + + postInstall = '' + mkdir -p ''${!outputDoc}/nix-support + echo "doc external-api-docs $out/share/doc/nix/external-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products + ''; + + meta = { + platforms = lib.platforms.all; + }; +}) diff --git a/src/internal-api-docs/.gitignore b/src/internal-api-docs/.gitignore new file mode 100644 index 000000000..dab28b6b0 --- /dev/null +++ b/src/internal-api-docs/.gitignore @@ -0,0 +1,3 @@ +/doxygen.cfg +/html +/latex diff --git a/src/internal-api-docs/.version b/src/internal-api-docs/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/internal-api-docs/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/doc/internal-api/doxygen.cfg.in b/src/internal-api-docs/doxygen.cfg.in similarity index 84% rename from doc/internal-api/doxygen.cfg.in rename to src/internal-api-docs/doxygen.cfg.in index 6c6c325bd..f1ef75b38 100644 --- a/doc/internal-api/doxygen.cfg.in +++ b/src/internal-api-docs/doxygen.cfg.in @@ -12,7 +12,9 @@ PROJECT_NAME = "Nix" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = @PACKAGE_VERSION@ +PROJECT_NUMBER = @PROJECT_NUMBER@ + +OUTPUT_DIRECTORY = @OUTPUT_DIRECTORY@ # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a @@ -36,27 +38,27 @@ GENERATE_LATEX = NO # so they can expand variables despite configure variables. INPUT = \ - src/libcmd \ - src/libexpr \ - src/libexpr/flake \ - tests/unit/libexpr \ - tests/unit/libexpr/value \ - tests/unit/libexpr/test \ - tests/unit/libexpr/test/value \ - src/libexpr/value \ - src/libfetchers \ - src/libmain \ - src/libstore \ - src/libstore/build \ - src/libstore/builtins \ - tests/unit/libstore \ - tests/unit/libstore/test \ - src/libutil \ - tests/unit/libutil \ - tests/unit/libutil/test \ - src/nix \ - src/nix-env \ - src/nix-store + @src@/libcmd \ + @src@/libexpr \ + @src@/libexpr/flake \ + @src@/nix-expr-tests \ + @src@/nix-expr-tests/value \ + @src@/nix-expr-test-support/test \ + @src@/nix-expr-test-support/test/value \ + @src@/libexpr/value \ + @src@/libfetchers \ + @src@/libmain \ + @src@/libstore \ + @src@/libstore/build \ + @src@/libstore/builtins \ + @src@/nix-store-tests \ + @src@/nix-store-test-support/test \ + @src@/libutil \ + @src@/nix-util-tests \ + @src@/nix-util-test-support/test \ + @src@/nix \ + @src@/nix-env \ + @src@/nix-store # If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names # in the source code. If set to NO, only conditional compilation will be diff --git a/src/internal-api-docs/meson.build b/src/internal-api-docs/meson.build new file mode 100644 index 000000000..54eb7e5dd --- /dev/null +++ b/src/internal-api-docs/meson.build @@ -0,0 +1,31 @@ +project('nix-internal-api-docs', + version : files('.version'), + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +fs = import('fs') + +doxygen_cfg = configure_file( + input : 'doxygen.cfg.in', + output : 'doxygen.cfg', + configuration : { + 'PROJECT_NUMBER': meson.project_version(), + 'OUTPUT_DIRECTORY' : meson.current_build_dir(), + 'src' : fs.parent(fs.parent(meson.project_source_root())) / 'src', + }, +) + +doxygen = find_program('doxygen', native : true, required : true) + +custom_target( + 'internal-api-docs', + command : [ doxygen , doxygen_cfg ], + input : [ + doxygen_cfg, + ], + output : 'html', + install : true, + install_dir : get_option('datadir') / 'doc/nix/internal-api', + build_always_stale : true, +) diff --git a/src/internal-api-docs/package.nix b/src/internal-api-docs/package.nix new file mode 100644 index 000000000..07ca6d4d9 --- /dev/null +++ b/src/internal-api-docs/package.nix @@ -0,0 +1,54 @@ +{ lib +, mkMesonDerivation + +, meson +, ninja +, doxygen + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-internal-api-docs"; + inherit version; + + workDir = ./.; + fileset = let + cpp = fileset.fileFilter (file: file.hasExt "cc" || file.hasExt "hh"); + in fileset.unions [ + ./.version + ../../.version + ./meson.build + ./doxygen.cfg.in + # Source is not compiled, but still must be available for Doxygen + # to gather comments. + (cpp ../.) + ]; + + nativeBuildInputs = [ + meson + ninja + doxygen + ]; + + preConfigure = + '' + chmod u+w ./.version + echo ${finalAttrs.version} > ./.version + ''; + + postInstall = '' + mkdir -p ''${!outputDoc}/nix-support + echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products + ''; + + meta = { + platforms = lib.platforms.all; + }; +}) diff --git a/src/libcmd/.version b/src/libcmd/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libcmd/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libcmd/build-utils-meson b/src/libcmd/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libcmd/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc index c5eb93c5d..905e70f32 100644 --- a/src/libcmd/built-path.cc +++ b/src/libcmd/built-path.cc @@ -1,6 +1,7 @@ #include "built-path.hh" #include "derivations.hh" #include "store-api.hh" +#include "comparator.hh" #include @@ -8,30 +9,24 @@ namespace nix { -#define CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, COMPARATOR) \ - bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \ - { \ - const MY_TYPE* me = this; \ - auto fields1 = std::tie(*me->drvPath, me->FIELD); \ - me = &other; \ - auto fields2 = std::tie(*me->drvPath, me->FIELD); \ - return fields1 COMPARATOR fields2; \ - } -#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, ==) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <) +// Custom implementation to avoid `ref` ptr equality +GENERATE_CMP_EXT( + , + std::strong_ordering, + SingleBuiltPathBuilt, + *me->drvPath, + me->output); -#define FIELD_TYPE std::pair -CMP(SingleBuiltPath, SingleBuiltPathBuilt, output) -#undef FIELD_TYPE +// Custom implementation to avoid `ref` ptr equality -#define FIELD_TYPE std::map -CMP(SingleBuiltPath, BuiltPathBuilt, outputs) -#undef FIELD_TYPE - -#undef CMP -#undef CMP_ONE +// TODO no `GENERATE_CMP_EXT` because no `std::set::operator<=>` on +// Darwin, per header. +GENERATE_EQUAL( + , + BuiltPathBuilt ::, + BuiltPathBuilt, + *me->drvPath, + me->outputs); StorePath SingleBuiltPath::outPath() const { diff --git a/src/libcmd/built-path.hh b/src/libcmd/built-path.hh index 99917e0ee..dc78d3e59 100644 --- a/src/libcmd/built-path.hh +++ b/src/libcmd/built-path.hh @@ -18,7 +18,8 @@ struct SingleBuiltPathBuilt { static SingleBuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view); nlohmann::json toJSON(const StoreDirConfig & store) const; - DECLARE_CMP(SingleBuiltPathBuilt); + bool operator ==(const SingleBuiltPathBuilt &) const noexcept; + std::strong_ordering operator <=>(const SingleBuiltPathBuilt &) const noexcept; }; using _SingleBuiltPathRaw = std::variant< @@ -33,6 +34,9 @@ struct SingleBuiltPath : _SingleBuiltPathRaw { using Opaque = DerivedPathOpaque; using Built = SingleBuiltPathBuilt; + bool operator == (const SingleBuiltPath &) const = default; + auto operator <=> (const SingleBuiltPath &) const = default; + inline const Raw & raw() const { return static_cast(*this); } @@ -59,11 +63,13 @@ struct BuiltPathBuilt { ref drvPath; std::map outputs; + bool operator == (const BuiltPathBuilt &) const noexcept; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //std::strong_ordering operator <=> (const BuiltPathBuilt &) const noexcept; + std::string to_string(const StoreDirConfig & store) const; static BuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view); nlohmann::json toJSON(const StoreDirConfig & store) const; - - DECLARE_CMP(BuiltPathBuilt); }; using _BuiltPathRaw = std::variant< @@ -82,6 +88,10 @@ struct BuiltPath : _BuiltPathRaw { using Opaque = DerivedPathOpaque; using Built = BuiltPathBuilt; + bool operator == (const BuiltPath &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=> (const BuiltPath &) const = default; + inline const Raw & raw() const { return static_cast(*this); } diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 369fa6004..67fef1909 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -1,3 +1,5 @@ +#include + #include "command.hh" #include "markdown.hh" #include "store-api.hh" @@ -6,8 +8,7 @@ #include "nixexpr.hh" #include "profiles.hh" #include "repl.hh" - -#include +#include "strings.hh" extern char * * environ __attribute__((weak)); @@ -127,12 +128,12 @@ ref EvalCommand::getEvalState() if (!evalState) { evalState = #if HAVE_BOEHMGC - std::allocate_shared(traceable_allocator(), - searchPath, getEvalStore(), getStore()) + std::allocate_shared( + traceable_allocator(), #else std::make_shared( - searchPath, getEvalStore(), getStore()) #endif + lookupPath, getEvalStore(), fetchSettings, evalSettings, getStore()) ; evalState->repair = repair; @@ -148,7 +149,7 @@ MixOperateOnOptions::MixOperateOnOptions() { addFlag({ .longName = "derivation", - .description = "Operate on the [store derivation](../../glossary.md#gloss-store-derivation) rather than its outputs.", + .description = "Operate on the [store derivation](@docroot@/glossary.md#gloss-store-derivation) rather than its outputs.", .category = installablesCategory, .handler = {&operateOn, OperateOn::Derivation}, }); diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 444ff81c9..fcef92487 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -1,18 +1,57 @@ +#include "fetch-settings.hh" #include "eval-settings.hh" #include "common-eval-args.hh" #include "shared.hh" +#include "config-global.hh" #include "filetransfer.hh" #include "eval.hh" #include "fetchers.hh" #include "registry.hh" #include "flake/flakeref.hh" +#include "flake/settings.hh" #include "store-api.hh" #include "command.hh" #include "tarball.hh" #include "fetch-to-store.hh" +#include "compatibility-settings.hh" +#include "eval-settings.hh" namespace nix { +fetchers::Settings fetchSettings; + +static GlobalConfig::Register rFetchSettings(&fetchSettings); + +EvalSettings evalSettings { + settings.readOnlyMode, + { + { + "flake", + [](ref store, std::string_view rest) { + experimentalFeatureSettings.require(Xp::Flakes); + // FIXME `parseFlakeRef` should take a `std::string_view`. + auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false); + debug("fetching flake search path element '%s''", rest); + auto storePath = flakeRef.resolve(store).fetchTree(store).first; + return store->toRealPath(storePath); + }, + }, + }, +}; + +static GlobalConfig::Register rEvalSettings(&evalSettings); + + +flake::Settings flakeSettings; + +static GlobalConfig::Register rFlakeSettings(&flakeSettings); + + +CompatibilitySettings compatibilitySettings {}; + +static GlobalConfig::Register rCompatibilitySettings(&compatibilitySettings); + + MixEvalArgs::MixEvalArgs() { addFlag({ @@ -20,7 +59,7 @@ MixEvalArgs::MixEvalArgs() .description = "Pass the value *expr* as the argument *name* to Nix functions.", .category = category, .labels = {"name", "expr"}, - .handler = {[&](std::string name, std::string expr) { autoArgs[name] = 'E' + expr; }} + .handler = {[&](std::string name, std::string expr) { autoArgs.insert_or_assign(name, AutoArg{AutoArgExpr{expr}}); }} }); addFlag({ @@ -28,87 +67,40 @@ MixEvalArgs::MixEvalArgs() .description = "Pass the string *string* as the argument *name* to Nix functions.", .category = category, .labels = {"name", "string"}, - .handler = {[&](std::string name, std::string s) { autoArgs[name] = 'S' + s; }}, + .handler = {[&](std::string name, std::string s) { autoArgs.insert_or_assign(name, AutoArg{AutoArgString{s}}); }}, + }); + + addFlag({ + .longName = "arg-from-file", + .description = "Pass the contents of file *path* as the argument *name* to Nix functions.", + .category = category, + .labels = {"name", "path"}, + .handler = {[&](std::string name, std::string path) { autoArgs.insert_or_assign(name, AutoArg{AutoArgFile{path}}); }}, + .completer = completePath + }); + + addFlag({ + .longName = "arg-from-stdin", + .description = "Pass the contents of stdin as the argument *name* to Nix functions.", + .category = category, + .labels = {"name"}, + .handler = {[&](std::string name) { autoArgs.insert_or_assign(name, AutoArg{AutoArgStdin{}}); }}, }); addFlag({ .longName = "include", .shortName = 'I', .description = R"( - Add *path* to the Nix search path. The Nix search path is - initialized from the colon-separated [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) environment - variable, and is used to look up the location of Nix expressions using [paths](@docroot@/language/values.md#type-path) enclosed in angle - brackets (i.e., ``). + Add *path* to search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md) - For instance, passing + This option may be given multiple times. - ``` - -I /home/eelco/Dev - -I /etc/nixos - ``` - - will cause Nix to look for paths relative to `/home/eelco/Dev` and - `/etc/nixos`, in that order. This is equivalent to setting the - `NIX_PATH` environment variable to - - ``` - /home/eelco/Dev:/etc/nixos - ``` - - It is also possible to match paths against a prefix. For example, - passing - - ``` - -I nixpkgs=/home/eelco/Dev/nixpkgs-branch - -I /etc/nixos - ``` - - will cause Nix to search for `` in - `/home/eelco/Dev/nixpkgs-branch/path` and `/etc/nixos/nixpkgs/path`. - - If a path in the Nix search path starts with `http://` or `https://`, - it is interpreted as the URL of a tarball that will be downloaded and - unpacked to a temporary location. The tarball must consist of a single - top-level directory. For example, passing - - ``` - -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/master.tar.gz - ``` - - tells Nix to download and use the current contents of the `master` - branch in the `nixpkgs` repository. - - The URLs of the tarballs from the official `nixos.org` channels - (see [the manual page for `nix-channel`](../nix-channel.md)) can be - abbreviated as `channel:`. For instance, the - following two flags are equivalent: - - ``` - -I nixpkgs=channel:nixos-21.05 - -I nixpkgs=https://nixos.org/channels/nixos-21.05/nixexprs.tar.xz - ``` - - You can also fetch source trees using [flake URLs](./nix3-flake.md#url-like-syntax) and add them to the - search path. For instance, - - ``` - -I nixpkgs=flake:nixpkgs - ``` - - specifies that the prefix `nixpkgs` shall refer to the source tree - downloaded from the `nixpkgs` entry in the flake registry. Similarly, - - ``` - -I nixpkgs=flake:github:NixOS/nixpkgs/nixos-22.05 - ``` - - makes `` refer to a particular branch of the - `NixOS/nixpkgs` repository on GitHub. + Paths added through `-I` take precedence over the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path) and the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH). )", .category = category, .labels = {"path"}, .handler = {[&](std::string s) { - searchPath.elements.emplace_back(SearchPath::Elem::parse(s)); + lookupPath.elements.emplace_back(LookupPath::Elem::parse(s)); }} }); @@ -127,8 +119,8 @@ MixEvalArgs::MixEvalArgs() .category = category, .labels = {"original-ref", "resolved-ref"}, .handler = {[&](std::string _from, std::string _to) { - auto from = parseFlakeRef(_from, absPath(".")); - auto to = parseFlakeRef(_to, absPath(".")); + auto from = parseFlakeRef(fetchSettings, _from, absPath(".")); + auto to = parseFlakeRef(fetchSettings, _to, absPath(".")); fetchers::Attrs extraAttrs; if (to.subdir != "") extraAttrs["dir"] = to.subdir; fetchers::overrideRegistry(from.input, to.input, extraAttrs); @@ -154,13 +146,23 @@ MixEvalArgs::MixEvalArgs() Bindings * MixEvalArgs::getAutoArgs(EvalState & state) { auto res = state.buildBindings(autoArgs.size()); - for (auto & i : autoArgs) { + for (auto & [name, arg] : autoArgs) { auto v = state.allocValue(); - if (i.second[0] == 'E') - state.mkThunk_(*v, state.parseExprFromString(i.second.substr(1), state.rootPath("."))); - else - v->mkString(((std::string_view) i.second).substr(1)); - res.insert(state.symbols.create(i.first), v); + std::visit(overloaded { + [&](const AutoArgExpr & arg) { + state.mkThunk_(*v, state.parseExprFromString(arg.expr, compatibilitySettings.nixShellShebangArgumentsRelativeToScript ? state.rootPath(absPath(getCommandBaseDir())) : state.rootPath("."))); + }, + [&](const AutoArgString & arg) { + v->mkString(arg.s); + }, + [&](const AutoArgFile & arg) { + v->mkString(readFile(arg.path.string())); + }, + [&](const AutoArgStdin & arg) { + v->mkString(readFile(STDIN_FILENO)); + } + }, arg); + res.insert(state.symbols.create(name), v); } return res.finish(); } @@ -176,7 +178,7 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas else if (hasPrefix(s, "flake:")) { experimentalFeatureSettings.require(Xp::Flakes); - auto flakeRef = parseFlakeRef(std::string(s.substr(6)), {}, true, false); + auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false); auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first; return state.rootPath(CanonPath(state.store->toRealPath(storePath))); } diff --git a/src/libcmd/common-eval-args.hh b/src/libcmd/common-eval-args.hh index 2eb63e15d..c62365b32 100644 --- a/src/libcmd/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh @@ -6,13 +6,42 @@ #include "common-args.hh" #include "search-path.hh" +#include + namespace nix { class Store; + +namespace fetchers { struct Settings; } + class EvalState; +struct EvalSettings; +struct CompatibilitySettings; class Bindings; struct SourcePath; +namespace flake { struct Settings; } + +/** + * @todo Get rid of global setttings variables + */ +extern fetchers::Settings fetchSettings; + +/** + * @todo Get rid of global setttings variables + */ +extern EvalSettings evalSettings; + +/** + * @todo Get rid of global setttings variables + */ +extern flake::Settings flakeSettings; + +/** + * Settings that control behaviors that have changed since Nix 2.3. + */ +extern CompatibilitySettings compatibilitySettings; + struct MixEvalArgs : virtual Args, virtual MixRepair { static constexpr auto category = "Common evaluation options"; @@ -21,14 +50,24 @@ struct MixEvalArgs : virtual Args, virtual MixRepair Bindings * getAutoArgs(EvalState & state); - SearchPath searchPath; + LookupPath lookupPath; std::optional evalStoreUrl; private: - std::map autoArgs; + struct AutoArgExpr { std::string expr; }; + struct AutoArgString { std::string s; }; + struct AutoArgFile { std::filesystem::path path; }; + struct AutoArgStdin { }; + + using AutoArg = std::variant; + + std::map autoArgs; }; +/** + * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) + */ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * baseDir = nullptr); } diff --git a/src/libcmd/compatibility-settings.hh b/src/libcmd/compatibility-settings.hh new file mode 100644 index 000000000..a129a957a --- /dev/null +++ b/src/libcmd/compatibility-settings.hh @@ -0,0 +1,36 @@ +#pragma once +#include "config.hh" + +namespace nix { +struct CompatibilitySettings : public Config +{ + + CompatibilitySettings() = default; + + // Added in Nix 2.24, July 2024. + Setting nixShellAlwaysLooksForShellNix{this, true, "nix-shell-always-looks-for-shell-nix", R"( + Before Nix 2.24, [`nix-shell`](@docroot@/command-ref/nix-shell.md) would only look at `shell.nix` if it was in the working directory - when no file was specified. + + Since Nix 2.24, `nix-shell` always looks for a `shell.nix`, whether that's in the working directory, or in a directory that was passed as an argument. + + You may set this to `false` to temporarily revert to the behavior of Nix 2.23 and older. + + Using this setting is not recommended. + It will be deprecated and removed. + )"}; + + // Added in Nix 2.24, July 2024. + Setting nixShellShebangArgumentsRelativeToScript{ + this, true, "nix-shell-shebang-arguments-relative-to-script", R"( + Before Nix 2.24, relative file path expressions in arguments in a `nix-shell` shebang were resolved relative to the working directory. + + Since Nix 2.24, `nix-shell` resolves these paths in a manner that is relative to the [base directory](@docroot@/glossary.md#gloss-base-directory), defined as the script's directory. + + You may set this to `false` to temporarily revert to the behavior of Nix 2.23 and older. + + Using this setting is not recommended. + It will be deprecated and removed. + )"}; +}; + +}; diff --git a/src/libcmd/installable-attr-path.cc b/src/libcmd/installable-attr-path.cc index 3ec1c1614..8917e7a01 100644 --- a/src/libcmd/installable-attr-path.cc +++ b/src/libcmd/installable-attr-path.cc @@ -75,6 +75,8 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths() std::set outputsToInstall; for (auto & output : packageInfo.queryOutputs(false, true)) outputsToInstall.insert(output.first); + if (outputsToInstall.empty()) + outputsToInstall.insert("out"); return OutputsSpec::Names { std::move(outputsToInstall) }; }, [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec { diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index ddec7537b..8796ad5ba 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -43,20 +43,6 @@ std::vector InstallableFlake::getActualAttrPaths() return res; } -Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake) -{ - auto vFlake = state.allocValue(); - - callFlake(state, lockedFlake, *vFlake); - - auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); - assert(aOutputs); - - state.forceValue(*aOutputs->value, aOutputs->value->determinePos(noPos)); - - return aOutputs->value; -} - static std::string showAttrPaths(const std::vector & paths) { std::string s; @@ -106,9 +92,14 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() fmt("while evaluating the flake output attribute '%s'", attrPath))) { return { *derivedPathWithInfo }; + } else { + throw Error( + "expected flake output attribute '%s' to be a derivation or path but found %s: %s", + attrPath, + showType(v), + ValuePrinter(*this->state, v, errorPrintOptions) + ); } - else - throw Error("flake output attribute '%s' is not a derivation or path", attrPath); } auto drvPath = attr->forceDerivation(); @@ -205,7 +196,8 @@ std::shared_ptr InstallableFlake::getLockedFlake() const flake::LockFlags lockFlagsApplyConfig = lockFlags; // FIXME why this side effect? lockFlagsApplyConfig.applyNixConfig = true; - _lockedFlake = std::make_shared(lockFlake(*state, flakeRef, lockFlagsApplyConfig)); + _lockedFlake = std::make_shared(lockFlake( + flakeSettings, *state, flakeRef, lockFlagsApplyConfig)); } return _lockedFlake; } diff --git a/src/libcmd/installable-flake.hh b/src/libcmd/installable-flake.hh index 314918c14..8e0a232ef 100644 --- a/src/libcmd/installable-flake.hh +++ b/src/libcmd/installable-flake.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "common-eval-args.hh" #include "installable-value.hh" namespace nix { @@ -52,8 +53,6 @@ struct InstallableFlake : InstallableValue std::vector getActualAttrPaths(); - Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake); - DerivedPathsWithInfo toDerivedPaths() override; std::pair toValue(EvalState & state) override; @@ -80,7 +79,7 @@ struct InstallableFlake : InstallableValue */ static inline FlakeRef defaultNixpkgsFlakeRef() { - return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}}); + return FlakeRef::fromAttrs(fetchSettings, {{"type","indirect"}, {"id", "nixpkgs"}}); } ref openEvalCache( diff --git a/src/libcmd/installable-value.hh b/src/libcmd/installable-value.hh index f300d392b..798cb5e1a 100644 --- a/src/libcmd/installable-value.hh +++ b/src/libcmd/installable-value.hh @@ -66,7 +66,7 @@ struct ExtraPathInfoValue : ExtraPathInfo }; /** - * An Installable which corresponds a Nix langauge value, in addition to + * An Installable which corresponds a Nix language value, in addition to * a collection of \ref DerivedPath "derived paths". */ struct InstallableValue : Installable diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 16d25d3cf..0fe956ec0 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -27,6 +27,8 @@ #include +#include "strings-inline.hh" + namespace nix { void completeFlakeInputPath( @@ -129,7 +131,7 @@ MixFlakeOptions::MixFlakeOptions() lockFlags.writeLockFile = false; lockFlags.inputOverrides.insert_or_assign( flake::parseInputPath(inputPath), - parseFlakeRef(flakeRef, absPath(getCommandBaseDir()), true)); + parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir()), true)); }}, .completer = {[&](AddCompletions & completions, size_t n, std::string_view prefix) { if (n == 0) { @@ -146,7 +148,7 @@ MixFlakeOptions::MixFlakeOptions() .category = category, .labels = {"flake-lock-path"}, .handler = {[&](std::string lockFilePath) { - lockFlags.referenceLockFilePath = lockFilePath; + lockFlags.referenceLockFilePath = {getFSSourceAccessor(), CanonPath(absPath(lockFilePath))}; }}, .completer = completePath }); @@ -170,14 +172,15 @@ MixFlakeOptions::MixFlakeOptions() .handler = {[&](std::string flakeRef) { auto evalState = getEvalState(); auto flake = flake::lockFlake( + flakeSettings, *evalState, - parseFlakeRef(flakeRef, absPath(getCommandBaseDir())), + parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir())), { .writeLockFile = false }); for (auto & [inputName, input] : flake.lockFile.root->inputs) { auto input2 = flake.lockFile.findInput({inputName}); // resolve 'follows' nodes if (auto input3 = std::dynamic_pointer_cast(input2)) { overrideRegistry( - fetchers::Input::fromAttrs({{"type","indirect"}, {"id", inputName}}), + fetchers::Input::fromAttrs(fetchSettings, {{"type","indirect"}, {"id", inputName}}), input3->lockedRef.input, {}); } @@ -288,11 +291,11 @@ void SourceExprCommand::completeInstallable(AddCompletions & completions, std::s state->autoCallFunction(*autoArgs, v1, v2); if (v2.type() == nAttrs) { - for (auto & i : *v2.attrs) { - std::string name = state->symbols[i.name]; + for (auto & i : *v2.attrs()) { + std::string_view name = state->symbols[i.name]; if (name.find(searchWord) == 0) { if (prefix_ == "") - completions.add(name); + completions.add(std::string(name)); else completions.add(prefix_ + "." + name); } @@ -338,10 +341,11 @@ void completeFlakeRefWithFragment( auto flakeRefS = std::string(prefix.substr(0, hash)); // TODO: ideally this would use the command base directory instead of assuming ".". - auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath(".")); + auto flakeRef = parseFlakeRef(fetchSettings, expandTilde(flakeRefS), absPath(".")); auto evalCache = openEvalCache(*evalState, - std::make_shared(lockFlake(*evalState, flakeRef, lockFlags))); + std::make_shared(lockFlake( + flakeSettings, *evalState, flakeRef, lockFlags))); auto root = evalCache->getRoot(); @@ -372,6 +376,7 @@ void completeFlakeRefWithFragment( auto attrPath2 = (*attr)->getAttrPath(attr2); /* Strip the attrpath prefix. */ attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size()); + // FIXME: handle names with dots completions.add(flakeRefS + "#" + prefixRoot + concatStringsSep(".", evalState->symbols.resolve(attrPath2))); } } @@ -403,7 +408,7 @@ void completeFlakeRef(AddCompletions & completions, ref store, std::strin Args::completeDir(completions, 0, prefix); /* Look for registry entries that match the prefix. */ - for (auto & registry : fetchers::getRegistries(store)) { + for (auto & registry : fetchers::getRegistries(fetchSettings, store)) { for (auto & entry : registry->entries) { auto from = entry.from.to_string(); if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) { @@ -442,13 +447,10 @@ ref openEvalCache( EvalState & state, std::shared_ptr lockedFlake) { - auto fingerprint = lockedFlake->getFingerprint(); - return make_ref( - evalSettings.useEvalCache && evalSettings.pureEval - ? std::optional { std::cref(fingerprint) } - : std::nullopt, - state, - [&state, lockedFlake]() + auto fingerprint = evalSettings.useEvalCache && evalSettings.pureEval + ? lockedFlake->getFingerprint(state.store) + : std::nullopt; + auto rootLoader = [&state, lockedFlake]() { /* For testing whether the evaluation cache is complete. */ @@ -460,11 +462,21 @@ ref openEvalCache( state.forceAttrs(*vFlake, noPos, "while parsing cached flake data"); - auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); + auto aOutputs = vFlake->attrs()->get(state.symbols.create("outputs")); assert(aOutputs); return aOutputs->value; - }); + }; + + if (fingerprint) { + auto search = state.evalCaches.find(fingerprint.value()); + if (search == state.evalCaches.end()) { + search = state.evalCaches.emplace(fingerprint.value(), make_ref(fingerprint, state, rootLoader)).first; + } + return search->second; + } else { + return make_ref(std::nullopt, state, rootLoader); + } } Installables SourceExprCommand::parseInstallables( @@ -527,7 +539,8 @@ Installables SourceExprCommand::parseInstallables( } try { - auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, absPath(getCommandBaseDir())); + auto [flakeRef, fragment] = parseFlakeRefWithFragment( + fetchSettings, std::string { prefix }, absPath(getCommandBaseDir())); result.push_back(make_ref( this, getEvalState(), @@ -594,6 +607,37 @@ std::vector Installable::build( return res; } +static void throwBuildErrors( + std::vector & buildResults, + const Store & store) +{ + std::vector failed; + for (auto & buildResult : buildResults) { + if (!buildResult.success()) { + failed.push_back(buildResult); + } + } + + auto failedResult = failed.begin(); + if (failedResult != failed.end()) { + if (failed.size() == 1) { + failedResult->rethrow(); + } else { + StringSet failedPaths; + for (; failedResult != failed.end(); failedResult++) { + if (!failedResult->errorMsg.empty()) { + logError(ErrorInfo{ + .level = lvlError, + .msg = failedResult->errorMsg, + }); + } + failedPaths.insert(failedResult->path.to_string(store)); + } + throw Error("build of %s failed", concatStringsSep(", ", quoteStrings(failedPaths))); + } + } +} + std::vector, BuiltPathWithResult>> Installable::build2( ref evalStore, ref store, @@ -655,10 +699,9 @@ std::vector, BuiltPathWithResult>> Installable::build if (settings.printMissing) printMissing(store, pathsToBuild, lvlInfo); - for (auto & buildResult : store->buildPathsWithResults(pathsToBuild, bMode, evalStore)) { - if (!buildResult.success()) - buildResult.rethrow(); - + auto buildResults = store->buildPathsWithResults(pathsToBuild, bMode, evalStore); + throwBuildErrors(buildResults, *store); + for (auto & buildResult : buildResults) { for (auto & aux : backmap[buildResult.path]) { std::visit(overloaded { [&](const DerivedPath::Built & bfd) { @@ -814,6 +857,7 @@ std::vector RawInstallablesCommand::getFlakeRefsForCompletion() std::vector res; for (auto i : rawInstallables) res.push_back(parseFlakeRefWithFragment( + fetchSettings, expandTilde(i), absPath(getCommandBaseDir())).first); return res; @@ -836,6 +880,7 @@ std::vector InstallableCommand::getFlakeRefsForCompletion() { return { parseFlakeRefWithFragment( + fetchSettings, expandTilde(_installable), absPath(getCommandBaseDir())).first }; diff --git a/src/libcmd/local.mk b/src/libcmd/local.mk index abb7459a7..a270333f4 100644 --- a/src/libcmd/local.mk +++ b/src/libcmd/local.mk @@ -6,10 +6,10 @@ libcmd_DIR := $(d) libcmd_SOURCES := $(wildcard $(d)/*.cc) -libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers +libcmd_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) $(INCLUDE_libflake) $(INCLUDE_libmain) libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) $(THREAD_LDFLAGS) -libcmd_LIBS = libstore libutil libexpr libmain libfetchers +libcmd_LIBS = libutil libstore libfetchers libflake libexpr libmain $(eval $(call install-file-in, $(buildprefix)$(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644)) diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc index a4e3c5a77..6a0d05d9f 100644 --- a/src/libcmd/markdown.cc +++ b/src/libcmd/markdown.cc @@ -1,21 +1,23 @@ #include "markdown.hh" -#include "util.hh" +#include "environment-variables.hh" +#include "error.hh" #include "finally.hh" #include "terminal.hh" -#include #if HAVE_LOWDOWN -#include +# include +# include #endif namespace nix { -std::string renderMarkdownToTerminal(std::string_view markdown) -{ #if HAVE_LOWDOWN +static std::string doRenderMarkdownToTerminal(std::string_view markdown) +{ int windowWidth = getWindowSize().second; - struct lowdown_opts opts { + struct lowdown_opts opts + { .type = LOWDOWN_TERM, .maxdepth = 20, .cols = (size_t) std::max(windowWidth - 5, 60), @@ -50,10 +52,22 @@ std::string renderMarkdownToTerminal(std::string_view markdown) if (!rndr_res) throw Error("allocation error while rendering Markdown"); - return filterANSIEscapes(std::string(buf->data, buf->size), !shouldANSI()); -#else - return std::string(markdown); -#endif + return filterANSIEscapes(std::string(buf->data, buf->size), !isTTY()); } +std::string renderMarkdownToTerminal(std::string_view markdown) +{ + if (auto e = getEnv("_NIX_TEST_RAW_MARKDOWN"); e && *e == "1") + return std::string(markdown); + else + return doRenderMarkdownToTerminal(markdown); } + +#else +std::string renderMarkdownToTerminal(std::string_view markdown) +{ + return std::string(markdown); +} +#endif + +} // namespace nix diff --git a/src/libcmd/markdown.hh b/src/libcmd/markdown.hh index a04d32a4f..66db1736c 100644 --- a/src/libcmd/markdown.hh +++ b/src/libcmd/markdown.hh @@ -1,10 +1,17 @@ #pragma once ///@file -#include "types.hh" +#include namespace nix { +/** + * Render the given Markdown text to the terminal. + * + * If Nix is compiled without Markdown support, this function will return the input text as-is. + * + * The renderer takes into account the terminal width, and wraps text accordingly. + */ std::string renderMarkdownToTerminal(std::string_view markdown); } diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build new file mode 100644 index 000000000..c484cf998 --- /dev/null +++ b/src/libcmd/meson.build @@ -0,0 +1,130 @@ +project('nix-cmd', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ +] +deps_public_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), + dependency('nix-fetchers'), + dependency('nix-expr'), + dependency('nix-flake'), + dependency('nix-main'), +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') +deps_public += nlohmann_json + +lowdown = dependency('lowdown', version : '>= 0.9.0', required : get_option('markdown')) +deps_private += lowdown +configdata.set('HAVE_LOWDOWN', lowdown.found().to_int()) + +readline_flavor = get_option('readline-flavor') +if readline_flavor == 'editline' + editline = dependency('libeditline', 'editline', version : '>=1.14') + deps_private += editline +elif readline_flavor == 'readline' + readline = dependency('readline') + deps_private += readline + configdata.set( + 'USE_READLINE', + 1, + description: 'Use readline instead of editline', + ) +else + error('illegal editline flavor', readline_flavor) +endif + +config_h = configure_file( + configuration : configdata, + output : 'config-cmd.hh', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + # '-include', 'config-fetchers.h', + '-include', 'config-expr.hh', + '-include', 'config-main.hh', + '-include', 'config-cmd.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'built-path.cc', + 'command-installable-value.cc', + 'command.cc', + 'common-eval-args.cc', + 'editor-for.cc', + 'installable-attr-path.cc', + 'installable-derived-path.cc', + 'installable-flake.cc', + 'installable-value.cc', + 'installables.cc', + 'legacy.cc', + 'markdown.cc', + 'misc-store-flags.cc', + 'network-proxy.cc', + 'repl-interacter.cc', + 'repl.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'built-path.hh', + 'command-installable-value.hh', + 'command.hh', + 'common-eval-args.hh', + 'compatibility-settings.hh', + 'editor-for.hh', + 'installable-attr-path.hh', + 'installable-derived-path.hh', + 'installable-flake.hh', + 'installable-value.hh', + 'installables.hh', + 'legacy.hh', + 'markdown.hh', + 'misc-store-flags.hh', + 'network-proxy.hh', + 'repl-interacter.hh', + 'repl.hh', +) + +this_library = library( + 'nixcmd', + sources, + dependencies : deps_public + deps_private + deps_other, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libcmd/meson.options b/src/libcmd/meson.options new file mode 100644 index 000000000..79ae4fa55 --- /dev/null +++ b/src/libcmd/meson.options @@ -0,0 +1,15 @@ +# vim: filetype=meson + +option( + 'markdown', + type: 'feature', + description: 'Enable Markdown rendering in the Nix binary (requires lowdown)', +) + +option( + 'readline-flavor', + type : 'combo', + choices : ['editline', 'readline'], + value : 'editline', + description : 'Which library to use for nice line editing with the Nix language REPL', +) diff --git a/src/libcmd/misc-store-flags.cc b/src/libcmd/misc-store-flags.cc index e66d3f63b..06552c032 100644 --- a/src/libcmd/misc-store-flags.cc +++ b/src/libcmd/misc-store-flags.cc @@ -81,9 +81,15 @@ Args::Flag fileIngestionMethod(FileIngestionMethod * method) How to compute the hash of the input. One of: - - `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function. + - `nar` (the default): + Serialises the input as a + [Nix Archive](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) + and passes that to the hash function. - - `flat`: Assumes that the input is a single file and directly passes it to the hash function; + - `flat`: + Assumes that the input is a single file and + [directly passes](@docroot@/store/file-system-object/content-address.md#serial-flat) + it to the hash function. )", .labels = {"file-ingestion-method"}, .handler = {[method](std::string s) { @@ -101,15 +107,23 @@ Args::Flag contentAddressMethod(ContentAddressMethod * method) How to compute the content-address of the store object. One of: - - `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function. + - [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive) + (the default): + Serialises the input as a + [Nix Archive](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) + and passes that to the hash function. - - `flat`: Assumes that the input is a single file and directly passes it to the hash function; + - [`flat`](@docroot@/store/store-object/content-address.md#method-flat): + Assumes that the input is a single file and + [directly passes](@docroot@/store/file-system-object/content-address.md#serial-flat) + it to the hash function. - - `text`: Like `flat`, but used for - [derivations](@docroot@/glossary.md#store-derivation) serialized in store object and + - [`text`](@docroot@/store/store-object/content-address.md#method-text): + Like `flat`, but used for + [derivations](@docroot@/glossary.md#store-derivation) serialized in store object and [`builtins.toFile`](@docroot@/language/builtins.html#builtins-toFile). For advanced use-cases only; - for regular usage prefer `nar` and `flat. + for regular usage prefer `nar` and `flat`. )", .labels = {"content-address-method"}, .handler = {[method](std::string s) { diff --git a/src/libcmd/network-proxy.cc b/src/libcmd/network-proxy.cc new file mode 100644 index 000000000..738bf6147 --- /dev/null +++ b/src/libcmd/network-proxy.cc @@ -0,0 +1,50 @@ +#include "network-proxy.hh" + +#include + +#include "environment-variables.hh" + +namespace nix { + +static const StringSet lowercaseVariables{"http_proxy", "https_proxy", "ftp_proxy", "all_proxy", "no_proxy"}; + +static StringSet getAllVariables() +{ + StringSet variables = lowercaseVariables; + for (const auto & variable : lowercaseVariables) { + std::string upperVariable; + std::transform( + variable.begin(), variable.end(), upperVariable.begin(), [](unsigned char c) { return std::toupper(c); }); + variables.insert(std::move(upperVariable)); + } + return variables; +} + +const StringSet networkProxyVariables = getAllVariables(); + +static StringSet getExcludingNoProxyVariables() +{ + static const StringSet excludeVariables{"no_proxy", "NO_PROXY"}; + StringSet variables; + std::set_difference( + networkProxyVariables.begin(), + networkProxyVariables.end(), + excludeVariables.begin(), + excludeVariables.end(), + std::inserter(variables, variables.begin())); + return variables; +} + +static const StringSet excludingNoProxyVariables = getExcludingNoProxyVariables(); + +bool haveNetworkProxyConnection() +{ + for (const auto & variable : excludingNoProxyVariables) { + if (getEnv(variable).has_value()) { + return true; + } + } + return false; +} + +} diff --git a/src/libcmd/network-proxy.hh b/src/libcmd/network-proxy.hh new file mode 100644 index 000000000..0b6856acb --- /dev/null +++ b/src/libcmd/network-proxy.hh @@ -0,0 +1,22 @@ +#pragma once +///@file + +#include "types.hh" + +namespace nix { + +/** + * Environment variables relating to network proxying. These are used by + * a few misc commands. + * + * See the Environment section of https://curl.se/docs/manpage.html for details. + */ +extern const StringSet networkProxyVariables; + +/** + * Heuristically check if there is a proxy connection by checking for defined + * proxy variables. + */ +bool haveNetworkProxyConnection(); + +} diff --git a/src/libcmd/package.nix b/src/libcmd/package.nix new file mode 100644 index 000000000..cde494901 --- /dev/null +++ b/src/libcmd/package.nix @@ -0,0 +1,104 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-util +, nix-store +, nix-fetchers +, nix-expr +, nix-flake +, nix-main +, editline +, readline +, lowdown +, nlohmann_json + +# Configuration Options + +, version + +# Whether to enable Markdown rendering in the Nix binary. +, enableMarkdown ? !stdenv.hostPlatform.isWindows + +# Which interactive line editor library to use for Nix's repl. +# +# Currently supported choices are: +# +# - editline (default) +# - readline +, readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline" +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-cmd"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + ({ inherit editline readline; }.${readlineFlavor}) + ] ++ lib.optional enableMarkdown lowdown; + + propagatedBuildInputs = [ + nix-util + nix-store + nix-fetchers + nix-expr + nix-flake + nix-main + nlohmann_json + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + mesonFlags = [ + (lib.mesonEnable "markdown" enableMarkdown) + (lib.mesonOption "readline-flavor" readlineFlavor) + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libcmd/repl-interacter.cc b/src/libcmd/repl-interacter.cc new file mode 100644 index 000000000..187af46ea --- /dev/null +++ b/src/libcmd/repl-interacter.cc @@ -0,0 +1,206 @@ +#include + +#ifdef USE_READLINE +#include +#include +#else +// editline < 1.15.2 don't wrap their API for C++ usage +// (added in https://github.com/troglobit/editline/commit/91398ceb3427b730995357e9d120539fb9bb7461). +// This results in linker errors due to to name-mangling of editline C symbols. +// For compatibility with these versions, we wrap the API here +// (wrapping multiple times on newer versions is no problem). +extern "C" { +#include +} +#endif + +#include "signals.hh" +#include "finally.hh" +#include "repl-interacter.hh" +#include "file-system.hh" +#include "repl.hh" +#include "environment-variables.hh" + +namespace nix { + +namespace { +// Used to communicate to NixRepl::getLine whether a signal occurred in ::readline. +volatile sig_atomic_t g_signal_received = 0; + +void sigintHandler(int signo) +{ + g_signal_received = signo; +} +}; + +static detail::ReplCompleterMixin * curRepl; // ugly + +#ifndef USE_READLINE +static char * completionCallback(char * s, int * match) +{ + auto possible = curRepl->completePrefix(s); + if (possible.size() == 1) { + *match = 1; + auto * res = strdup(possible.begin()->c_str() + strlen(s)); + if (!res) + throw Error("allocation failure"); + return res; + } else if (possible.size() > 1) { + auto checkAllHaveSameAt = [&](size_t pos) { + auto & first = *possible.begin(); + for (auto & p : possible) { + if (p.size() <= pos || p[pos] != first[pos]) + return false; + } + return true; + }; + size_t start = strlen(s); + size_t len = 0; + while (checkAllHaveSameAt(start + len)) + ++len; + if (len > 0) { + *match = 1; + auto * res = strdup(std::string(*possible.begin(), start, len).c_str()); + if (!res) + throw Error("allocation failure"); + return res; + } + } + + *match = 0; + return nullptr; +} + +static int listPossibleCallback(char * s, char *** avp) +{ + auto possible = curRepl->completePrefix(s); + + if (possible.size() > (std::numeric_limits::max() / sizeof(char *))) + throw Error("too many completions"); + + int ac = 0; + char ** vp = nullptr; + + auto check = [&](auto * p) { + if (!p) { + if (vp) { + while (--ac >= 0) + free(vp[ac]); + free(vp); + } + throw Error("allocation failure"); + } + return p; + }; + + vp = check((char **) malloc(possible.size() * sizeof(char *))); + + for (auto & p : possible) + vp[ac++] = check(strdup(p.c_str())); + + *avp = vp; + + return ac; +} +#endif + +ReadlineLikeInteracter::Guard ReadlineLikeInteracter::init(detail::ReplCompleterMixin * repl) +{ + // Allow nix-repl specific settings in .inputrc + rl_readline_name = "nix-repl"; + try { + createDirs(dirOf(historyFile)); + } catch (SystemError & e) { + logWarning(e.info()); + } +#ifndef USE_READLINE + el_hist_size = 1000; +#endif + read_history(historyFile.c_str()); + auto oldRepl = curRepl; + curRepl = repl; + Guard restoreRepl([oldRepl] { curRepl = oldRepl; }); +#ifndef USE_READLINE + rl_set_complete_func(completionCallback); + rl_set_list_possib_func(listPossibleCallback); +#endif + return restoreRepl; +} + +static constexpr const char * promptForType(ReplPromptType promptType) +{ + switch (promptType) { + case ReplPromptType::ReplPrompt: + return "nix-repl> "; + case ReplPromptType::ContinuationPrompt: + return " "; + } + assert(false); +} + +bool ReadlineLikeInteracter::getLine(std::string & input, ReplPromptType promptType) +{ +#ifndef _WIN32 // TODO use more signals.hh for this + struct sigaction act, old; + sigset_t savedSignalMask, set; + + auto setupSignals = [&]() { + act.sa_handler = sigintHandler; + sigfillset(&act.sa_mask); + act.sa_flags = 0; + if (sigaction(SIGINT, &act, &old)) + throw SysError("installing handler for SIGINT"); + + sigemptyset(&set); + sigaddset(&set, SIGINT); + if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask)) + throw SysError("unblocking SIGINT"); + }; + auto restoreSignals = [&]() { + if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) + throw SysError("restoring signals"); + + if (sigaction(SIGINT, &old, 0)) + throw SysError("restoring handler for SIGINT"); + }; + + setupSignals(); +#endif + char * s = readline(promptForType(promptType)); + Finally doFree([&]() { free(s); }); +#ifndef _WIN32 // TODO use more signals.hh for this + restoreSignals(); +#endif + + if (g_signal_received) { + g_signal_received = 0; + input.clear(); + return true; + } + + // editline doesn't echo the input to the output when non-interactive, unlike readline + // this results in a different behavior when running tests. The echoing is + // quite useful for reading the test output, so we add it here. + if (auto e = getEnv("_NIX_TEST_REPL_ECHO"); s && e && *e == "1") + { +#ifndef USE_READLINE + // This is probably not right for multi-line input, but we don't use that + // in the characterisation tests, so it's fine. + std::cout << promptForType(promptType) << s << std::endl; +#endif + } + + if (!s) + return false; + input += s; + input += '\n'; + + return true; +} + +ReadlineLikeInteracter::~ReadlineLikeInteracter() +{ + write_history(historyFile.c_str()); +} + +}; diff --git a/src/libcmd/repl-interacter.hh b/src/libcmd/repl-interacter.hh new file mode 100644 index 000000000..cc70efd07 --- /dev/null +++ b/src/libcmd/repl-interacter.hh @@ -0,0 +1,48 @@ +#pragma once +/// @file + +#include "finally.hh" +#include "types.hh" +#include +#include + +namespace nix { + +namespace detail { +/** Provides the completion hooks for the repl, without exposing its complete + * internals. */ +struct ReplCompleterMixin { + virtual StringSet completePrefix(const std::string & prefix) = 0; +}; +}; + +enum class ReplPromptType { + ReplPrompt, + ContinuationPrompt, +}; + +class ReplInteracter +{ +public: + using Guard = Finally>; + + virtual Guard init(detail::ReplCompleterMixin * repl) = 0; + /** Returns a boolean of whether the interacter got EOF */ + virtual bool getLine(std::string & input, ReplPromptType promptType) = 0; + virtual ~ReplInteracter(){}; +}; + +class ReadlineLikeInteracter : public virtual ReplInteracter +{ + std::string historyFile; +public: + ReadlineLikeInteracter(std::string historyFile) + : historyFile(historyFile) + { + } + virtual Guard init(detail::ReplCompleterMixin * repl) override; + virtual bool getLine(std::string & input, ReplPromptType promptType) override; + virtual ~ReadlineLikeInteracter() override; +}; + +}; diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 8b83608fa..efc04b029 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -1,34 +1,17 @@ #include #include #include -#include - -#include - -#ifdef USE_READLINE -#include -#include -#else -// editline < 1.15.2 don't wrap their API for C++ usage -// (added in https://github.com/troglobit/editline/commit/91398ceb3427b730995357e9d120539fb9bb7461). -// This results in linker errors due to to name-mangling of editline C symbols. -// For compatibility with these versions, we wrap the API here -// (wrapping multiple times on newer versions is no problem). -extern "C" { -#include -} -#endif +#include "repl-interacter.hh" #include "repl.hh" #include "ansicolor.hh" -#include "signals.hh" #include "shared.hh" +#include "config-global.hh" #include "eval.hh" -#include "eval-cache.hh" -#include "eval-inline.hh" #include "eval-settings.hh" #include "attr-path.hh" +#include "signals.hh" #include "store-api.hh" #include "log-store.hh" #include "common-eval-args.hh" @@ -38,18 +21,21 @@ extern "C" { #include "flake/flake.hh" #include "flake/lockfile.hh" #include "users.hh" -#include "terminal.hh" #include "editor-for.hh" #include "finally.hh" #include "markdown.hh" #include "local-fs-store.hh" #include "print.hh" +#include "ref.hh" +#include "value.hh" #if HAVE_BOEHMGC #define GC_INCLUDE_NEW #include #endif +#include "strings.hh" + namespace nix { /** @@ -75,6 +61,7 @@ enum class ProcessLineResult { struct NixRepl : AbstractNixRepl + , detail::ReplCompleterMixin #if HAVE_BOEHMGC , gc #endif @@ -90,17 +77,16 @@ struct NixRepl int displ; StringSet varNames; - const Path historyFile; + std::unique_ptr interacter; - NixRepl(const SearchPath & searchPath, nix::ref store,ref state, + NixRepl(const LookupPath & lookupPath, nix::ref store,ref state, std::function getValues); - virtual ~NixRepl(); + virtual ~NixRepl() = default; ReplExitStatus mainLoop() override; void initEnv() override; - StringSet completePrefix(const std::string & prefix); - bool getLine(std::string & input, const std::string & prompt); + virtual StringSet completePrefix(const std::string & prefix) override; StorePath getDerivationPath(Value & v); ProcessLineResult processLine(std::string line); @@ -123,7 +109,8 @@ struct NixRepl .force = true, .derivationPaths = true, .maxDepth = maxDepth, - .prettyIndent = 2 + .prettyIndent = 2, + .errors = ErrorPrintBehavior::ThrowTopLevel, }); } }; @@ -137,111 +124,33 @@ std::string removeWhitespace(std::string s) } -NixRepl::NixRepl(const SearchPath & searchPath, nix::ref store, ref state, +NixRepl::NixRepl(const LookupPath & lookupPath, nix::ref store, ref state, std::function getValues) : AbstractNixRepl(state) , debugTraceIndex(0) , getValues(getValues) , staticEnv(new StaticEnv(nullptr, state->staticBaseEnv.get())) - , historyFile(getDataDir() + "/nix/repl-history") + , interacter(make_unique(getDataDir() + "/nix/repl-history")) { } - -NixRepl::~NixRepl() -{ - write_history(historyFile.c_str()); -} - void runNix(Path program, const Strings & args, const std::optional & input = {}) { auto subprocessEnv = getEnv(); subprocessEnv["NIX_CONFIG"] = globalConfig.toKeyValue(); - + //isInteractive avoid grabling interactive commands runProgram2(RunOptions { .program = settings.nixBinDir+ "/" + program, .args = args, .environment = subprocessEnv, .input = input, + .isInteractive = true, }); return; } -static NixRepl * curRepl; // ugly - -static char * completionCallback(char * s, int *match) { - auto possible = curRepl->completePrefix(s); - if (possible.size() == 1) { - *match = 1; - auto *res = strdup(possible.begin()->c_str() + strlen(s)); - if (!res) throw Error("allocation failure"); - return res; - } else if (possible.size() > 1) { - auto checkAllHaveSameAt = [&](size_t pos) { - auto &first = *possible.begin(); - for (auto &p : possible) { - if (p.size() <= pos || p[pos] != first[pos]) - return false; - } - return true; - }; - size_t start = strlen(s); - size_t len = 0; - while (checkAllHaveSameAt(start + len)) ++len; - if (len > 0) { - *match = 1; - auto *res = strdup(std::string(*possible.begin(), start, len).c_str()); - if (!res) throw Error("allocation failure"); - return res; - } - } - - *match = 0; - return nullptr; -} - -static int listPossibleCallback(char *s, char ***avp) { - auto possible = curRepl->completePrefix(s); - - if (possible.size() > (INT_MAX / sizeof(char*))) - throw Error("too many completions"); - - int ac = 0; - char **vp = nullptr; - - auto check = [&](auto *p) { - if (!p) { - if (vp) { - while (--ac >= 0) - free(vp[ac]); - free(vp); - } - throw Error("allocation failure"); - } - return p; - }; - - vp = check((char **)malloc(possible.size() * sizeof(char*))); - - for (auto & p : possible) - vp[ac++] = check(strdup(p.c_str())); - - *avp = vp; - - return ac; -} - -namespace { - // Used to communicate to NixRepl::getLine whether a signal occurred in ::readline. - volatile sig_atomic_t g_signal_received = 0; - - void sigintHandler(int signo) { - g_signal_received = signo; - } -} - static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positions, const DebugTrace & dt) { if (dt.isError) @@ -281,24 +190,7 @@ ReplExitStatus NixRepl::mainLoop() loadFiles(); - // Allow nix-repl specific settings in .inputrc - rl_readline_name = "nix-repl"; - try { - createDirs(dirOf(historyFile)); - } catch (SystemError & e) { - logWarning(e.info()); - } -#ifndef USE_READLINE - el_hist_size = 1000; -#endif - read_history(historyFile.c_str()); - auto oldRepl = curRepl; - curRepl = this; - Finally restoreRepl([&] { curRepl = oldRepl; }); -#ifndef USE_READLINE - rl_set_complete_func(completionCallback); - rl_set_list_possib_func(listPossibleCallback); -#endif + auto _guard = interacter->init(static_cast(this)); std::string input; @@ -307,7 +199,7 @@ ReplExitStatus NixRepl::mainLoop() logger->pause(); // When continuing input from previous lines, don't print a prompt, just align to the same // number of chars as the prompt. - if (!getLine(input, input.empty() ? "nix-repl> " : " ")) { + if (!interacter->getLine(input, input.empty() ? ReplPromptType::ReplPrompt : ReplPromptType::ContinuationPrompt)) { // Ctrl-D should exit the debugger. state->debugStop = false; logger->cout(""); @@ -325,7 +217,7 @@ ReplExitStatus NixRepl::mainLoop() case ProcessLineResult::PromptAgain: break; default: - abort(); + unreachable(); } } catch (ParseError & e) { if (e.msg().find("unexpected end of file") != std::string::npos) { @@ -336,13 +228,7 @@ ReplExitStatus NixRepl::mainLoop() printMsg(lvlError, e.msg()); } } catch (EvalError & e) { - // in debugger mode, an EvalError should trigger another repl session. - // when that session returns the exception will land here. No need to show it again; - // show the error for this repl session instead. - if (state->debugRepl && !state->debugTraces.empty()) - showDebugTrace(std::cout, state->positions, state->debugTraces.front()); - else - printMsg(lvlError, e.msg()); + printMsg(lvlError, e.msg()); } catch (Error & e) { printMsg(lvlError, e.msg()); } catch (Interrupted & e) { @@ -356,51 +242,6 @@ ReplExitStatus NixRepl::mainLoop() } } - -bool NixRepl::getLine(std::string & input, const std::string & prompt) -{ - struct sigaction act, old; - sigset_t savedSignalMask, set; - - auto setupSignals = [&]() { - act.sa_handler = sigintHandler; - sigfillset(&act.sa_mask); - act.sa_flags = 0; - if (sigaction(SIGINT, &act, &old)) - throw SysError("installing handler for SIGINT"); - - sigemptyset(&set); - sigaddset(&set, SIGINT); - if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask)) - throw SysError("unblocking SIGINT"); - }; - auto restoreSignals = [&]() { - if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) - throw SysError("restoring signals"); - - if (sigaction(SIGINT, &old, 0)) - throw SysError("restoring handler for SIGINT"); - }; - - setupSignals(); - char * s = readline(prompt.c_str()); - Finally doFree([&]() { free(s); }); - restoreSignals(); - - if (g_signal_received) { - g_signal_received = 0; - input.clear(); - return true; - } - - if (!s) - return false; - input += s; - input += '\n'; - return true; -} - - StringSet NixRepl::completePrefix(const std::string & prefix) { StringSet completions; @@ -421,11 +262,14 @@ StringSet NixRepl::completePrefix(const std::string & prefix) try { auto dir = std::string(cur, 0, slash); auto prefix2 = std::string(cur, slash + 1); - for (auto & entry : readDirectory(dir == "" ? "/" : dir)) { - if (entry.name[0] != '.' && hasPrefix(entry.name, prefix2)) - completions.insert(prev + dir + "/" + entry.name); + for (auto & entry : std::filesystem::directory_iterator{dir == "" ? "/" : dir}) { + checkInterrupt(); + auto name = entry.path().filename().string(); + if (name[0] != '.' && hasPrefix(name, prefix2)) + completions.insert(prev + entry.path().string()); } } catch (Error &) { + } catch (std::filesystem::filesystem_error &) { } } else if ((dot = cur.rfind('.')) == std::string::npos) { /* This is a variable name; look it up in the current scope. */ @@ -452,7 +296,7 @@ StringSet NixRepl::completePrefix(const std::string & prefix) e->eval(*state, *env, v); state->forceAttrs(v, noPos, "while evaluating an attrset for the purpose of completion (this error should not be displayed; file an issue?)"); - for (auto & i : *v.attrs) { + for (auto & i : *v.attrs()) { std::string_view name = state->symbols[i.name]; if (name.substr(0, cur2.size()) != cur2) continue; completions.insert(concatStrings(prev, expr, ".", name)); @@ -464,6 +308,8 @@ StringSet NixRepl::completePrefix(const std::string & prefix) // Quietly ignore evaluation errors. } catch (BadURL & e) { // Quietly ignore BadURL flake-related errors. + } catch (FileNotFound & e) { + // Quietly ignore non-existent file beeing `import`-ed. } } @@ -519,7 +365,7 @@ ProcessLineResult NixRepl::processLine(std::string line) if (line.empty()) return ProcessLineResult::PromptAgain; - _isInterrupted = false; + setInterrupted(false); std::string command, arg; @@ -548,6 +394,7 @@ ProcessLineResult NixRepl::processLine(std::string line) << " :l, :load Load Nix expression and add it to scope\n" << " :lf, :load-flake Load Nix flake and add it to scope\n" << " :p, :print Evaluate and print expression recursively\n" + << " Strings are printed directly, without escaping.\n" << " :q, :quit Exit nix-repl\n" << " :r, :reload Reload all files\n" << " :sh Build dependencies of derivation, then start\n" @@ -651,7 +498,7 @@ ProcessLineResult NixRepl::processLine(std::string line) auto path = state->coerceToPath(noPos, v, context, "while evaluating the filename to edit"); return {path, 0}; } else if (v.isLambda()) { - auto pos = state->positions[v.lambda.fun->pos]; + auto pos = state->positions[v.payload.lambda.fun->pos]; if (auto path = std::get_if(&pos.origin)) return {*path, pos.line}; else @@ -669,7 +516,7 @@ ProcessLineResult NixRepl::processLine(std::string line) // runProgram redirects stdout to a StringSink, // using runProgram2 to allow editors to display their UI - runProgram2(RunOptions { .program = editor, .searchPath = true, .args = args }); + runProgram2(RunOptions { .program = editor, .lookupPath = true, .args = args , .isInteractive = true }); // Reload right after exiting the editor state->resetFileCache(); @@ -755,7 +602,11 @@ ProcessLineResult NixRepl::processLine(std::string line) else if (command == ":p" || command == ":print") { Value v; evalString(arg, v); - printValue(std::cout, v); + if (v.type() == nString) { + std::cout << v.string_view(); + } else { + printValue(std::cout, v); + } std::cout << std::endl; } @@ -766,6 +617,35 @@ ProcessLineResult NixRepl::processLine(std::string line) else if (command == ":doc") { Value v; + + auto expr = parseString(arg); + std::string fallbackName; + PosIdx fallbackPos; + DocComment fallbackDoc; + if (auto select = dynamic_cast(expr)) { + Value vAttrs; + auto name = select->evalExceptFinalSelect(*state, *env, vAttrs); + fallbackName = state->symbols[name]; + + state->forceAttrs(vAttrs, noPos, "while evaluating an attribute set to look for documentation"); + auto attrs = vAttrs.attrs(); + assert(attrs); + auto attr = attrs->get(name); + if (!attr) { + // When missing, trigger the normal exception + // e.g. :doc builtins.foo + // behaves like + // nix-repl> builtins.foo + // error: attribute 'foo' missing + evalString(arg, v); + assert(false); + } + if (attr->pos) { + fallbackPos = attr->pos; + fallbackDoc = state->getDocCommentForPos(fallbackPos); + } + } + evalString(arg, v); if (auto doc = state->getDoc(v)) { std::string markdown; @@ -783,6 +663,19 @@ ProcessLineResult NixRepl::processLine(std::string line) markdown += stripIndentation(doc->doc); logger->cout(trim(renderMarkdownToTerminal(markdown))); + } else if (fallbackPos) { + std::stringstream ss; + ss << "Attribute `" << fallbackName << "`\n\n"; + ss << " … defined at " << state->positions[fallbackPos] << "\n\n"; + if (fallbackDoc) { + ss << fallbackDoc.getInnerText(state->positions); + } else { + ss << "No documentation found.\n\n"; + } + + auto markdown = ss.str(); + logger->cout(trim(renderMarkdownToTerminal(markdown))); + } else throw Error("value does not have documentation"); } @@ -840,14 +733,14 @@ void NixRepl::loadFlake(const std::string & flakeRefS) if (flakeRefS.empty()) throw Error("cannot use ':load-flake' without a path specified. (Use '.' for the current working directory.)"); - auto flakeRef = parseFlakeRef(flakeRefS, absPath("."), true); + auto flakeRef = parseFlakeRef(fetchSettings, flakeRefS, absPath("."), true); if (evalSettings.pureEval && !flakeRef.input.isLocked()) throw Error("cannot use ':load-flake' on locked flake reference '%s' (use --impure to override)", flakeRefS); Value v; flake::callFlake(*state, - flake::lockFlake(*state, flakeRef, + flake::lockFlake(flakeSettings, *state, flakeRef, flake::LockFlags { .updateLockFile = false, .useRegistries = !evalSettings.pureEval, @@ -899,17 +792,17 @@ void NixRepl::loadFiles() void NixRepl::addAttrsToScope(Value & attrs) { state->forceAttrs(attrs, [&]() { return attrs.determinePos(noPos); }, "while evaluating an attribute set to be merged in the global scope"); - if (displ + attrs.attrs->size() >= envSize) + if (displ + attrs.attrs()->size() >= envSize) throw Error("environment full; cannot add more variables"); - for (auto & i : *attrs.attrs) { + for (auto & i : *attrs.attrs()) { staticEnv->vars.emplace_back(i.name, displ); env->values[displ++] = i.value; varNames.emplace(state->symbols[i.name]); } staticEnv->sort(); staticEnv->deduplicate(); - notice("Added %1% variables.", attrs.attrs->size()); + notice("Added %1% variables.", attrs.attrs()->size()); } @@ -941,11 +834,11 @@ void NixRepl::evalString(std::string s, Value & v) std::unique_ptr AbstractNixRepl::create( - const SearchPath & searchPath, nix::ref store, ref state, + const LookupPath & lookupPath, nix::ref store, ref state, std::function getValues) { return std::make_unique( - searchPath, + lookupPath, openStore(), state, getValues @@ -961,9 +854,9 @@ ReplExitStatus AbstractNixRepl::runSimple( NixRepl::AnnotatedValues values; return values; }; - SearchPath searchPath = {}; + LookupPath lookupPath = {}; auto repl = std::make_unique( - searchPath, + lookupPath, openStore(), evalState, getValues diff --git a/src/libcmd/repl.hh b/src/libcmd/repl.hh index 21aa8bfc7..3fd4b2c39 100644 --- a/src/libcmd/repl.hh +++ b/src/libcmd/repl.hh @@ -3,11 +3,6 @@ #include "eval.hh" -#if HAVE_BOEHMGC -#define GC_INCLUDE_NEW -#include -#endif - namespace nix { struct AbstractNixRepl @@ -25,7 +20,7 @@ struct AbstractNixRepl typedef std::vector> AnnotatedValues; static std::unique_ptr create( - const SearchPath & searchPath, nix::ref store, ref state, + const LookupPath & lookupPath, nix::ref store, ref state, std::function getValues); static ReplExitStatus runSimple( diff --git a/src/libexpr-c/.version b/src/libexpr-c/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libexpr-c/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libexpr-c/build-utils-meson b/src/libexpr-c/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libexpr-c/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libexpr-c/local.mk b/src/libexpr-c/local.mk new file mode 100644 index 000000000..227a4095b --- /dev/null +++ b/src/libexpr-c/local.mk @@ -0,0 +1,25 @@ +libraries += libexprc + +libexprc_NAME = libnixexprc + +libexprc_DIR := $(d) + +libexprc_SOURCES := \ + $(wildcard $(d)/*.cc) \ + +# Not just for this library itself, but also for downstream libraries using this library + +INCLUDE_libexprc := -I $(d) +libexprc_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libutilc) \ + $(INCLUDE_libfetchers) \ + $(INCLUDE_libstore) $(INCLUDE_libstorec) \ + $(INCLUDE_libexpr) $(INCLUDE_libexprc) + +libexprc_LIBS = libutil libutilc libstore libstorec libfetchers libexpr + +libexprc_LDFLAGS += $(THREAD_LDFLAGS) + +$(eval $(call install-file-in, $(d)/nix-expr-c.pc, $(libdir)/pkgconfig, 0644)) + +libexprc_FORCE_INSTALL := 1 + diff --git a/src/libexpr-c/meson.build b/src/libexpr-c/meson.build new file mode 100644 index 000000000..6db5b83b8 --- /dev/null +++ b/src/libexpr-c/meson.build @@ -0,0 +1,93 @@ +project('nix-expr-c', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), + dependency('nix-expr'), +] +deps_public_maybe_subproject = [ + dependency('nix-util-c'), + dependency('nix-store-c'), +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +# TODO rename, because it will conflict with downstream projects +configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) + +config_h = configure_file( + configuration : configdata, + output : 'config-expr.h', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + + # From C++ libraries, only for internals + '-include', 'config-util.hh', + '-include', 'config-store.hh', + '-include', 'config-expr.hh', + + # From C libraries, for our public, installed headers too + '-include', 'config-util.h', + '-include', 'config-store.h', + '-include', 'config-expr.h', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'nix_api_expr.cc', + 'nix_api_external.cc', + 'nix_api_value.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'nix_api_expr.h', + 'nix_api_external.h', + 'nix_api_value.h', +) + +# TODO move this header to libexpr, maybe don't use it in tests? +headers += files('nix_api_expr_internal.h') + +subdir('build-utils-meson/export-all-symbols') + +this_library = library( + 'nixexprc', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + link_args: linker_export_flags, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libexpr-c/nix-expr-c.pc.in b/src/libexpr-c/nix-expr-c.pc.in new file mode 100644 index 000000000..06897064d --- /dev/null +++ b/src/libexpr-c/nix-expr-c.pc.in @@ -0,0 +1,10 @@ +prefix=@prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: Nix +Description: Nix Language Evaluator - C API +Version: @PACKAGE_VERSION@ +Requires: nix-store-c +Libs: -L${libdir} -lnixexprc +Cflags: -I${includedir}/nix diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc new file mode 100644 index 000000000..8f21d7022 --- /dev/null +++ b/src/libexpr-c/nix_api_expr.cc @@ -0,0 +1,213 @@ +#include +#include +#include + +#include "eval.hh" +#include "eval-gc.hh" +#include "globals.hh" +#include "eval-settings.hh" + +#include "nix_api_expr.h" +#include "nix_api_expr_internal.h" +#include "nix_api_store.h" +#include "nix_api_store_internal.h" +#include "nix_api_util.h" +#include "nix_api_util_internal.h" + +#if HAVE_BOEHMGC +# include +# define GC_INCLUDE_NEW 1 +# include "gc_cpp.h" +#endif + +nix_err nix_libexpr_init(nix_c_context * context) +{ + if (context) + context->last_err_code = NIX_OK; + { + auto ret = nix_libutil_init(context); + if (ret != NIX_OK) + return ret; + } + { + auto ret = nix_libstore_init(context); + if (ret != NIX_OK) + return ret; + } + try { + nix::initGC(); + } + NIXC_CATCH_ERRS +} + +nix_err nix_expr_eval_from_string( + nix_c_context * context, EvalState * state, const char * expr, const char * path, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + nix::Expr * parsedExpr = state->state.parseExprFromString(expr, state->state.rootPath(nix::CanonPath(path))); + state->state.eval(parsedExpr, value->value); + state->state.forceValue(value->value, nix::noPos); + } + NIXC_CATCH_ERRS +} + +nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, nix_value * arg, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + state->state.callFunction(fn->value, arg->value, value->value, nix::noPos); + state->state.forceValue(value->value, nix::noPos); + } + NIXC_CATCH_ERRS +} + +nix_err nix_value_call_multi(nix_c_context * context, EvalState * state, nix_value * fn, size_t nargs, nix_value ** args, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + state->state.callFunction(fn->value, nargs, (nix::Value * *)args, value->value, nix::noPos); + state->state.forceValue(value->value, nix::noPos); + } + NIXC_CATCH_ERRS +} + +nix_err nix_value_force(nix_c_context * context, EvalState * state, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + state->state.forceValue(value->value, nix::noPos); + } + NIXC_CATCH_ERRS +} + +nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + state->state.forceValueDeep(value->value); + } + NIXC_CATCH_ERRS +} + +EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c, Store * store) +{ + if (context) + context->last_err_code = NIX_OK; + try { + nix::Strings lookupPath; + if (lookupPath_c != nullptr) + for (size_t i = 0; lookupPath_c[i] != nullptr; i++) + lookupPath.push_back(lookupPath_c[i]); + + void * p = ::operator new( + sizeof(EvalState), + static_cast(alignof(EvalState))); + auto * p2 = static_cast(p); + new (p) EvalState { + .fetchSettings = nix::fetchers::Settings{}, + .settings = nix::EvalSettings{ + nix::settings.readOnlyMode, + }, + .state = nix::EvalState( + nix::LookupPath::parse(lookupPath), + store->ptr, + p2->fetchSettings, + p2->settings), + }; + loadConfFile(p2->settings); + return p2; + } + NIXC_CATCH_ERRS_NULL +} + +void nix_state_free(EvalState * state) +{ + delete state; +} + +#if HAVE_BOEHMGC +std::unordered_map< + const void *, + unsigned int, + std::hash, + std::equal_to, + traceable_allocator
\|\|