diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 60742cf6a..4488c7b7d 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -23,6 +23,7 @@ Maintainers: tick if completed or explain if not relevant - unit tests - `src/*/tests` - integration tests - `tests/nixos/*` - [ ] documentation in the manual + - [ ] documentation in the internal API docs - [ ] code and comments are self-explanatory - [ ] commit message explains why the change was made - [ ] new feature or incompatible change: updated release notes diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 158eb057c..c06c77043 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,7 +58,7 @@ jobs: with: fetch-depth: 0 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v19 + - uses: cachix/install-nix-action@v20 with: install_url: https://releases.nixos.org/nix/nix-2.13.3/install - uses: cachix/cachix-action@v12 @@ -79,7 +79,7 @@ jobs: steps: - uses: actions/checkout@v3 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v19 + - uses: cachix/install-nix-action@v20 with: install_url: '${{needs.installer.outputs.installerURL}}' install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" @@ -91,6 +91,8 @@ jobs: - run: exec sh -c "nix-instantiate -E 'builtins.currentTime' --eval" - run: exec zsh -c "nix-instantiate -E 'builtins.currentTime' --eval" - run: exec fish -c "nix-instantiate -E 'builtins.currentTime' --eval" + - run: exec bash -c "nix-channel --add https://releases.nixos.org/nixos/unstable/nixos-23.05pre466020.60c1d71f2ba nixpkgs" + - run: exec bash -c "nix-channel --update && nix-env -iA nixpkgs.hello && hello" docker_push_image: needs: [check_secrets, tests] @@ -104,7 +106,7 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v19 + - uses: cachix/install-nix-action@v20 with: install_url: https://releases.nixos.org/nix/nix-2.13.3/install - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV diff --git a/boehmgc-coroutine-sp-fallback.diff b/boehmgc-coroutine-sp-fallback.diff index 2826486fb..5066d8278 100644 --- a/boehmgc-coroutine-sp-fallback.diff +++ b/boehmgc-coroutine-sp-fallback.diff @@ -1,8 +1,8 @@ diff --git a/darwin_stop_world.c b/darwin_stop_world.c -index 3dbaa3fb..36a1d1f7 100644 +index 0468aaec..b348d869 100644 --- a/darwin_stop_world.c +++ b/darwin_stop_world.c -@@ -352,6 +352,7 @@ GC_INNER void GC_push_all_stacks(void) +@@ -356,6 +356,7 @@ GC_INNER void GC_push_all_stacks(void) int nthreads = 0; word total_size = 0; mach_msg_type_number_t listcount = (mach_msg_type_number_t)THREAD_TABLE_SZ; @@ -10,7 +10,7 @@ index 3dbaa3fb..36a1d1f7 100644 if (!EXPECT(GC_thr_initialized, TRUE)) GC_thr_init(); -@@ -407,6 +408,19 @@ GC_INNER void GC_push_all_stacks(void) +@@ -411,6 +412,19 @@ GC_INNER void GC_push_all_stacks(void) GC_push_all_stack_sections(lo, hi, p->traced_stack_sect); } if (altstack_lo) { @@ -30,6 +30,22 @@ index 3dbaa3fb..36a1d1f7 100644 total_size += altstack_hi - altstack_lo; GC_push_all_stack(altstack_lo, altstack_hi); } +diff --git a/include/gc.h b/include/gc.h +index edab6c22..f2c61282 100644 +--- a/include/gc.h ++++ b/include/gc.h +@@ -2172,6 +2172,11 @@ GC_API void GC_CALL GC_win32_free_heap(void); + (*GC_amiga_allocwrapper_do)(a,GC_malloc_atomic_ignore_off_page) + #endif /* _AMIGA && !GC_AMIGA_MAKINGLIB */ + ++#if !__APPLE__ ++/* Patch doesn't work on apple */ ++#define NIX_BOEHM_PATCH_VERSION 1 ++#endif ++ + #ifdef __cplusplus + } /* extern "C" */ + #endif diff --git a/pthread_stop_world.c b/pthread_stop_world.c index b5d71e62..aed7b0bf 100644 --- a/pthread_stop_world.c diff --git a/configure.ac b/configure.ac index f1f45f868..e587bd563 100644 --- a/configure.ac +++ b/configure.ac @@ -289,13 +289,24 @@ PKG_CHECK_MODULES([GTEST], [gtest_main]) # Look for rapidcheck. +AC_ARG_VAR([RAPIDCHECK_HEADERS], [include path of gtest headers shipped by RAPIDCHECK]) # No pkg-config yet, https://github.com/emil-e/rapidcheck/issues/302 AC_LANG_PUSH(C++) AC_SUBST(RAPIDCHECK_HEADERS) [CXXFLAGS="-I $RAPIDCHECK_HEADERS $CXXFLAGS"] +[LIBS="-lrapidcheck -lgtest $LIBS"] AC_CHECK_HEADERS([rapidcheck/gtest.h], [], [], [#include ]) -dnl No good for C++ libs with mangled symbols -dnl AC_CHECK_LIB([rapidcheck], []) +dnl AC_CHECK_LIB doesn't work for C++ libs with mangled symbols +AC_LINK_IFELSE([ + AC_LANG_PROGRAM([[ + #include + #include + ]], [[ + return RUN_ALL_TESTS(); + ]]) + ], + [], + [AC_MSG_ERROR([librapidcheck is not found.])]) AC_LANG_POP(C++) fi diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index bc19d0303..86f2ca567 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -1,4 +1,4 @@ -{ toplevel }: +cliDumpStr: with builtins; with import ./utils.nix; @@ -63,7 +63,10 @@ let * [`${command} ${name}`](./${appendName filename name}.md) - ${subcmd.description} ''; - maybeDocumentation = if details ? doc then details.doc else ""; + maybeDocumentation = + if details ? doc + then replaceStrings ["@stores@"] [storeDocs] details.doc + else ""; maybeOptions = if details.flags == {} then "" else '' # Options @@ -110,13 +113,13 @@ let }; in [ cmd ] ++ concatMap subcommand (attrNames details.commands or {}); - parsedToplevel = builtins.fromJSON toplevel; + cliDump = builtins.fromJSON cliDumpStr; manpages = processCommand { command = "nix"; - details = parsedToplevel; + details = cliDump.args; filename = "nix"; - toplevel = parsedToplevel; + toplevel = cliDump.args; }; tableOfContents = let @@ -124,4 +127,18 @@ let " - [${page.command}](command-ref/new-cli/${page.name})"; in concatStringsSep "\n" (map showEntry manpages) + "\n"; + storeDocs = + let + showStore = name: { settings, doc }: + '' + ## ${name} + + ${doc} + + **Settings**: + + ${showSettings { useAnchors = false; } settings} + ''; + in concatStrings (attrValues (mapAttrs showStore cliDump.stores)); + in (listToAttrs manpages) // { "SUMMARY.md" = tableOfContents; } diff --git a/doc/manual/generate-options.nix b/doc/manual/generate-options.nix deleted file mode 100644 index a4ec36477..000000000 --- a/doc/manual/generate-options.nix +++ /dev/null @@ -1,41 +0,0 @@ -let - inherit (builtins) attrNames concatStringsSep isAttrs isBool; - inherit (import ./utils.nix) concatStrings squash splitLines; -in - -optionsInfo: -let - showOption = name: - let - inherit (optionsInfo.${name}) description documentDefault defaultValue aliases; - result = squash '' - - [`${name}`](#conf-${name}) - - ${indent " " body} - ''; - # separate body to cleanly handle indentation - body = '' - ${description} - - **Default:** ${showDefault documentDefault defaultValue} - - ${showAliases aliases} - ''; - showDefault = documentDefault: defaultValue: - if documentDefault then - # a StringMap value type is specified as a string, but - # this shows the value type. The empty stringmap is `null` in - # JSON, but that converts to `{ }` here. - if defaultValue == "" || defaultValue == [] || isAttrs defaultValue - then "*empty*" - else if isBool defaultValue then - if defaultValue then "`true`" else "`false`" - else "`${toString defaultValue}`" - else "*machine-specific*"; - showAliases = aliases: - if aliases == [] then "" else - "**Deprecated alias:** ${(concatStringsSep ", " (map (s: "`${s}`") aliases))}"; - indent = prefix: s: - concatStringsSep "\n" (map (x: if x == "" then x else "${prefix}${x}") (splitLines s)); - in result; -in concatStrings (map showOption (attrNames optionsInfo)) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 531168e3f..df941d460 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -13,6 +13,13 @@ man-pages := $(foreach n, \ nix.conf.5 nix-daemon.8 \ , $(d)/$(n)) +# man pages for subcommands +# convert from `$(d)/src/command-ref/nix-{1}/{2}.md` to `$(d)/nix-{1}-{2}.1` +# FIXME: unify with how nix3-cli man pages are generated +man-pages += $(foreach subcommand, \ + $(filter-out %opt-common.md %env-common.md, $(wildcard $(d)/src/command-ref/nix-*/*.md)), \ + $(d)/$(subst /,-,$(subst $(d)/src/command-ref/,,$(subst .md,.1,$(subcommand))))) + clean-files += $(d)/*.1 $(d)/*.5 $(d)/*.8 # Provide a dummy environment for nix, so that it will not access files outside the macOS sandbox. @@ -29,13 +36,35 @@ nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command - # re-implement mdBook's include directive to make it usable for terminal output and for proper @docroot@ substitution define process-includes while read -r line; do \ - filename=$$(sed 's/{{#include \(.*\)}}/\1/'<<< $$line); \ - matchline=$$(sed 's|/|\\/|g' <<< $$line); \ - sed -i "/$$matchline/r $$(dirname $(2))/$$filename" $(2); \ + set -euo pipefail; \ + filename="$$(dirname $(1))/$$(sed 's/{{#include \(.*\)}}/\1/'<<< $$line)"; \ + test -f "$$filename" || ( echo "#include-d file '$$filename' does not exist." >&2; exit 1; ); \ + matchline="$$(sed 's|/|\\/|g' <<< $$line)"; \ + sed -i "/$$matchline/r $$filename" $(2); \ sed -i "s/$$matchline//" $(2); \ done < <(grep '{{#include' $(1)) endef +$(d)/nix-env-%.1: $(d)/src/command-ref/nix-env/%.md + @printf "Title: %s\n\n" "$(subst nix-env-,nix-env --,$$(basename "$@" .1))" > $^.tmp + $(render-subcommand) + +$(d)/nix-store-%.1: $(d)/src/command-ref/nix-store/%.md + @printf -- 'Title: %s\n\n' "$(subst nix-store-,nix-store --,$$(basename "$@" .1))" > $^.tmp + $(render-subcommand) + +# FIXME: there surely is some more deduplication to be achieved here with even darker Make magic +define render-subcommand + @cat $^ >> $^.tmp + @$(call process-includes,$^,$^.tmp) + $(trace-gen) lowdown -sT man --nroff-nolinks -M section=1 $^.tmp -o $@ + @# fix up `lowdown`'s automatic escaping of `--` + @# https://github.com/kristapsdz/lowdown/blob/edca6ce6d5336efb147321a43c47a698de41bb7c/entity.c#L202 + @sed -i 's/\e\[u2013\]/--/' $@ + @rm $^.tmp +endef + + $(d)/%.1: $(d)/src/command-ref/%.md @printf "Title: %s\n\n" "$$(basename $@ .1)" > $^.tmp @cat $^ >> $^.tmp @@ -60,17 +89,17 @@ $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli @$(call process-includes,$@,$@) $(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix - @rm -rf $@ - $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix { toplevel = builtins.readFile $<; }' + @rm -rf $@ $@.tmp + $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix (builtins.readFile $<)' @mv $@.tmp $@ -$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix +$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix @cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp - $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-options.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; + $(trace-gen) $(nix-eval) --expr '(import doc/manual/utils.nix).showSettings { useAnchors = true; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; @mv $@.tmp $@ $(d)/nix.json: $(bindir)/nix - $(trace-gen) $(dummy-env) $(bindir)/nix __dump-args > $@.tmp + $(trace-gen) $(dummy-env) $(bindir)/nix __dump-cli > $@.tmp @mv $@.tmp $@ $(d)/conf-file.json: $(bindir)/nix @@ -122,6 +151,8 @@ $(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/ cp -r doc/manual "$$tmp"; \ find "$$tmp" -name '*.md' | while read -r file; do \ $(call process-includes,$$file,$$file); \ + done; \ + find "$$tmp" -name '*.md' | while read -r file; do \ docroot="$$(realpath --relative-to="$$(dirname "$$file")" $$tmp/manual/src)"; \ sed -i "s,@docroot@,$$docroot,g" "$$file"; \ done; \ diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index f3e2c9ae6..5bf274550 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -47,7 +47,38 @@ - [nix-build](command-ref/nix-build.md) - [nix-shell](command-ref/nix-shell.md) - [nix-store](command-ref/nix-store.md) + - [nix-store --add-fixed](command-ref/nix-store/add-fixed.md) + - [nix-store --add](command-ref/nix-store/add.md) + - [nix-store --delete](command-ref/nix-store/delete.md) + - [nix-store --dump-db](command-ref/nix-store/dump-db.md) + - [nix-store --dump](command-ref/nix-store/dump.md) + - [nix-store --export](command-ref/nix-store/export.md) + - [nix-store --gc](command-ref/nix-store/gc.md) + - [nix-store --generate-binary-cache-key](command-ref/nix-store/generate-binary-cache-key.md) + - [nix-store --import](command-ref/nix-store/import.md) + - [nix-store --load-db](command-ref/nix-store/load-db.md) + - [nix-store --optimise](command-ref/nix-store/optimise.md) + - [nix-store --print-env](command-ref/nix-store/print-env.md) + - [nix-store --query](command-ref/nix-store/query.md) + - [nix-store --read-log](command-ref/nix-store/read-log.md) + - [nix-store --realise](command-ref/nix-store/realise.md) + - [nix-store --repair-path](command-ref/nix-store/repair-path.md) + - [nix-store --restore](command-ref/nix-store/restore.md) + - [nix-store --serve](command-ref/nix-store/serve.md) + - [nix-store --verify-path](command-ref/nix-store/verify-path.md) + - [nix-store --verify](command-ref/nix-store/verify.md) - [nix-env](command-ref/nix-env.md) + - [nix-env --delete-generations](command-ref/nix-env/delete-generations.md) + - [nix-env --install](command-ref/nix-env/install.md) + - [nix-env --list-generations](command-ref/nix-env/list-generations.md) + - [nix-env --query](command-ref/nix-env/query.md) + - [nix-env --rollback](command-ref/nix-env/rollback.md) + - [nix-env --set-flag](command-ref/nix-env/set-flag.md) + - [nix-env --set](command-ref/nix-env/set.md) + - [nix-env --switch-generation](command-ref/nix-env/switch-generation.md) + - [nix-env --switch-profile](command-ref/nix-env/switch-profile.md) + - [nix-env --uninstall](command-ref/nix-env/uninstall.md) + - [nix-env --upgrade](command-ref/nix-env/upgrade.md) - [Utilities](command-ref/utilities.md) - [nix-channel](command-ref/nix-channel.md) - [nix-collect-garbage](command-ref/nix-collect-garbage.md) @@ -64,6 +95,7 @@ - [Glossary](glossary.md) - [Contributing](contributing/contributing.md) - [Hacking](contributing/hacking.md) + - [Experimental Features](contributing/experimental-features.md) - [CLI guideline](contributing/cli-guideline.md) - [Release Notes](release-notes/release-notes.md) - [Release X.Y (202?-??-??)](release-notes/rl-next.md) diff --git a/doc/manual/src/command-ref/env-common.md b/doc/manual/src/command-ref/env-common.md index c5d38db47..bf00be84f 100644 --- a/doc/manual/src/command-ref/env-common.md +++ b/doc/manual/src/command-ref/env-common.md @@ -2,18 +2,29 @@ Most Nix commands interpret the following environment variables: - - [`IN_NIX_SHELL`]{#env-IN_NIX_SHELL}\ + - [`IN_NIX_SHELL`](#env-IN_NIX_SHELL)\ Indicator that tells if the current environment was set up by `nix-shell`. It can have the values `pure` or `impure`. - - [`NIX_PATH`]{#env-NIX_PATH}\ + - [`NIX_PATH`](#env-NIX_PATH)\ A colon-separated list of directories used to look up the location of Nix - expressions using [paths](../language/values.md#type-path) + expressions using [paths](@docroot@/language/values.md#type-path) enclosed in angle brackets (i.e., ``), e.g. `/home/eelco/Dev:/etc/nixos`. It can be extended using the - [`-I` option](./opt-common.md#opt-I). + [`-I` option](@docroot@/command-ref/opt-common.md#opt-I). - - [`NIX_IGNORE_SYMLINK_STORE`]{#env-NIX_IGNORE_SYMLINK_STORE}\ + If `NIX_PATH` is not set at all, Nix will fall back to the following list in [impure](@docroot@/command-ref/conf-file.md#conf-pure-eval) and [unrestricted](@docroot@/command-ref/conf-file.md#conf-restrict-eval) evaluation mode: + + 1. `$HOME/.nix-defexpr/channels` + 2. `nixpkgs=/nix/var/nix/profiles/per-user/root/channels/nixpkgs` + 3. `/nix/var/nix/profiles/per-user/root/channels` + + If `NIX_PATH` is set to an empty string, resolving search paths will always fail. + For example, attempting to use `` will produce: + + error: file 'nixpkgs' was not found in the Nix search path + + - [`NIX_IGNORE_SYMLINK_STORE`](#env-NIX_IGNORE_SYMLINK_STORE)\ Normally, the Nix store directory (typically `/nix/store`) is not allowed to contain any symlink components. This is to prevent “impure” builds. Builders sometimes “canonicalise” paths by @@ -35,58 +46,58 @@ Most Nix commands interpret the following environment variables: Consult the mount 8 manual page for details. - - [`NIX_STORE_DIR`]{#env-NIX_STORE_DIR}\ + - [`NIX_STORE_DIR`](#env-NIX_STORE_DIR)\ Overrides the location of the Nix store (default `prefix/store`). - - [`NIX_DATA_DIR`]{#env-NIX_DATA_DIR}\ + - [`NIX_DATA_DIR`](#env-NIX_DATA_DIR)\ Overrides the location of the Nix static data directory (default `prefix/share`). - - [`NIX_LOG_DIR`]{#env-NIX_LOG_DIR}\ + - [`NIX_LOG_DIR`](#env-NIX_LOG_DIR)\ Overrides the location of the Nix log directory (default `prefix/var/log/nix`). - - [`NIX_STATE_DIR`]{#env-NIX_STATE_DIR}\ + - [`NIX_STATE_DIR`](#env-NIX_STATE_DIR)\ Overrides the location of the Nix state directory (default `prefix/var/nix`). - - [`NIX_CONF_DIR`]{#env-NIX_CONF_DIR}\ + - [`NIX_CONF_DIR`](#env-NIX_CONF_DIR)\ Overrides the location of the system Nix configuration directory (default `prefix/etc/nix`). - - [`NIX_CONFIG`]{#env-NIX_CONFIG}\ + - [`NIX_CONFIG`](#env-NIX_CONFIG)\ Applies settings from Nix configuration from the environment. The content is treated as if it was read from a Nix configuration file. Settings are separated by the newline character. - - [`NIX_USER_CONF_FILES`]{#env-NIX_USER_CONF_FILES}\ + - [`NIX_USER_CONF_FILES`](#env-NIX_USER_CONF_FILES)\ Overrides the location of the user Nix configuration files to load from (defaults to the XDG spec locations). The variable is treated as a list separated by the `:` token. - - [`TMPDIR`]{#env-TMPDIR}\ + - [`TMPDIR`](#env-TMPDIR)\ Use the specified directory to store temporary files. In particular, this includes temporary build directories; these can take up substantial amounts of disk space. The default is `/tmp`. - - [`NIX_REMOTE`]{#env-NIX_REMOTE}\ + - [`NIX_REMOTE`](#env-NIX_REMOTE)\ This variable should be set to `daemon` if you want to use the Nix daemon to execute Nix operations. This is necessary in [multi-user - Nix installations](../installation/multi-user.md). If the Nix + Nix installations](@docroot@/installation/multi-user.md). If the Nix daemon's Unix socket is at some non-standard path, this variable should be set to `unix://path/to/socket`. Otherwise, it should be left unset. - - [`NIX_SHOW_STATS`]{#env-NIX_SHOW_STATS}\ + - [`NIX_SHOW_STATS`](#env-NIX_SHOW_STATS)\ If set to `1`, Nix will print some evaluation statistics, such as the number of values allocated. - - [`NIX_COUNT_CALLS`]{#env-NIX_COUNT_CALLS}\ + - [`NIX_COUNT_CALLS`](#env-NIX_COUNT_CALLS)\ If set to `1`, Nix will print how often functions were called during Nix expression evaluation. This is useful for profiling your Nix expressions. - - [`GC_INITIAL_HEAP_SIZE`]{#env-GC_INITIAL_HEAP_SIZE}\ + - [`GC_INITIAL_HEAP_SIZE`](#env-GC_INITIAL_HEAP_SIZE)\ If Nix has been configured to use the Boehm garbage collector, this variable sets the initial size of the heap in bytes. It defaults to 384 MiB. Setting it to a low value reduces memory consumption, but @@ -103,4 +114,4 @@ New Nix commands conform to the [XDG Base Directory Specification], and use the Classic Nix commands can also be made to follow this standard using the [`use-xdg-base-directories`] configuration option. [XDG Base Directory Specification]: https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html -[`use-xdg-base-directories`]: ../command-ref/conf-file.md#conf-use-xdg-base-directories \ No newline at end of file +[`use-xdg-base-directories`]: @docroot@/command-ref/conf-file.md#conf-use-xdg-base-directories \ No newline at end of file diff --git a/doc/manual/src/command-ref/nix-build.md b/doc/manual/src/command-ref/nix-build.md index 937b046b8..44de4cf53 100644 --- a/doc/manual/src/command-ref/nix-build.md +++ b/doc/manual/src/command-ref/nix-build.md @@ -38,7 +38,7 @@ directory containing at least a file named `default.nix`. `nix-build` is essentially a wrapper around [`nix-instantiate`](nix-instantiate.md) (to translate a high-level Nix expression to a low-level [store derivation]) and [`nix-store ---realise`](nix-store.md#operation---realise) (to build the store +--realise`](@docroot@/command-ref/nix-store/realise.md) (to build the store derivation). [store derivation]: ../glossary.md#gloss-store-derivation @@ -51,9 +51,8 @@ derivation). # Options -All options not listed here are passed to `nix-store ---realise`, except for `--arg` and `--attr` / `-A` which are passed to -`nix-instantiate`. +All options not listed here are passed to `nix-store --realise`, +except for `--arg` and `--attr` / `-A` which are passed to `nix-instantiate`. - [`--no-out-link`](#opt-no-out-link) @@ -70,7 +69,9 @@ All options not listed here are passed to `nix-store Change the name of the symlink to the output path created from `result` to *outlink*. -The following common options are supported: +{{#include ./opt-common.md}} + +{{#include ./env-common.md}} # Examples diff --git a/doc/manual/src/command-ref/nix-channel.md b/doc/manual/src/command-ref/nix-channel.md index 24353525f..2027cc98d 100644 --- a/doc/manual/src/command-ref/nix-channel.md +++ b/doc/manual/src/command-ref/nix-channel.md @@ -45,6 +45,10 @@ Note that `--add` does not automatically perform an update. The list of subscribed channels is stored in `~/.nix-channels`. +{{#include ./opt-common.md}} + +{{#include ./env-common.md}} + # Examples To subscribe to the Nixpkgs channel and install the GNU Hello package: @@ -70,7 +74,7 @@ $ nix-instantiate --eval -E '(import {}).lib.version' # Files - - `/nix/var/nix/profiles/per-user/username/channels`\ + - `${XDG_STATE_HOME-$HOME/.local/state}/nix/profiles/channels`\ `nix-channel` uses a `nix-env` profile to keep track of previous versions of the subscribed channels. Every time you run `nix-channel --update`, a new channel generation (that is, a symlink to the @@ -79,7 +83,7 @@ $ nix-instantiate --eval -E '(import {}).lib.version' - `~/.nix-defexpr/channels`\ This is a symlink to - `/nix/var/nix/profiles/per-user/username/channels`. It ensures that + `${XDG_STATE_HOME-$HOME/.local/state}/nix/profiles/channels`. It ensures that `nix-env` can find your channels. In a multi-user installation, you may also have `~/.nix-defexpr/channels_root`, which links to the channels of the root user. diff --git a/doc/manual/src/command-ref/nix-collect-garbage.md b/doc/manual/src/command-ref/nix-collect-garbage.md index 296165993..51db5fc67 100644 --- a/doc/manual/src/command-ref/nix-collect-garbage.md +++ b/doc/manual/src/command-ref/nix-collect-garbage.md @@ -9,7 +9,7 @@ # Description The command `nix-collect-garbage` is mostly an alias of [`nix-store ---gc`](nix-store.md#operation---gc), that is, it deletes all +--gc`](@docroot@/command-ref/nix-store/gc.md), that is, it deletes all unreachable paths in the Nix store to clean up your system. However, it provides two additional options: `-d` (`--delete-old`), which deletes all old generations of all profiles in `/nix/var/nix/profiles` @@ -20,6 +20,10 @@ and `--delete-older-than` *period*, where period is a value such as of days in all profiles in `/nix/var/nix/profiles` (except for the generations that were active at that point in time). +{{#include ./opt-common.md}} + +{{#include ./env-common.md}} + # Example To delete from the Nix store everything that is not used by the current diff --git a/doc/manual/src/command-ref/nix-copy-closure.md b/doc/manual/src/command-ref/nix-copy-closure.md index cd8e351bb..0801e8246 100644 --- a/doc/manual/src/command-ref/nix-copy-closure.md +++ b/doc/manual/src/command-ref/nix-copy-closure.md @@ -63,12 +63,16 @@ authentication, you can avoid typing the passphrase with `ssh-agent`. - `-v`\ Show verbose output. +{{#include ./opt-common.md}} + # Environment variables - `NIX_SSHOPTS`\ Additional options to be passed to `ssh` on the command line. +{{#include ./env-common.md}} + # Examples Copy Firefox with all its dependencies to a remote machine: diff --git a/doc/manual/src/command-ref/nix-env.md b/doc/manual/src/command-ref/nix-env.md index f4fa5b50c..42b5bca77 100644 --- a/doc/manual/src/command-ref/nix-env.md +++ b/doc/manual/src/command-ref/nix-env.md @@ -4,15 +4,14 @@ # Synopsis -`nix-env` +`nix-env` *operation* [*options*] [*arguments…*] [`--option` *name* *value*] [`--arg` *name* *value*] [`--argstr` *name* *value*] [{`--file` | `-f`} *path*] - [{`--profile` | `-p`} *path(] + [{`--profile` | `-p`} *path*] [`--system-filter` *system*] [`--dry-run`] - *operation* [*options…*] [*arguments…*] # Description @@ -24,7 +23,29 @@ environments: different users can have different environments, and individual users can switch between different environments. `nix-env` takes exactly one *operation* flag which indicates the -subcommand to be performed. These are documented below. +subcommand to be performed. The following operations are available: + +- [`--install`](./nix-env/install.md) +- [`--upgrade`](./nix-env/upgrade.md) +- [`--uninstall`](./nix-env/uninstall.md) +- [`--set`](./nix-env/set.md) +- [`--set-flag`](./nix-env/set-flag.md) +- [`--query`](./nix-env/query.md) +- [`--switch-profile`](./nix-env/switch-profile.md) +- [`--list-generations`](./nix-env/list-generations.md) +- [`--delete-generations`](./nix-env/delete-generations.md) +- [`--switch-generation`](./nix-env/switch-generation.md) +- [`--rollback`](./nix-env/rollback.md) + +These pages can be viewed offline: + +- `man nix-env-`. + + Example: `man nix-env-install` + +- `nix-env --help --` + + Example: `nix-env --help --install` # Selectors @@ -60,46 +81,6 @@ match. Here are some examples: Matches any package name containing the strings `firefox` or `chromium`. -# Common options - -This section lists the options that are common to all operations. These -options are allowed for every subcommand, though they may not always -have an effect. - - - `--file` / `-f` *path*\ - Specifies the Nix expression (designated below as the *active Nix - expression*) used by the `--install`, `--upgrade`, and `--query - --available` operations to obtain derivations. The default is - `~/.nix-defexpr`. - - If the argument starts with `http://` or `https://`, it is - interpreted as the URL of a tarball that will be downloaded and - unpacked to a temporary location. The tarball must include a single - top-level directory containing at least a file named `default.nix`. - - - `--profile` / `-p` *path*\ - Specifies the profile to be used by those operations that operate on - a profile (designated below as the *active profile*). A profile is a - sequence of user environments called *generations*, one of which is - the *current generation*. - - - `--dry-run`\ - For the `--install`, `--upgrade`, `--uninstall`, - `--switch-generation`, `--delete-generations` and `--rollback` - operations, this flag will cause `nix-env` to print what *would* be - done if this flag had not been specified, without actually doing it. - - `--dry-run` also prints out which paths will be - [substituted](../glossary.md) (i.e., downloaded) and which paths - will be built from source (because no substitute is available). - - - `--system-filter` *system*\ - By default, operations such as `--query - --available` show derivations matching any platform. This option - allows you to use derivations for the specified platform *system*. - - - # Files - `~/.nix-defexpr`\ @@ -145,750 +126,3 @@ have an effect. symlink points to `prefix/var/nix/profiles/default`. The `PATH` environment variable should include `~/.nix-profile/bin` for the user environment to be visible to the user. - -# Operation `--install` - -## Synopsis - -`nix-env` {`--install` | `-i`} *args…* - [{`--prebuilt-only` | `-b`}] - [{`--attr` | `-A`}] - [`--from-expression`] [`-E`] - [`--from-profile` *path*] - [`--preserve-installed` | `-P`] - [`--remove-all` | `-r`] - -## Description - -The install operation creates a new user environment, based on the -current generation of the active profile, to which a set of store paths -described by *args* is added. The arguments *args* map to store paths in -a number of possible ways: - - - By default, *args* is a set of derivation names denoting derivations - in the active Nix expression. These are realised, and the resulting - output paths are installed. Currently installed derivations with a - name equal to the name of a derivation being added are removed - unless the option `--preserve-installed` is specified. - - If there are multiple derivations matching a name in *args* that - have the same name (e.g., `gcc-3.3.6` and `gcc-4.1.1`), then the - derivation with the highest *priority* is used. A derivation can - define a priority by declaring the `meta.priority` attribute. This - attribute should be a number, with a higher value denoting a lower - priority. The default priority is `0`. - - If there are multiple matching derivations with the same priority, - then the derivation with the highest version will be installed. - - You can force the installation of multiple derivations with the same - name by being specific about the versions. For instance, `nix-env -i - gcc-3.3.6 gcc-4.1.1` will install both version of GCC (and will - probably cause a user environment conflict\!). - - - If `--attr` (`-A`) is specified, the arguments are *attribute - paths* that select attributes from the top-level Nix - expression. This is faster than using derivation names and - unambiguous. To find out the attribute paths of available - packages, use `nix-env -qaP`. - - - If `--from-profile` *path* is given, *args* is a set of names - denoting installed store paths in the profile *path*. This is an - easy way to copy user environment elements from one profile to - another. - - - If `--from-expression` is given, *args* are Nix - [functions](../language/constructs.md#functions) - that are called with the active Nix expression as their single - argument. The derivations returned by those function calls are - installed. This allows derivations to be specified in an - unambiguous way, which is necessary if there are multiple - derivations with the same name. - - - If *args* are [store derivation]s, then these are - [realised](nix-store.md#operation---realise), and the resulting output paths - are installed. - - [store derivation]: ../glossary.md#gloss-store-derivation - - - If *args* are store paths that are not store derivations, then these - are [realised](nix-store.md#operation---realise) and installed. - - - By default all outputs are installed for each derivation. That can - be reduced by setting `meta.outputsToInstall`. - -## Flags - - - `--prebuilt-only` / `-b`\ - Use only derivations for which a substitute is registered, i.e., - there is a pre-built binary available that can be downloaded in lieu - of building the derivation. Thus, no packages will be built from - source. - - - `--preserve-installed`; `-P`\ - Do not remove derivations with a name matching one of the - derivations being installed. Usually, trying to have two versions of - the same package installed in the same generation of a profile will - lead to an error in building the generation, due to file name - clashes between the two versions. However, this is not the case for - all packages. - - - `--remove-all`; `-r`\ - Remove all previously installed packages first. This is equivalent - to running `nix-env -e '.*'` first, except that everything happens - in a single transaction. - -## Examples - -To install a package using a specific attribute path from the active Nix expression: - -```console -$ nix-env -iA gcc40mips -installing `gcc-4.0.2' -$ nix-env -iA xorg.xorgserver -installing `xorg-server-1.2.0' -``` - -To install a specific version of `gcc` using the derivation name: - -```console -$ nix-env --install gcc-3.3.2 -installing `gcc-3.3.2' -uninstalling `gcc-3.1' -``` - -Using attribute path for selecting a package is preferred, -as it is much faster and there will not be multiple matches. - -Note the previously installed version is removed, since -`--preserve-installed` was not specified. - -To install an arbitrary version: - -```console -$ nix-env --install gcc -installing `gcc-3.3.2' -``` - -To install all derivations in the Nix expression `foo.nix`: - -```console -$ nix-env -f ~/foo.nix -i '.*' -``` - -To copy the store path with symbolic name `gcc` from another profile: - -```console -$ nix-env -i --from-profile /nix/var/nix/profiles/foo gcc -``` - -To install a specific [store derivation] (typically created by -`nix-instantiate`): - -```console -$ nix-env -i /nix/store/fibjb1bfbpm5mrsxc4mh2d8n37sxh91i-gcc-3.4.3.drv -``` - -To install a specific output path: - -```console -$ nix-env -i /nix/store/y3cgx0xj1p4iv9x0pnnmdhr8iyg741vk-gcc-3.4.3 -``` - -To install from a Nix expression specified on the command-line: - -```console -$ nix-env -f ./foo.nix -i -E \ - 'f: (f {system = "i686-linux";}).subversionWithJava' -``` - -I.e., this evaluates to `(f: (f {system = -"i686-linux";}).subversionWithJava) (import ./foo.nix)`, thus selecting -the `subversionWithJava` attribute from the set returned by calling the -function defined in `./foo.nix`. - -A dry-run tells you which paths will be downloaded or built from source: - -```console -$ nix-env -f '' -iA hello --dry-run -(dry run; not doing anything) -installing ‘hello-2.10’ -this path will be fetched (0.04 MiB download, 0.19 MiB unpacked): - /nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10 - ... -``` - -To install Firefox from the latest revision in the Nixpkgs/NixOS 14.12 -channel: - -```console -$ nix-env -f https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz -iA firefox -``` - -# Operation `--upgrade` - -## Synopsis - -`nix-env` {`--upgrade` | `-u`} *args* - [`--lt` | `--leq` | `--eq` | `--always`] - [{`--prebuilt-only` | `-b`}] - [{`--attr` | `-A`}] - [`--from-expression`] [`-E`] - [`--from-profile` *path*] - [`--preserve-installed` | `-P`] - -## Description - -The upgrade operation creates a new user environment, based on the -current generation of the active profile, in which all store paths are -replaced for which there are newer versions in the set of paths -described by *args*. Paths for which there are no newer versions are -left untouched; this is not an error. It is also not an error if an -element of *args* matches no installed derivations. - -For a description of how *args* is mapped to a set of store paths, see -[`--install`](#operation---install). If *args* describes multiple -store paths with the same symbolic name, only the one with the highest -version is installed. - -## Flags - - - `--lt`\ - Only upgrade a derivation to newer versions. This is the default. - - - `--leq`\ - In addition to upgrading to newer versions, also “upgrade” to - derivations that have the same version. Version are not a unique - identification of a derivation, so there may be many derivations - that have the same version. This flag may be useful to force - “synchronisation” between the installed and available derivations. - - - `--eq`\ - *Only* “upgrade” to derivations that have the same version. This may - not seem very useful, but it actually is, e.g., when there is a new - release of Nixpkgs and you want to replace installed applications - with the same versions built against newer dependencies (to reduce - the number of dependencies floating around on your system). - - - `--always`\ - In addition to upgrading to newer versions, also “upgrade” to - derivations that have the same or a lower version. I.e., derivations - may actually be downgraded depending on what is available in the - active Nix expression. - -For the other flags, see `--install`. - -## Examples - -```console -$ nix-env --upgrade -A nixpkgs.gcc -upgrading `gcc-3.3.1' to `gcc-3.4' -``` - -When there are no updates available, nothing will happen: - -```console -$ nix-env --upgrade -A nixpkgs.pan -``` - -Using `-A` is preferred when possible, as it is faster and unambiguous but -it is also possible to upgrade to a specific version by matching the derivation name: - -```console -$ nix-env -u gcc-3.3.2 --always -upgrading `gcc-3.4' to `gcc-3.3.2' -``` - -To try to upgrade everything -(matching packages based on the part of the derivation name without version): - -```console -$ nix-env -u -upgrading `hello-2.1.2' to `hello-2.1.3' -upgrading `mozilla-1.2' to `mozilla-1.4' -``` - -## Versions - -The upgrade operation determines whether a derivation `y` is an upgrade -of a derivation `x` by looking at their respective `name` attributes. -The names (e.g., `gcc-3.3.1` are split into two parts: the package name -(`gcc`), and the version (`3.3.1`). The version part starts after the -first dash not followed by a letter. `y` is considered an upgrade of `x` -if their package names match, and the version of `y` is higher than that -of `x`. - -The versions are compared by splitting them into contiguous components -of numbers and letters. E.g., `3.3.1pre5` is split into `[3, 3, 1, -"pre", 5]`. These lists are then compared lexicographically (from left -to right). Corresponding components `a` and `b` are compared as follows. -If they are both numbers, integer comparison is used. If `a` is an empty -string and `b` is a number, `a` is considered less than `b`. The special -string component `pre` (for *pre-release*) is considered to be less than -other components. String components are considered less than number -components. Otherwise, they are compared lexicographically (i.e., using -case-sensitive string comparison). - -This is illustrated by the following examples: - - 1.0 < 2.3 - 2.1 < 2.3 - 2.3 = 2.3 - 2.5 > 2.3 - 3.1 > 2.3 - 2.3.1 > 2.3 - 2.3.1 > 2.3a - 2.3pre1 < 2.3 - 2.3pre3 < 2.3pre12 - 2.3a < 2.3c - 2.3pre1 < 2.3c - 2.3pre1 < 2.3q - -# Operation `--uninstall` - -## Synopsis - -`nix-env` {`--uninstall` | `-e`} *drvnames…* - -## Description - -The uninstall operation creates a new user environment, based on the -current generation of the active profile, from which the store paths -designated by the symbolic names *drvnames* are removed. - -## Examples - -```console -$ nix-env --uninstall gcc -$ nix-env -e '.*' (remove everything) -``` - -# Operation `--set` - -## Synopsis - -`nix-env` `--set` *drvname* - -## Description - -The `--set` operation modifies the current generation of a profile so -that it contains exactly the specified derivation, and nothing else. - -## Examples - -The following updates a profile such that its current generation will -contain just Firefox: - -```console -$ nix-env -p /nix/var/nix/profiles/browser --set firefox -``` - -# Operation `--set-flag` - -## Synopsis - -`nix-env` `--set-flag` *name* *value* *drvnames* - -## Description - -The `--set-flag` operation allows meta attributes of installed packages -to be modified. There are several attributes that can be usefully -modified, because they affect the behaviour of `nix-env` or the user -environment build script: - - - `priority` can be changed to resolve filename clashes. The user - environment build script uses the `meta.priority` attribute of - derivations to resolve filename collisions between packages. Lower - priority values denote a higher priority. For instance, the GCC - wrapper package and the Binutils package in Nixpkgs both have a file - `bin/ld`, so previously if you tried to install both you would get a - collision. Now, on the other hand, the GCC wrapper declares a higher - priority than Binutils, so the former’s `bin/ld` is symlinked in the - user environment. - - - `keep` can be set to `true` to prevent the package from being - upgraded or replaced. This is useful if you want to hang on to an - older version of a package. - - - `active` can be set to `false` to “disable” the package. That is, no - symlinks will be generated to the files of the package, but it - remains part of the profile (so it won’t be garbage-collected). It - can be set back to `true` to re-enable the package. - -## Examples - -To prevent the currently installed Firefox from being upgraded: - -```console -$ nix-env --set-flag keep true firefox -``` - -After this, `nix-env -u` will ignore Firefox. - -To disable the currently installed Firefox, then install a new Firefox -while the old remains part of the profile: - -```console -$ nix-env -q -firefox-2.0.0.9 (the current one) - -$ nix-env --preserve-installed -i firefox-2.0.0.11 -installing `firefox-2.0.0.11' -building path(s) `/nix/store/myy0y59q3ig70dgq37jqwg1j0rsapzsl-user-environment' -collision between `/nix/store/...-firefox-2.0.0.11/bin/firefox' - and `/nix/store/...-firefox-2.0.0.9/bin/firefox'. -(i.e., can’t have two active at the same time) - -$ nix-env --set-flag active false firefox -setting flag on `firefox-2.0.0.9' - -$ nix-env --preserve-installed -i firefox-2.0.0.11 -installing `firefox-2.0.0.11' - -$ nix-env -q -firefox-2.0.0.11 (the enabled one) -firefox-2.0.0.9 (the disabled one) -``` - -To make files from `binutils` take precedence over files from `gcc`: - -```console -$ nix-env --set-flag priority 5 binutils -$ nix-env --set-flag priority 10 gcc -``` - -# Operation `--query` - -## Synopsis - -`nix-env` {`--query` | `-q`} *names…* - [`--installed` | `--available` | `-a`] - [{`--status` | `-s`}] - [{`--attr-path` | `-P`}] - [`--no-name`] - [{`--compare-versions` | `-c`}] - [`--system`] - [`--drv-path`] - [`--out-path`] - [`--description`] - [`--meta`] - [`--xml`] - [`--json`] - [{`--prebuilt-only` | `-b`}] - [{`--attr` | `-A`} *attribute-path*] - -## Description - -The query operation displays information about either the store paths -that are installed in the current generation of the active profile -(`--installed`), or the derivations that are available for installation -in the active Nix expression (`--available`). It only prints information -about derivations whose symbolic name matches one of *names*. - -The derivations are sorted by their `name` attributes. - -## Source selection - -The following flags specify the set of things on which the query -operates. - - - `--installed`\ - The query operates on the store paths that are installed in the - current generation of the active profile. This is the default. - - - `--available`; `-a`\ - The query operates on the derivations that are available in the - active Nix expression. - -## Queries - -The following flags specify what information to display about the -selected derivations. Multiple flags may be specified, in which case the -information is shown in the order given here. Note that the name of the -derivation is shown unless `--no-name` is specified. - - - `--xml`\ - Print the result in an XML representation suitable for automatic - processing by other tools. The root element is called `items`, which - contains a `item` element for each available or installed - derivation. The fields discussed below are all stored in attributes - of the `item` elements. - - - `--json`\ - Print the result in a JSON representation suitable for automatic - processing by other tools. - - - `--prebuilt-only` / `-b`\ - Show only derivations for which a substitute is registered, i.e., - there is a pre-built binary available that can be downloaded in lieu - of building the derivation. Thus, this shows all packages that - probably can be installed quickly. - - - `--status`; `-s`\ - Print the *status* of the derivation. The status consists of three - characters. The first is `I` or `-`, indicating whether the - derivation is currently installed in the current generation of the - active profile. This is by definition the case for `--installed`, - but not for `--available`. The second is `P` or `-`, indicating - whether the derivation is present on the system. This indicates - whether installation of an available derivation will require the - derivation to be built. The third is `S` or `-`, indicating whether - a substitute is available for the derivation. - - - `--attr-path`; `-P`\ - Print the *attribute path* of the derivation, which can be used to - unambiguously select it using the `--attr` option available in - commands that install derivations like `nix-env --install`. This - option only works together with `--available` - - - `--no-name`\ - Suppress printing of the `name` attribute of each derivation. - - - `--compare-versions` / `-c`\ - Compare installed versions to available versions, or vice versa (if - `--available` is given). This is useful for quickly seeing whether - upgrades for installed packages are available in a Nix expression. A - column is added with the following meaning: - - - `<` *version*\ - A newer version of the package is available or installed. - - - `=` *version*\ - At most the same version of the package is available or - installed. - - - `>` *version*\ - Only older versions of the package are available or installed. - - - `- ?`\ - No version of the package is available or installed. - - - `--system`\ - Print the `system` attribute of the derivation. - - - `--drv-path`\ - Print the path of the [store derivation]. - - - `--out-path`\ - Print the output path of the derivation. - - - `--description`\ - Print a short (one-line) description of the derivation, if - available. The description is taken from the `meta.description` - attribute of the derivation. - - - `--meta`\ - Print all of the meta-attributes of the derivation. This option is - only available with `--xml` or `--json`. - -## Examples - -To show installed packages: - -```console -$ nix-env -q -bison-1.875c -docbook-xml-4.2 -firefox-1.0.4 -MPlayer-1.0pre7 -ORBit2-2.8.3 -… -``` - -To show available packages: - -```console -$ nix-env -qa -firefox-1.0.7 -GConf-2.4.0.1 -MPlayer-1.0pre7 -ORBit2-2.8.3 -… -``` - -To show the status of available packages: - -```console -$ nix-env -qas --P- firefox-1.0.7 (not installed but present) ---S GConf-2.4.0.1 (not present, but there is a substitute for fast installation) ---S MPlayer-1.0pre3 (i.e., this is not the installed MPlayer, even though the version is the same!) -IP- ORBit2-2.8.3 (installed and by definition present) -… -``` - -To show available packages in the Nix expression `foo.nix`: - -```console -$ nix-env -f ./foo.nix -qa -foo-1.2.3 -``` - -To compare installed versions to what’s available: - -```console -$ nix-env -qc -... -acrobat-reader-7.0 - ? (package is not available at all) -autoconf-2.59 = 2.59 (same version) -firefox-1.0.4 < 1.0.7 (a more recent version is available) -... -``` - -To show all packages with “`zip`” in the name: - -```console -$ nix-env -qa '.*zip.*' -bzip2-1.0.6 -gzip-1.6 -zip-3.0 -… -``` - -To show all packages with “`firefox`” or “`chromium`” in the name: - -```console -$ nix-env -qa '.*(firefox|chromium).*' -chromium-37.0.2062.94 -chromium-beta-38.0.2125.24 -firefox-32.0.3 -firefox-with-plugins-13.0.1 -… -``` - -To show all packages in the latest revision of the Nixpkgs repository: - -```console -$ nix-env -f https://github.com/NixOS/nixpkgs/archive/master.tar.gz -qa -``` - -# Operation `--switch-profile` - -## Synopsis - -`nix-env` {`--switch-profile` | `-S`} *path* - -## Description - -This operation makes *path* the current profile for the user. That is, -the symlink `~/.nix-profile` is made to point to *path*. - -## Examples - -```console -$ nix-env -S ~/my-profile -``` - -# Operation `--list-generations` - -## Synopsis - -`nix-env` `--list-generations` - -## Description - -This operation print a list of all the currently existing generations -for the active profile. These may be switched to using the -`--switch-generation` operation. It also prints the creation date of the -generation, and indicates the current generation. - -## Examples - -```console -$ nix-env --list-generations - 95 2004-02-06 11:48:24 - 96 2004-02-06 11:49:01 - 97 2004-02-06 16:22:45 - 98 2004-02-06 16:24:33 (current) -``` - -# Operation `--delete-generations` - -## Synopsis - -`nix-env` `--delete-generations` *generations* - -## Description - -This operation deletes the specified generations of the current profile. -The generations can be a list of generation numbers, the special value -`old` to delete all non-current generations, a value such as `30d` to -delete all generations older than the specified number of days (except -for the generation that was active at that point in time), or a value -such as `+5` to keep the last `5` generations ignoring any newer than -current, e.g., if `30` is the current generation `+5` will delete -generation `25` and all older generations. Periodically deleting old -generations is important to make garbage collection effective. - -## Examples - -```console -$ nix-env --delete-generations 3 4 8 -``` - -```console -$ nix-env --delete-generations +5 -``` - -```console -$ nix-env --delete-generations 30d -``` - -```console -$ nix-env -p other_profile --delete-generations old -``` - -# Operation `--switch-generation` - -## Synopsis - -`nix-env` {`--switch-generation` | `-G`} *generation* - -## Description - -This operation makes generation number *generation* the current -generation of the active profile. That is, if the `profile` is the path -to the active profile, then the symlink `profile` is made to point to -`profile-generation-link`, which is in turn a symlink to the actual user -environment in the Nix store. - -Switching will fail if the specified generation does not exist. - -## Examples - -```console -$ nix-env -G 42 -switching from generation 50 to 42 -``` - -# Operation `--rollback` - -## Synopsis - -`nix-env` `--rollback` - -## Description - -This operation switches to the “previous” generation of the active -profile, that is, the highest numbered generation lower than the current -generation, if it exists. It is just a convenience wrapper around -`--list-generations` and `--switch-generation`. - -## Examples - -```console -$ nix-env --rollback -switching from generation 92 to 91 -``` - -```console -$ nix-env --rollback -error: no generation older than the current (91) exists -``` - -# Environment variables - - - `NIX_PROFILE`\ - Location of the Nix profile. Defaults to the target of the symlink - `~/.nix-profile`, if it exists, or `/nix/var/nix/profiles/default` - otherwise. diff --git a/doc/manual/src/command-ref/nix-env/delete-generations.md b/doc/manual/src/command-ref/nix-env/delete-generations.md new file mode 100644 index 000000000..6f0af5384 --- /dev/null +++ b/doc/manual/src/command-ref/nix-env/delete-generations.md @@ -0,0 +1,46 @@ +# Name + +`nix-env --delete-generations` - delete profile generations + +# Synopsis + +`nix-env` `--delete-generations` *generations* + +# Description + +This operation deletes the specified generations of the current profile. +The generations can be a list of generation numbers, the special value +`old` to delete all non-current generations, a value such as `30d` to +delete all generations older than the specified number of days (except +for the generation that was active at that point in time), or a value +such as `+5` to keep the last `5` generations ignoring any newer than +current, e.g., if `30` is the current generation `+5` will delete +generation `25` and all older generations. Periodically deleting old +generations is important to make garbage collection effective. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ./env-common.md}} + +{{#include ../env-common.md}} + +# Examples + +```console +$ nix-env --delete-generations 3 4 8 +``` + +```console +$ nix-env --delete-generations +5 +``` + +```console +$ nix-env --delete-generations 30d +``` + +```console +$ nix-env -p other_profile --delete-generations old +``` + diff --git a/doc/manual/src/command-ref/nix-env/env-common.md b/doc/manual/src/command-ref/nix-env/env-common.md new file mode 100644 index 000000000..735817959 --- /dev/null +++ b/doc/manual/src/command-ref/nix-env/env-common.md @@ -0,0 +1,6 @@ +# Environment variables + +- `NIX_PROFILE`\ + Location of the Nix profile. Defaults to the target of the symlink + `~/.nix-profile`, if it exists, or `/nix/var/nix/profiles/default` + otherwise. diff --git a/doc/manual/src/command-ref/nix-env/install.md b/doc/manual/src/command-ref/nix-env/install.md new file mode 100644 index 000000000..d754accfe --- /dev/null +++ b/doc/manual/src/command-ref/nix-env/install.md @@ -0,0 +1,187 @@ +# Name + +`nix-env --install` - add packages to user environment + +# Synopsis + +`nix-env` {`--install` | `-i`} *args…* + [{`--prebuilt-only` | `-b`}] + [{`--attr` | `-A`}] + [`--from-expression`] [`-E`] + [`--from-profile` *path*] + [`--preserve-installed` | `-P`] + [`--remove-all` | `-r`] + +# Description + +The install operation creates a new user environment, based on the +current generation of the active profile, to which a set of store paths +described by *args* is added. The arguments *args* map to store paths in +a number of possible ways: + + - By default, *args* is a set of derivation names denoting derivations + in the active Nix expression. These are realised, and the resulting + output paths are installed. Currently installed derivations with a + name equal to the name of a derivation being added are removed + unless the option `--preserve-installed` is specified. + + If there are multiple derivations matching a name in *args* that + have the same name (e.g., `gcc-3.3.6` and `gcc-4.1.1`), then the + derivation with the highest *priority* is used. A derivation can + define a priority by declaring the `meta.priority` attribute. This + attribute should be a number, with a higher value denoting a lower + priority. The default priority is `0`. + + If there are multiple matching derivations with the same priority, + then the derivation with the highest version will be installed. + + You can force the installation of multiple derivations with the same + name by being specific about the versions. For instance, `nix-env -i + gcc-3.3.6 gcc-4.1.1` will install both version of GCC (and will + probably cause a user environment conflict\!). + + - If `--attr` (`-A`) is specified, the arguments are *attribute + paths* that select attributes from the top-level Nix + expression. This is faster than using derivation names and + unambiguous. To find out the attribute paths of available + packages, use `nix-env -qaP`. + + - If `--from-profile` *path* is given, *args* is a set of names + denoting installed store paths in the profile *path*. This is an + easy way to copy user environment elements from one profile to + another. + + - If `--from-expression` is given, *args* are Nix + [functions](@docroot@/language/constructs.md#functions) + that are called with the active Nix expression as their single + argument. The derivations returned by those function calls are + installed. This allows derivations to be specified in an + unambiguous way, which is necessary if there are multiple + derivations with the same name. + + - If *args* are [store derivations](@docroot@/glossary.md#gloss-store-derivation), then these are + [realised](@docroot@/command-ref/nix-store/realise.md), and the resulting output paths + are installed. + + - If *args* are store paths that are not store derivations, then these + are [realised](@docroot@/command-ref/nix-store/realise.md) and installed. + + - By default all outputs are installed for each derivation. That can + be reduced by setting `meta.outputsToInstall`. + +# Flags + + - `--prebuilt-only` / `-b`\ + Use only derivations for which a substitute is registered, i.e., + there is a pre-built binary available that can be downloaded in lieu + of building the derivation. Thus, no packages will be built from + source. + + - `--preserve-installed` / `-P`\ + Do not remove derivations with a name matching one of the + derivations being installed. Usually, trying to have two versions of + the same package installed in the same generation of a profile will + lead to an error in building the generation, due to file name + clashes between the two versions. However, this is not the case for + all packages. + + - `--remove-all` / `-r`\ + Remove all previously installed packages first. This is equivalent + to running `nix-env -e '.*'` first, except that everything happens + in a single transaction. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ./env-common.md}} + +{{#include ../env-common.md}} + +# Examples + +To install a package using a specific attribute path from the active Nix expression: + +```console +$ nix-env -iA gcc40mips +installing `gcc-4.0.2' +$ nix-env -iA xorg.xorgserver +installing `xorg-server-1.2.0' +``` + +To install a specific version of `gcc` using the derivation name: + +```console +$ nix-env --install gcc-3.3.2 +installing `gcc-3.3.2' +uninstalling `gcc-3.1' +``` + +Using attribute path for selecting a package is preferred, +as it is much faster and there will not be multiple matches. + +Note the previously installed version is removed, since +`--preserve-installed` was not specified. + +To install an arbitrary version: + +```console +$ nix-env --install gcc +installing `gcc-3.3.2' +``` + +To install all derivations in the Nix expression `foo.nix`: + +```console +$ nix-env -f ~/foo.nix -i '.*' +``` + +To copy the store path with symbolic name `gcc` from another profile: + +```console +$ nix-env -i --from-profile /nix/var/nix/profiles/foo gcc +``` + +To install a specific [store derivation] (typically created by +`nix-instantiate`): + +```console +$ nix-env -i /nix/store/fibjb1bfbpm5mrsxc4mh2d8n37sxh91i-gcc-3.4.3.drv +``` + +To install a specific output path: + +```console +$ nix-env -i /nix/store/y3cgx0xj1p4iv9x0pnnmdhr8iyg741vk-gcc-3.4.3 +``` + +To install from a Nix expression specified on the command-line: + +```console +$ nix-env -f ./foo.nix -i -E \ + 'f: (f {system = "i686-linux";}).subversionWithJava' +``` + +I.e., this evaluates to `(f: (f {system = +"i686-linux";}).subversionWithJava) (import ./foo.nix)`, thus selecting +the `subversionWithJava` attribute from the set returned by calling the +function defined in `./foo.nix`. + +A dry-run tells you which paths will be downloaded or built from source: + +```console +$ nix-env -f '' -iA hello --dry-run +(dry run; not doing anything) +installing ‘hello-2.10’ +this path will be fetched (0.04 MiB download, 0.19 MiB unpacked): + /nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10 + ... +``` + +To install Firefox from the latest revision in the Nixpkgs/NixOS 14.12 +channel: + +```console +$ nix-env -f https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz -iA firefox +``` + diff --git a/doc/manual/src/command-ref/nix-env/list-generations.md b/doc/manual/src/command-ref/nix-env/list-generations.md new file mode 100644 index 000000000..a4881ece8 --- /dev/null +++ b/doc/manual/src/command-ref/nix-env/list-generations.md @@ -0,0 +1,33 @@ +# Name + +`nix-env --list-generations` - list profile generations + +# Synopsis + +`nix-env` `--list-generations` + +# Description + +This operation print a list of all the currently existing generations +for the active profile. These may be switched to using the +`--switch-generation` operation. It also prints the creation date of the +generation, and indicates the current generation. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ./env-common.md}} + +{{#include ../env-common.md}} + +# Examples + +```console +$ nix-env --list-generations + 95 2004-02-06 11:48:24 + 96 2004-02-06 11:49:01 + 97 2004-02-06 16:22:45 + 98 2004-02-06 16:24:33 (current) +``` + diff --git a/doc/manual/src/command-ref/nix-env/opt-common.md b/doc/manual/src/command-ref/nix-env/opt-common.md new file mode 100644 index 000000000..636281b6d --- /dev/null +++ b/doc/manual/src/command-ref/nix-env/opt-common.md @@ -0,0 +1,35 @@ +# Options + +The following options are allowed for all `nix-env` operations, but may not always have an effect. + + - `--file` / `-f` *path*\ + Specifies the Nix expression (designated below as the *active Nix + expression*) used by the `--install`, `--upgrade`, and `--query + --available` operations to obtain derivations. The default is + `~/.nix-defexpr`. + + If the argument starts with `http://` or `https://`, it is + interpreted as the URL of a tarball that will be downloaded and + unpacked to a temporary location. The tarball must include a single + top-level directory containing at least a file named `default.nix`. + + - `--profile` / `-p` *path*\ + Specifies the profile to be used by those operations that operate on + a profile (designated below as the *active profile*). A profile is a + sequence of user environments called *generations*, one of which is + the *current generation*. + + - `--dry-run`\ + For the `--install`, `--upgrade`, `--uninstall`, + `--switch-generation`, `--delete-generations` and `--rollback` + operations, this flag will cause `nix-env` to print what *would* be + done if this flag had not been specified, without actually doing it. + + `--dry-run` also prints out which paths will be + [substituted](@docroot@/glossary.md) (i.e., downloaded) and which paths + will be built from source (because no substitute is available). + + - `--system-filter` *system*\ + By default, operations such as `--query + --available` show derivations matching any platform. This option + allows you to use derivations for the specified platform *system*. diff --git a/doc/manual/src/command-ref/nix-env/query.md b/doc/manual/src/command-ref/nix-env/query.md new file mode 100644 index 000000000..18f0ee210 --- /dev/null +++ b/doc/manual/src/command-ref/nix-env/query.md @@ -0,0 +1,215 @@ +# Name + +`nix-env --query` - display information about packages + +# Synopsis + +`nix-env` {`--query` | `-q`} *names…* + [`--installed` | `--available` | `-a`] + [{`--status` | `-s`}] + [{`--attr-path` | `-P`}] + [`--no-name`] + [{`--compare-versions` | `-c`}] + [`--system`] + [`--drv-path`] + [`--out-path`] + [`--description`] + [`--meta`] + [`--xml`] + [`--json`] + [{`--prebuilt-only` | `-b`}] + [{`--attr` | `-A`} *attribute-path*] + +# Description + +The query operation displays information about either the store paths +that are installed in the current generation of the active profile +(`--installed`), or the derivations that are available for installation +in the active Nix expression (`--available`). It only prints information +about derivations whose symbolic name matches one of *names*. + +The derivations are sorted by their `name` attributes. + +# Source selection + +The following flags specify the set of things on which the query +operates. + + - `--installed`\ + The query operates on the store paths that are installed in the + current generation of the active profile. This is the default. + + - `--available`; `-a`\ + The query operates on the derivations that are available in the + active Nix expression. + +# Queries + +The following flags specify what information to display about the +selected derivations. Multiple flags may be specified, in which case the +information is shown in the order given here. Note that the name of the +derivation is shown unless `--no-name` is specified. + + - `--xml`\ + Print the result in an XML representation suitable for automatic + processing by other tools. The root element is called `items`, which + contains a `item` element for each available or installed + derivation. The fields discussed below are all stored in attributes + of the `item` elements. + + - `--json`\ + Print the result in a JSON representation suitable for automatic + processing by other tools. + + - `--prebuilt-only` / `-b`\ + Show only derivations for which a substitute is registered, i.e., + there is a pre-built binary available that can be downloaded in lieu + of building the derivation. Thus, this shows all packages that + probably can be installed quickly. + + - `--status`; `-s`\ + Print the *status* of the derivation. The status consists of three + characters. The first is `I` or `-`, indicating whether the + derivation is currently installed in the current generation of the + active profile. This is by definition the case for `--installed`, + but not for `--available`. The second is `P` or `-`, indicating + whether the derivation is present on the system. This indicates + whether installation of an available derivation will require the + derivation to be built. The third is `S` or `-`, indicating whether + a substitute is available for the derivation. + + - `--attr-path`; `-P`\ + Print the *attribute path* of the derivation, which can be used to + unambiguously select it using the `--attr` option available in + commands that install derivations like `nix-env --install`. This + option only works together with `--available` + + - `--no-name`\ + Suppress printing of the `name` attribute of each derivation. + + - `--compare-versions` / `-c`\ + Compare installed versions to available versions, or vice versa (if + `--available` is given). This is useful for quickly seeing whether + upgrades for installed packages are available in a Nix expression. A + column is added with the following meaning: + + - `<` *version*\ + A newer version of the package is available or installed. + + - `=` *version*\ + At most the same version of the package is available or + installed. + + - `>` *version*\ + Only older versions of the package are available or installed. + + - `- ?`\ + No version of the package is available or installed. + + - `--system`\ + Print the `system` attribute of the derivation. + + - `--drv-path`\ + Print the path of the [store derivation](@docroot@/glossary.md#gloss-store-derivation). + + - `--out-path`\ + Print the output path of the derivation. + + - `--description`\ + Print a short (one-line) description of the derivation, if + available. The description is taken from the `meta.description` + attribute of the derivation. + + - `--meta`\ + Print all of the meta-attributes of the derivation. This option is + only available with `--xml` or `--json`. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ./env-common.md}} + +{{#include ../env-common.md}} + +# Examples + +To show installed packages: + +```console +$ nix-env -q +bison-1.875c +docbook-xml-4.2 +firefox-1.0.4 +MPlayer-1.0pre7 +ORBit2-2.8.3 +… +``` + +To show available packages: + +```console +$ nix-env -qa +firefox-1.0.7 +GConf-2.4.0.1 +MPlayer-1.0pre7 +ORBit2-2.8.3 +… +``` + +To show the status of available packages: + +```console +$ nix-env -qas +-P- firefox-1.0.7 (not installed but present) +--S GConf-2.4.0.1 (not present, but there is a substitute for fast installation) +--S MPlayer-1.0pre3 (i.e., this is not the installed MPlayer, even though the version is the same!) +IP- ORBit2-2.8.3 (installed and by definition present) +… +``` + +To show available packages in the Nix expression `foo.nix`: + +```console +$ nix-env -f ./foo.nix -qa +foo-1.2.3 +``` + +To compare installed versions to what’s available: + +```console +$ nix-env -qc +... +acrobat-reader-7.0 - ? (package is not available at all) +autoconf-2.59 = 2.59 (same version) +firefox-1.0.4 < 1.0.7 (a more recent version is available) +... +``` + +To show all packages with “`zip`” in the name: + +```console +$ nix-env -qa '.*zip.*' +bzip2-1.0.6 +gzip-1.6 +zip-3.0 +… +``` + +To show all packages with “`firefox`” or “`chromium`” in the name: + +```console +$ nix-env -qa '.*(firefox|chromium).*' +chromium-37.0.2062.94 +chromium-beta-38.0.2125.24 +firefox-32.0.3 +firefox-with-plugins-13.0.1 +… +``` + +To show all packages in the latest revision of the Nixpkgs repository: + +```console +$ nix-env -f https://github.com/NixOS/nixpkgs/archive/master.tar.gz -qa +``` + diff --git a/doc/manual/src/command-ref/nix-env/rollback.md b/doc/manual/src/command-ref/nix-env/rollback.md new file mode 100644 index 000000000..1e3958cfc --- /dev/null +++ b/doc/manual/src/command-ref/nix-env/rollback.md @@ -0,0 +1,34 @@ +# Name + +`nix-env --rollback` - set user environment to previous generation + +# Synopsis + +`nix-env` `--rollback` + +# Description + +This operation switches to the “previous” generation of the active +profile, that is, the highest numbered generation lower than the current +generation, if it exists. It is just a convenience wrapper around +`--list-generations` and `--switch-generation`. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ./env-common.md}} + +{{#include ../env-common.md}} + +# Examples + +```console +$ nix-env --rollback +switching from generation 92 to 91 +``` + +```console +$ nix-env --rollback +error: no generation older than the current (91) exists +``` diff --git a/doc/manual/src/command-ref/nix-env/set-flag.md b/doc/manual/src/command-ref/nix-env/set-flag.md new file mode 100644 index 000000000..63f0a0ff9 --- /dev/null +++ b/doc/manual/src/command-ref/nix-env/set-flag.md @@ -0,0 +1,82 @@ +# Name + +`nix-env --set-flag` - modify meta attributes of installed packages + +# Synopsis + +`nix-env` `--set-flag` *name* *value* *drvnames* + +# Description + +The `--set-flag` operation allows meta attributes of installed packages +to be modified. There are several attributes that can be usefully +modified, because they affect the behaviour of `nix-env` or the user +environment build script: + + - `priority` can be changed to resolve filename clashes. The user + environment build script uses the `meta.priority` attribute of + derivations to resolve filename collisions between packages. Lower + priority values denote a higher priority. For instance, the GCC + wrapper package and the Binutils package in Nixpkgs both have a file + `bin/ld`, so previously if you tried to install both you would get a + collision. Now, on the other hand, the GCC wrapper declares a higher + priority than Binutils, so the former’s `bin/ld` is symlinked in the + user environment. + + - `keep` can be set to `true` to prevent the package from being + upgraded or replaced. This is useful if you want to hang on to an + older version of a package. + + - `active` can be set to `false` to “disable” the package. That is, no + symlinks will be generated to the files of the package, but it + remains part of the profile (so it won’t be garbage-collected). It + can be set back to `true` to re-enable the package. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} + +# Examples + +To prevent the currently installed Firefox from being upgraded: + +```console +$ nix-env --set-flag keep true firefox +``` + +After this, `nix-env -u` will ignore Firefox. + +To disable the currently installed Firefox, then install a new Firefox +while the old remains part of the profile: + +```console +$ nix-env -q +firefox-2.0.0.9 (the current one) + +$ nix-env --preserve-installed -i firefox-2.0.0.11 +installing `firefox-2.0.0.11' +building path(s) `/nix/store/myy0y59q3ig70dgq37jqwg1j0rsapzsl-user-environment' +collision between `/nix/store/...-firefox-2.0.0.11/bin/firefox' + and `/nix/store/...-firefox-2.0.0.9/bin/firefox'. +(i.e., can’t have two active at the same time) + +$ nix-env --set-flag active false firefox +setting flag on `firefox-2.0.0.9' + +$ nix-env --preserve-installed -i firefox-2.0.0.11 +installing `firefox-2.0.0.11' + +$ nix-env -q +firefox-2.0.0.11 (the enabled one) +firefox-2.0.0.9 (the disabled one) +``` + +To make files from `binutils` take precedence over files from `gcc`: + +```console +$ nix-env --set-flag priority 5 binutils +$ nix-env --set-flag priority 10 gcc +``` + diff --git a/doc/manual/src/command-ref/nix-env/set.md b/doc/manual/src/command-ref/nix-env/set.md new file mode 100644 index 000000000..c1cf75739 --- /dev/null +++ b/doc/manual/src/command-ref/nix-env/set.md @@ -0,0 +1,30 @@ +# Name + +`nix-env --set` - set profile to contain a specified derivation + +## Synopsis + +`nix-env` `--set` *drvname* + +## Description + +The `--set` operation modifies the current generation of a profile so +that it contains exactly the specified derivation, and nothing else. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ./env-common.md}} + +{{#include ../env-common.md}} + +## Examples + +The following updates a profile such that its current generation will +contain just Firefox: + +```console +$ nix-env -p /nix/var/nix/profiles/browser --set firefox +``` + diff --git a/doc/manual/src/command-ref/nix-env/switch-generation.md b/doc/manual/src/command-ref/nix-env/switch-generation.md new file mode 100644 index 000000000..e550325c4 --- /dev/null +++ b/doc/manual/src/command-ref/nix-env/switch-generation.md @@ -0,0 +1,33 @@ +# Name + +`nix-env --switch-generation` - set user environment to given profile generation + +# Synopsis + +`nix-env` {`--switch-generation` | `-G`} *generation* + +# Description + +This operation makes generation number *generation* the current +generation of the active profile. That is, if the `profile` is the path +to the active profile, then the symlink `profile` is made to point to +`profile-generation-link`, which is in turn a symlink to the actual user +environment in the Nix store. + +Switching will fail if the specified generation does not exist. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ./env-common.md}} + +{{#include ../env-common.md}} + +# Examples + +```console +$ nix-env -G 42 +switching from generation 50 to 42 +``` + diff --git a/doc/manual/src/command-ref/nix-env/switch-profile.md b/doc/manual/src/command-ref/nix-env/switch-profile.md new file mode 100644 index 000000000..b389e4140 --- /dev/null +++ b/doc/manual/src/command-ref/nix-env/switch-profile.md @@ -0,0 +1,26 @@ +# Name + +`nix-env --switch-profile` - set user environment to given profile + +# Synopsis + +`nix-env` {`--switch-profile` | `-S`} *path* + +# Description + +This operation makes *path* the current profile for the user. That is, +the symlink `~/.nix-profile` is made to point to *path*. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ./env-common.md}} + +{{#include ../env-common.md}} + +# Examples + +```console +$ nix-env -S ~/my-profile +``` diff --git a/doc/manual/src/command-ref/nix-env/uninstall.md b/doc/manual/src/command-ref/nix-env/uninstall.md new file mode 100644 index 000000000..e9ec8a15e --- /dev/null +++ b/doc/manual/src/command-ref/nix-env/uninstall.md @@ -0,0 +1,28 @@ +# Name + +`nix-env --uninstall` - remove packages from user environment + +# Synopsis + +`nix-env` {`--uninstall` | `-e`} *drvnames…* + +# Description + +The uninstall operation creates a new user environment, based on the +current generation of the active profile, from which the store paths +designated by the symbolic names *drvnames* are removed. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ./env-common.md}} + +{{#include ../env-common.md}} + +# Examples + +```console +$ nix-env --uninstall gcc +$ nix-env -e '.*' (remove everything) +``` diff --git a/doc/manual/src/command-ref/nix-env/upgrade.md b/doc/manual/src/command-ref/nix-env/upgrade.md new file mode 100644 index 000000000..f88ffcbee --- /dev/null +++ b/doc/manual/src/command-ref/nix-env/upgrade.md @@ -0,0 +1,141 @@ +# Name + +`nix-env --upgrade` - upgrade packages in user environment + +# Synopsis + +`nix-env` {`--upgrade` | `-u`} *args* + [`--lt` | `--leq` | `--eq` | `--always`] + [{`--prebuilt-only` | `-b`}] + [{`--attr` | `-A`}] + [`--from-expression`] [`-E`] + [`--from-profile` *path*] + [`--preserve-installed` | `-P`] + +# Description + +The upgrade operation creates a new user environment, based on the +current generation of the active profile, in which all store paths are +replaced for which there are newer versions in the set of paths +described by *args*. Paths for which there are no newer versions are +left untouched; this is not an error. It is also not an error if an +element of *args* matches no installed derivations. + +For a description of how *args* is mapped to a set of store paths, see +[`--install`](#operation---install). If *args* describes multiple +store paths with the same symbolic name, only the one with the highest +version is installed. + +# Flags + + - `--lt`\ + Only upgrade a derivation to newer versions. This is the default. + + - `--leq`\ + In addition to upgrading to newer versions, also “upgrade” to + derivations that have the same version. Version are not a unique + identification of a derivation, so there may be many derivations + that have the same version. This flag may be useful to force + “synchronisation” between the installed and available derivations. + + - `--eq`\ + *Only* “upgrade” to derivations that have the same version. This may + not seem very useful, but it actually is, e.g., when there is a new + release of Nixpkgs and you want to replace installed applications + with the same versions built against newer dependencies (to reduce + the number of dependencies floating around on your system). + + - `--always`\ + In addition to upgrading to newer versions, also “upgrade” to + derivations that have the same or a lower version. I.e., derivations + may actually be downgraded depending on what is available in the + active Nix expression. + + - `--prebuilt-only` / `-b`\ + Use only derivations for which a substitute is registered, i.e., + there is a pre-built binary available that can be downloaded in lieu + of building the derivation. Thus, no packages will be built from + source. + + - `--preserve-installed` / `-P`\ + Do not remove derivations with a name matching one of the + derivations being installed. Usually, trying to have two versions of + the same package installed in the same generation of a profile will + lead to an error in building the generation, due to file name + clashes between the two versions. However, this is not the case for + all packages. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ./env-common.md}} + +{{#include ../env-common.md}} + +# Examples + +```console +$ nix-env --upgrade -A nixpkgs.gcc +upgrading `gcc-3.3.1' to `gcc-3.4' +``` + +When there are no updates available, nothing will happen: + +```console +$ nix-env --upgrade -A nixpkgs.pan +``` + +Using `-A` is preferred when possible, as it is faster and unambiguous but +it is also possible to upgrade to a specific version by matching the derivation name: + +```console +$ nix-env -u gcc-3.3.2 --always +upgrading `gcc-3.4' to `gcc-3.3.2' +``` + +To try to upgrade everything +(matching packages based on the part of the derivation name without version): + +```console +$ nix-env -u +upgrading `hello-2.1.2' to `hello-2.1.3' +upgrading `mozilla-1.2' to `mozilla-1.4' +``` + +# Versions + +The upgrade operation determines whether a derivation `y` is an upgrade +of a derivation `x` by looking at their respective `name` attributes. +The names (e.g., `gcc-3.3.1` are split into two parts: the package name +(`gcc`), and the version (`3.3.1`). The version part starts after the +first dash not followed by a letter. `y` is considered an upgrade of `x` +if their package names match, and the version of `y` is higher than that +of `x`. + +The versions are compared by splitting them into contiguous components +of numbers and letters. E.g., `3.3.1pre5` is split into `[3, 3, 1, +"pre", 5]`. These lists are then compared lexicographically (from left +to right). Corresponding components `a` and `b` are compared as follows. +If they are both numbers, integer comparison is used. If `a` is an empty +string and `b` is a number, `a` is considered less than `b`. The special +string component `pre` (for *pre-release*) is considered to be less than +other components. String components are considered less than number +components. Otherwise, they are compared lexicographically (i.e., using +case-sensitive string comparison). + +This is illustrated by the following examples: + + 1.0 < 2.3 + 2.1 < 2.3 + 2.3 = 2.3 + 2.5 > 2.3 + 3.1 > 2.3 + 2.3.1 > 2.3 + 2.3.1 > 2.3a + 2.3pre1 < 2.3 + 2.3pre3 < 2.3pre12 + 2.3a < 2.3c + 2.3pre1 < 2.3c + 2.3pre1 < 2.3q + diff --git a/doc/manual/src/command-ref/nix-hash.md b/doc/manual/src/command-ref/nix-hash.md index 90fedfb14..37c8facec 100644 --- a/doc/manual/src/command-ref/nix-hash.md +++ b/doc/manual/src/command-ref/nix-hash.md @@ -21,7 +21,7 @@ The hash is computed over a *serialisation* of each path: a dump of the file system tree rooted at the path. This allows directories and symlinks to be hashed as well as regular files. The dump is in the *NAR format* produced by [`nix-store ---dump`](nix-store.md#operation---dump). Thus, `nix-hash path` +--dump`](@docroot@/command-ref/nix-store/dump.md). Thus, `nix-hash path` yields the same cryptographic hash as `nix-store --dump path | md5sum`. diff --git a/doc/manual/src/command-ref/nix-instantiate.md b/doc/manual/src/command-ref/nix-instantiate.md index 432fb2608..e55fb2afd 100644 --- a/doc/manual/src/command-ref/nix-instantiate.md +++ b/doc/manual/src/command-ref/nix-instantiate.md @@ -76,7 +76,9 @@ standard input. this option is not enabled, there may be uninstantiated store paths in the final output. - +{{#include ./opt-common.md}} + +{{#include ./env-common.md}} # Examples diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index 840bccd25..576e5ba0b 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -101,7 +101,7 @@ All options not listed here are passed to `nix-store When a `--pure` shell is started, keep the listed environment variables. -The following common options are supported: +{{#include ./opt-common.md}} # Environment variables @@ -110,6 +110,8 @@ The following common options are supported: `bash` found in ``, falling back to the `bash` found in `PATH` if not found. +{{#include ./env-common.md}} + # Examples To build the dependencies of the package Pan, and start an interactive @@ -118,7 +120,8 @@ shell in which to build it: ```console $ nix-shell '' -A pan [nix-shell]$ eval ${unpackPhase:-unpackPhase} -[nix-shell]$ cd pan-* +[nix-shell]$ cd $sourceRoot +[nix-shell]$ eval ${patchPhase:-patchPhase} [nix-shell]$ eval ${configurePhase:-configurePhase} [nix-shell]$ eval ${buildPhase:-buildPhase} [nix-shell]$ ./pan/gui/pan diff --git a/doc/manual/src/command-ref/nix-store.md b/doc/manual/src/command-ref/nix-store.md index 17cbd0461..c7c5fdd2f 100644 --- a/doc/manual/src/command-ref/nix-store.md +++ b/doc/manual/src/command-ref/nix-store.md @@ -13,838 +13,35 @@ The command `nix-store` performs primitive operations on the Nix store. You generally do not need to run this command manually. -`nix-store` takes exactly one *operation* flag which indicates the -subcommand to be performed. These are documented below. - -# Common options - -This section lists the options that are common to all operations. These -options are allowed for every subcommand, though they may not always -have an effect. - - - [`--add-root`](#opt-add-root) *path* - - Causes the result of a realisation (`--realise` and - `--force-realise`) to be registered as a root of the garbage - collector. *path* will be created as a symlink to the resulting - store path. In addition, a uniquely named symlink to *path* will - be created in `/nix/var/nix/gcroots/auto/`. For instance, - - ```console - $ nix-store --add-root /home/eelco/bla/result -r ... - - $ ls -l /nix/var/nix/gcroots/auto - lrwxrwxrwx 1 ... 2005-03-13 21:10 dn54lcypm8f8... -> /home/eelco/bla/result - - $ ls -l /home/eelco/bla/result - lrwxrwxrwx 1 ... 2005-03-13 21:10 /home/eelco/bla/result -> /nix/store/1r11343n6qd4...-f-spot-0.0.10 - ``` - - Thus, when `/home/eelco/bla/result` is removed, the GC root in the - `auto` directory becomes a dangling symlink and will be ignored by - the collector. - - > **Warning** - > - > Note that it is not possible to move or rename GC roots, since - > the symlink in the `auto` directory will still point to the old - > location. - - If there are multiple results, then multiple symlinks will be - created by sequentially numbering symlinks beyond the first one - (e.g., `foo`, `foo-2`, `foo-3`, and so on). - - - [`--stdin`](#opt-stdin) - - Read *paths…* from the standard input. - Useful for chaining nix-store commands. - -# Operation `--realise` - -## Synopsis - -`nix-store` {`--realise` | `-r`} *paths…* [`--dry-run`] - -## Description - -The operation `--realise` essentially “builds” the specified store -paths. Realisation is a somewhat overloaded term: - - - If the store path is a *derivation*, realisation ensures that the - output paths of the derivation are [valid] (i.e., - the output path and its closure exist in the file system). This - can be done in several ways. First, it is possible that the - outputs are already valid, in which case we are done - immediately. Otherwise, there may be [substitutes] - that produce the outputs (e.g., by downloading them). Finally, the - outputs can be produced by running the build task described - by the derivation. - - - If the store path is not a derivation, realisation ensures that the - specified path is valid (i.e., it and its closure exist in the file - system). If the path is already valid, we are done immediately. - Otherwise, the path and any missing paths in its closure may be - produced through substitutes. If there are no (successful) - substitutes, realisation fails. - -[valid]: ../glossary.md#gloss-validity -[substitutes]: ../glossary.md#gloss-substitute - -The output path of each derivation is printed on standard output. (For -non-derivations argument, the argument itself is printed.) - -The following flags are available: - - - `--dry-run`\ - Print on standard error a description of what packages would be - built or downloaded, without actually performing the operation. - - - `--ignore-unknown`\ - If a non-derivation path does not have a substitute, then silently - ignore it. - - - `--check`\ - This option allows you to check whether a derivation is - deterministic. It rebuilds the specified derivation and checks - whether the result is bitwise-identical with the existing outputs, - printing an error if that’s not the case. The outputs of the - specified derivation must already exist. When used with `-K`, if an - output path is not identical to the corresponding output from the - previous build, the new output path is left in - `/nix/store/name.check.` - -Special exit codes: - - - `100`\ - Generic build failure, the builder process returned with a non-zero - exit code. - - - `101`\ - Build timeout, the build was aborted because it did not complete - within the specified `timeout`. - - - `102`\ - Hash mismatch, the build output was rejected because it does not - match the [`outputHash` attribute of the - derivation](../language/advanced-attributes.md). - - - `104`\ - Not deterministic, the build succeeded in check mode but the - resulting output is not binary reproducible. - -With the `--keep-going` flag it's possible for multiple failures to -occur, in this case the 1xx status codes are or combined using binary -or. - - 1100100 - ^^^^ - |||`- timeout - ||`-- output hash mismatch - |`--- build failure - `---- not deterministic - -## Examples - -This operation is typically used to build [store derivation]s produced by -[`nix-instantiate`](./nix-instantiate.md): - -[store derivation]: ../glossary.md#gloss-store-derivation - -```console -$ nix-store -r $(nix-instantiate ./test.nix) -/nix/store/31axcgrlbfsxzmfff1gyj1bf62hvkby2-aterm-2.3.1 -``` - -This is essentially what [`nix-build`](nix-build.md) does. - -To test whether a previously-built derivation is deterministic: - -```console -$ nix-build '' -A hello --check -K -``` - -Use [`--read-log`](#operation---read-log) to show the stderr and stdout of a build: - -```console -$ nix-store --read-log $(nix-instantiate ./test.nix) -``` - -# Operation `--serve` - -## Synopsis - -`nix-store` `--serve` [`--write`] - -## Description - -The operation `--serve` provides access to the Nix store over stdin and -stdout, and is intended to be used as a means of providing Nix store -access to a restricted ssh user. - -The following flags are available: - - - `--write`\ - Allow the connected client to request the realization of - derivations. In effect, this can be used to make the host act as a - remote builder. - -## Examples - -To turn a host into a build server, the `authorized_keys` file can be -used to provide build access to a given SSH public key: - -```console -$ cat <>/root/.ssh/authorized_keys -command="nice -n20 nix-store --serve --write" ssh-rsa AAAAB3NzaC1yc2EAAAA... -EOF -``` - -# Operation `--gc` - -## Synopsis - -`nix-store` `--gc` [`--print-roots` | `--print-live` | `--print-dead`] [`--max-freed` *bytes*] - -## Description - -Without additional flags, the operation `--gc` performs a garbage -collection on the Nix store. That is, all paths in the Nix store not -reachable via file system references from a set of “roots”, are deleted. - -The following suboperations may be specified: - - - `--print-roots`\ - This operation prints on standard output the set of roots used by - the garbage collector. - - - `--print-live`\ - This operation prints on standard output the set of “live” store - paths, which are all the store paths reachable from the roots. Live - paths should never be deleted, since that would break consistency — - it would become possible that applications are installed that - reference things that are no longer present in the store. - - - `--print-dead`\ - This operation prints out on standard output the set of “dead” store - paths, which is just the opposite of the set of live paths: any path - in the store that is not live (with respect to the roots) is dead. - -By default, all unreachable paths are deleted. The following options -control what gets deleted and in what order: - - - `--max-freed` *bytes*\ - Keep deleting paths until at least *bytes* bytes have been deleted, - then stop. The argument *bytes* can be followed by the - multiplicative suffix `K`, `M`, `G` or `T`, denoting KiB, MiB, GiB - or TiB units. - -The behaviour of the collector is also influenced by the -`keep-outputs` and `keep-derivations` settings in the Nix -configuration file. - -By default, the collector prints the total number of freed bytes when it -finishes (or when it is interrupted). With `--print-dead`, it prints the -number of bytes that would be freed. - -## Examples - -To delete all unreachable paths, just do: - -```console -$ nix-store --gc -deleting `/nix/store/kq82idx6g0nyzsp2s14gfsc38npai7lf-cairo-1.0.4.tar.gz.drv' -... -8825586 bytes freed (8.42 MiB) -``` - -To delete at least 100 MiBs of unreachable paths: - -```console -$ nix-store --gc --max-freed $((100 * 1024 * 1024)) -``` - -# Operation `--delete` - -## Synopsis - -`nix-store` `--delete` [`--ignore-liveness`] *paths…* - -## Description - -The operation `--delete` deletes the store paths *paths* from the Nix -store, but only if it is safe to do so; that is, when the path is not -reachable from a root of the garbage collector. This means that you can -only delete paths that would also be deleted by `nix-store --gc`. Thus, -`--delete` is a more targeted version of `--gc`. - -With the option `--ignore-liveness`, reachability from the roots is -ignored. However, the path still won’t be deleted if there are other -paths in the store that refer to it (i.e., depend on it). - -## Example - -```console -$ nix-store --delete /nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4 -0 bytes freed (0.00 MiB) -error: cannot delete path `/nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4' since it is still alive -``` - -# Operation `--query` - -## Synopsis - -`nix-store` {`--query` | `-q`} - {`--outputs` | `--requisites` | `-R` | `--references` | - `--referrers` | `--referrers-closure` | `--deriver` | `-d` | - `--graph` | `--tree` | `--binding` *name* | `-b` *name* | `--hash` | - `--size` | `--roots`} - [`--use-output`] [`-u`] [`--force-realise`] [`-f`] - *paths…* - -## Description - -The operation `--query` displays information about [store path]s. -The queries are described below. At most one query can be -specified. The default query is `--outputs`. - -The paths *paths* may also be symlinks from outside of the Nix store, to -the Nix store. In that case, the query is applied to the target of the -symlink. - -## Common query options - - - `--use-output`; `-u`\ - For each argument to the query that is a [store derivation], apply the - query to the output path of the derivation instead. - - - `--force-realise`; `-f`\ - Realise each argument to the query first (see [`nix-store - --realise`](#operation---realise)). - -## Queries - - - `--outputs`\ - Prints out the [output path]s of the store - derivations *paths*. These are the paths that will be produced when - the derivation is built. - - - `--requisites`; `-R`\ - Prints out the [closure] of the given *paths*. - - This query has one option: - - - `--include-outputs` - Also include the existing output paths of [store derivation]s, - and their closures. - - This query can be used to implement various kinds of deployment. A - *source deployment* is obtained by distributing the closure of a - store derivation. A *binary deployment* is obtained by distributing - the closure of an output path. A *cache deployment* (combined - source/binary deployment, including binaries of build-time-only - dependencies) is obtained by distributing the closure of a store - derivation and specifying the option `--include-outputs`. - - - `--references`\ - Prints the set of [references]s of the store paths - *paths*, that is, their immediate dependencies. (For *all* - dependencies, use `--requisites`.) - - [reference]: ../glossary.md#gloss-reference - - - `--referrers`\ - Prints the set of *referrers* of the store paths *paths*, that is, - the store paths currently existing in the Nix store that refer to - one of *paths*. Note that contrary to the references, the set of - referrers is not constant; it can change as store paths are added or - removed. - - - `--referrers-closure`\ - Prints the closure of the set of store paths *paths* under the - referrers relation; that is, all store paths that directly or - indirectly refer to one of *paths*. These are all the path currently - in the Nix store that are dependent on *paths*. - - - `--deriver`; `-d`\ - Prints the [deriver] of the store paths *paths*. If - the path has no deriver (e.g., if it is a source file), or if the - deriver is not known (e.g., in the case of a binary-only - deployment), the string `unknown-deriver` is printed. - - [deriver]: ../glossary.md#gloss-deriver - - - `--graph`\ - Prints the references graph of the store paths *paths* in the format - of the `dot` tool of AT\&T's [Graphviz - package](http://www.graphviz.org/). This can be used to visualise - dependency graphs. To obtain a build-time dependency graph, apply - this to a store derivation. To obtain a runtime dependency graph, - apply it to an output path. - - - `--tree`\ - Prints the references graph of the store paths *paths* as a nested - ASCII tree. References are ordered by descending closure size; this - tends to flatten the tree, making it more readable. The query only - recurses into a store path when it is first encountered; this - prevents a blowup of the tree representation of the graph. - - - `--graphml`\ - Prints the references graph of the store paths *paths* in the - [GraphML](http://graphml.graphdrawing.org/) file format. This can be - used to visualise dependency graphs. To obtain a build-time - dependency graph, apply this to a [store derivation]. To obtain a - runtime dependency graph, apply it to an output path. - - - `--binding` *name*; `-b` *name*\ - Prints the value of the attribute *name* (i.e., environment - variable) of the [store derivation]s *paths*. It is an error for a - derivation to not have the specified attribute. - - - `--hash`\ - Prints the SHA-256 hash of the contents of the store paths *paths* - (that is, the hash of the output of `nix-store --dump` on the given - paths). Since the hash is stored in the Nix database, this is a fast - operation. - - - `--size`\ - Prints the size in bytes of the contents of the store paths *paths* - — to be precise, the size of the output of `nix-store --dump` on - the given paths. Note that the actual disk space required by the - store paths may be higher, especially on filesystems with large - cluster sizes. - - - `--roots`\ - Prints the garbage collector roots that point, directly or - indirectly, at the store paths *paths*. - -## Examples - -Print the closure (runtime dependencies) of the `svn` program in the -current user environment: - -```console -$ nix-store -qR $(which svn) -/nix/store/5mbglq5ldqld8sj57273aljwkfvj22mc-subversion-1.1.4 -/nix/store/9lz9yc6zgmc0vlqmn2ipcpkjlmbi51vv-glibc-2.3.4 -... -``` - -Print the build-time dependencies of `svn`: - -```console -$ nix-store -qR $(nix-store -qd $(which svn)) -/nix/store/02iizgn86m42q905rddvg4ja975bk2i4-grep-2.5.1.tar.bz2.drv -/nix/store/07a2bzxmzwz5hp58nf03pahrv2ygwgs3-gcc-wrapper.sh -/nix/store/0ma7c9wsbaxahwwl04gbw3fcd806ski4-glibc-2.3.4.drv -... lots of other paths ... -``` - -The difference with the previous example is that we ask the closure of -the derivation (`-qd`), not the closure of the output path that contains -`svn`. - -Show the build-time dependencies as a tree: - -```console -$ nix-store -q --tree $(nix-store -qd $(which svn)) -/nix/store/7i5082kfb6yjbqdbiwdhhza0am2xvh6c-subversion-1.1.4.drv -+---/nix/store/d8afh10z72n8l1cr5w42366abiblgn54-builder.sh -+---/nix/store/fmzxmpjx2lh849ph0l36snfj9zdibw67-bash-3.0.drv -| +---/nix/store/570hmhmx3v57605cqg9yfvvyh0nnb8k8-bash -| +---/nix/store/p3srsbd8dx44v2pg6nbnszab5mcwx03v-builder.sh -... -``` - -Show all paths that depend on the same OpenSSL library as `svn`: - -```console -$ nix-store -q --referrers $(nix-store -q --binding openssl $(nix-store -qd $(which svn))) -/nix/store/23ny9l9wixx21632y2wi4p585qhva1q8-sylpheed-1.0.0 -/nix/store/5mbglq5ldqld8sj57273aljwkfvj22mc-subversion-1.1.4 -/nix/store/dpmvp969yhdqs7lm2r1a3gng7pyq6vy4-subversion-1.1.3 -/nix/store/l51240xqsgg8a7yrbqdx1rfzyv6l26fx-lynx-2.8.5 -``` - -Show all paths that directly or indirectly depend on the Glibc (C -library) used by `svn`: - -```console -$ nix-store -q --referrers-closure $(ldd $(which svn) | grep /libc.so | awk '{print $3}') -/nix/store/034a6h4vpz9kds5r6kzb9lhh81mscw43-libgnomeprintui-2.8.2 -/nix/store/15l3yi0d45prm7a82pcrknxdh6nzmxza-gawk-3.1.4 -... -``` - -Note that `ldd` is a command that prints out the dynamic libraries used -by an ELF executable. - -Make a picture of the runtime dependency graph of the current user -environment: - -```console -$ nix-store -q --graph ~/.nix-profile | dot -Tps > graph.ps -$ gv graph.ps -``` - -Show every garbage collector root that points to a store path that -depends on `svn`: - -```console -$ nix-store -q --roots $(which svn) -/nix/var/nix/profiles/default-81-link -/nix/var/nix/profiles/default-82-link -/nix/var/nix/profiles/per-user/eelco/profile-97-link -``` - -# Operation `--add` - -## Synopsis - -`nix-store` `--add` *paths…* - -## Description - -The operation `--add` adds the specified paths to the Nix store. It -prints the resulting paths in the Nix store on standard output. - -## Example - -```console -$ nix-store --add ./foo.c -/nix/store/m7lrha58ph6rcnv109yzx1nk1cj7k7zf-foo.c -``` - -# Operation `--add-fixed` - -## Synopsis - -`nix-store` `--add-fixed` [`--recursive`] *algorithm* *paths…* - -## Description - -The operation `--add-fixed` adds the specified paths to the Nix store. -Unlike `--add` paths are registered using the specified hashing -algorithm, resulting in the same output path as a fixed-output -derivation. This can be used for sources that are not available from a -public url or broke since the download expression was written. - -This operation has the following options: - - - `--recursive`\ - Use recursive instead of flat hashing mode, used when adding - directories to the store. - -## Example - -```console -$ nix-store --add-fixed sha256 ./hello-2.10.tar.gz -/nix/store/3x7dwzq014bblazs7kq20p9hyzz0qh8g-hello-2.10.tar.gz -``` - -# Operation `--verify` - -## Synopsis - -`nix-store` `--verify` [`--check-contents`] [`--repair`] - -## Description - -The operation `--verify` verifies the internal consistency of the Nix -database, and the consistency between the Nix database and the Nix -store. Any inconsistencies encountered are automatically repaired. -Inconsistencies are generally the result of the Nix store or database -being modified by non-Nix tools, or of bugs in Nix itself. - -This operation has the following options: - - - `--check-contents`\ - Checks that the contents of every valid store path has not been - altered by computing a SHA-256 hash of the contents and comparing it - with the hash stored in the Nix database at build time. Paths that - have been modified are printed out. For large stores, - `--check-contents` is obviously quite slow. - - - `--repair`\ - If any valid path is missing from the store, or (if - `--check-contents` is given) the contents of a valid path has been - modified, then try to repair the path by redownloading it. See - `nix-store --repair-path` for details. - -# Operation `--verify-path` - -## Synopsis - -`nix-store` `--verify-path` *paths…* - -## Description - -The operation `--verify-path` compares the contents of the given store -paths to their cryptographic hashes stored in Nix’s database. For every -changed path, it prints a warning message. The exit status is 0 if no -path has changed, and 1 otherwise. - -## Example - -To verify the integrity of the `svn` command and all its dependencies: - -```console -$ nix-store --verify-path $(nix-store -qR $(which svn)) -``` - -# Operation `--repair-path` - -## Synopsis - -`nix-store` `--repair-path` *paths…* - -## Description - -The operation `--repair-path` attempts to “repair” the specified paths -by redownloading them using the available substituters. If no -substitutes are available, then repair is not possible. - -> **Warning** -> -> During repair, there is a very small time window during which the old -> path (if it exists) is moved out of the way and replaced with the new -> path. If repair is interrupted in between, then the system may be left -> in a broken state (e.g., if the path contains a critical system -> component like the GNU C Library). - -## Example - -```console -$ nix-store --verify-path /nix/store/dj7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13 -path `/nix/store/dj7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13' was modified! - expected hash `2db57715ae90b7e31ff1f2ecb8c12ec1cc43da920efcbe3b22763f36a1861588', - got `481c5aa5483ebc97c20457bb8bca24deea56550d3985cda0027f67fe54b808e4' - -$ nix-store --repair-path /nix/store/dj7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13 -fetching path `/nix/store/d7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13'... -… -``` - -# Operation `--dump` - -## Synopsis - -`nix-store` `--dump` *path* - -## Description - -The operation `--dump` produces a NAR (Nix ARchive) file containing the -contents of the file system tree rooted at *path*. The archive is -written to standard output. - -A NAR archive is like a TAR or Zip archive, but it contains only the -information that Nix considers important. For instance, timestamps are -elided because all files in the Nix store have their timestamp set to 1 -anyway. Likewise, all permissions are left out except for the execute -bit, because all files in the Nix store have 444 or 555 permission. - -Also, a NAR archive is *canonical*, meaning that “equal” paths always -produce the same NAR archive. For instance, directory entries are -always sorted so that the actual on-disk order doesn’t influence the -result. This means that the cryptographic hash of a NAR dump of a -path is usable as a fingerprint of the contents of the path. Indeed, -the hashes of store paths stored in Nix’s database (see `nix-store -q ---hash`) are SHA-256 hashes of the NAR dump of each store path. - -NAR archives support filenames of unlimited length and 64-bit file -sizes. They can contain regular files, directories, and symbolic links, -but not other types of files (such as device nodes). - -A Nix archive can be unpacked using `nix-store ---restore`. - -# Operation `--restore` - -## Synopsis - -`nix-store` `--restore` *path* - -## Description - -The operation `--restore` unpacks a NAR archive to *path*, which must -not already exist. The archive is read from standard input. - -# Operation `--export` - -## Synopsis - -`nix-store` `--export` *paths…* - -## Description - -The operation `--export` writes a serialisation of the specified store -paths to standard output in a format that can be imported into another -Nix store with `nix-store --import`. This is like `nix-store ---dump`, except that the NAR archive produced by that command doesn’t -contain the necessary meta-information to allow it to be imported into -another Nix store (namely, the set of references of the path). - -This command does not produce a *closure* of the specified paths, so if -a store path references other store paths that are missing in the target -Nix store, the import will fail. To copy a whole closure, do something -like: - -```console -$ nix-store --export $(nix-store -qR paths) > out -``` - -To import the whole closure again, run: - -```console -$ nix-store --import < out -``` - -# Operation `--import` - -## Synopsis - -`nix-store` `--import` - -## Description - -The operation `--import` reads a serialisation of a set of store paths -produced by `nix-store --export` from standard input and adds those -store paths to the Nix store. Paths that already exist in the Nix store -are ignored. If a path refers to another path that doesn’t exist in the -Nix store, the import fails. - -# Operation `--optimise` - -## Synopsis - -`nix-store` `--optimise` - -## Description - -The operation `--optimise` reduces Nix store disk space usage by finding -identical files in the store and hard-linking them to each other. It -typically reduces the size of the store by something like 25-35%. Only -regular files and symlinks are hard-linked in this manner. Files are -considered identical when they have the same NAR archive serialisation: -that is, regular files must have the same contents and permission -(executable or non-executable), and symlinks must have the same -contents. - -After completion, or when the command is interrupted, a report on the -achieved savings is printed on standard error. - -Use `-vv` or `-vvv` to get some progress indication. - -## Example - -```console -$ nix-store --optimise -hashing files in `/nix/store/qhqx7l2f1kmwihc9bnxs7rc159hsxnf3-gcc-4.1.1' -... -541838819 bytes (516.74 MiB) freed by hard-linking 54143 files; -there are 114486 files with equal contents out of 215894 files in total -``` - -# Operation `--read-log` - -## Synopsis - -`nix-store` {`--read-log` | `-l`} *paths…* - -## Description - -The operation `--read-log` prints the build log of the specified store -paths on standard output. The build log is whatever the builder of a -derivation wrote to standard output and standard error. If a store path -is not a derivation, the deriver of the store path is used. - -Build logs are kept in `/nix/var/log/nix/drvs`. However, there is no -guarantee that a build log is available for any particular store path. -For instance, if the path was downloaded as a pre-built binary through a -substitute, then the log is unavailable. - -## Example - -```console -$ nix-store -l $(which ktorrent) -building /nix/store/dhc73pvzpnzxhdgpimsd9sw39di66ph1-ktorrent-2.2.1 -unpacking sources -unpacking source archive /nix/store/p8n1jpqs27mgkjw07pb5269717nzf5f8-ktorrent-2.2.1.tar.gz -ktorrent-2.2.1/ -ktorrent-2.2.1/NEWS -... -``` - -# Operation `--dump-db` - -## Synopsis - -`nix-store` `--dump-db` [*paths…*] - -## Description - -The operation `--dump-db` writes a dump of the Nix database to standard -output. It can be loaded into an empty Nix store using `--load-db`. This -is useful for making backups and when migrating to different database -schemas. - -By default, `--dump-db` will dump the entire Nix database. When one or -more store paths is passed, only the subset of the Nix database for -those store paths is dumped. As with `--export`, the user is responsible -for passing all the store paths for a closure. See `--export` for an -example. - -# Operation `--load-db` - -## Synopsis - -`nix-store` `--load-db` - -## Description - -The operation `--load-db` reads a dump of the Nix database created by -`--dump-db` from standard input and loads it into the Nix database. - -# Operation `--print-env` - -## Synopsis - -`nix-store` `--print-env` *drvpath* - -## Description - -The operation `--print-env` prints out the environment of a derivation -in a format that can be evaluated by a shell. The command line arguments -of the builder are placed in the variable `_args`. - -## Example - -```console -$ nix-store --print-env $(nix-instantiate '' -A firefox) -… -export src; src='/nix/store/plpj7qrwcz94z2psh6fchsi7s8yihc7k-firefox-12.0.source.tar.bz2' -export stdenv; stdenv='/nix/store/7c8asx3yfrg5dg1gzhzyq2236zfgibnn-stdenv' -export system; system='x86_64-linux' -export _args; _args='-e /nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25c-default-builder.sh' -``` - -# Operation `--generate-binary-cache-key` - -## Synopsis - -`nix-store` `--generate-binary-cache-key` *key-name* *secret-key-file* *public-key-file* - -## Description - -This command generates an [Ed25519 key pair](http://ed25519.cr.yp.to/) -that can be used to create a signed binary cache. It takes three -mandatory parameters: - -1. A key name, such as `cache.example.org-1`, that is used to look up - keys on the client when it verifies signatures. It can be anything, - but it’s suggested to use the host name of your cache (e.g. - `cache.example.org`) with a suffix denoting the number of the key - (to be incremented every time you need to revoke a key). - -2. The file name where the secret key is to be stored. - -3. The file name where the public key is to be stored. +`nix-store` takes exactly one *operation* flag which indicates the subcommand to be performed. The following operations are available: + +- [`--realise`](./nix-store/realise.md) +- [`--serve`](./nix-store/serve.md) +- [`--gc`](./nix-store/gc.md) +- [`--delete`](./nix-store/delete.md) +- [`--query`](./nix-store/query.md) +- [`--add`](./nix-store/add.md) +- [`--add-fixed`](./nix-store/add-fixed.md) +- [`--verify`](./nix-store/verify.md) +- [`--verify-path`](./nix-store/verify-path.md) +- [`--repair-path`](./nix-store/repair-path.md) +- [`--dump`](./nix-store/dump.md) +- [`--restore`](./nix-store/restore.md) +- [`--export`](./nix-store/export.md) +- [`--import`](./nix-store/import.md) +- [`--optimise`](./nix-store/optimise.md) +- [`--read-log`](./nix-store/read-log.md) +- [`--dump-db`](./nix-store/dump-db.md) +- [`--load-db`](./nix-store/load-db.md) +- [`--print-env`](./nix-store/print-env.md) +- [`--generate-binary-cache-key`](./nix-store/generate-binary-cache-key.md) + +These pages can be viewed offline: + +- `man nix-store-`. + + Example: `man nix-store-realise` + +- `nix-store --help --` + + Example: `nix-store --help --realise` diff --git a/doc/manual/src/command-ref/nix-store/add-fixed.md b/doc/manual/src/command-ref/nix-store/add-fixed.md new file mode 100644 index 000000000..d25db091c --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/add-fixed.md @@ -0,0 +1,35 @@ +# Name + +`nix-store --add-fixed` - add paths to store using given hashing algorithm + +## Synopsis + +`nix-store` `--add-fixed` [`--recursive`] *algorithm* *paths…* + +## Description + +The operation `--add-fixed` adds the specified paths to the Nix store. +Unlike `--add` paths are registered using the specified hashing +algorithm, resulting in the same output path as a fixed-output +derivation. This can be used for sources that are not available from a +public url or broke since the download expression was written. + +This operation has the following options: + + - `--recursive`\ + Use recursive instead of flat hashing mode, used when adding + directories to the store. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} + +## Example + +```console +$ nix-store --add-fixed sha256 ./hello-2.10.tar.gz +/nix/store/3x7dwzq014bblazs7kq20p9hyzz0qh8g-hello-2.10.tar.gz +``` + diff --git a/doc/manual/src/command-ref/nix-store/add.md b/doc/manual/src/command-ref/nix-store/add.md new file mode 100644 index 000000000..87d504cd3 --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/add.md @@ -0,0 +1,25 @@ +# Name + +`nix-store --add` - add paths to Nix store + +# Synopsis + +`nix-store` `--add` *paths…* + +# Description + +The operation `--add` adds the specified paths to the Nix store. It +prints the resulting paths in the Nix store on standard output. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} + +# Example + +```console +$ nix-store --add ./foo.c +/nix/store/m7lrha58ph6rcnv109yzx1nk1cj7k7zf-foo.c +``` diff --git a/doc/manual/src/command-ref/nix-store/delete.md b/doc/manual/src/command-ref/nix-store/delete.md new file mode 100644 index 000000000..550c5ea29 --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/delete.md @@ -0,0 +1,33 @@ +# Name + +`nix-store --delete` - delete store paths + +# Synopsis + +`nix-store` `--delete` [`--ignore-liveness`] *paths…* + +# Description + +The operation `--delete` deletes the store paths *paths* from the Nix +store, but only if it is safe to do so; that is, when the path is not +reachable from a root of the garbage collector. This means that you can +only delete paths that would also be deleted by `nix-store --gc`. Thus, +`--delete` is a more targeted version of `--gc`. + +With the option `--ignore-liveness`, reachability from the roots is +ignored. However, the path still won’t be deleted if there are other +paths in the store that refer to it (i.e., depend on it). + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} + +# Example + +```console +$ nix-store --delete /nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4 +0 bytes freed (0.00 MiB) +error: cannot delete path `/nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4' since it is still alive +``` diff --git a/doc/manual/src/command-ref/nix-store/dump-db.md b/doc/manual/src/command-ref/nix-store/dump-db.md new file mode 100644 index 000000000..b2c77ced0 --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/dump-db.md @@ -0,0 +1,26 @@ +# Name + +`nix-store --dump-db` - export Nix database + +# Synopsis + +`nix-store` `--dump-db` [*paths…*] + +# Description + +The operation `--dump-db` writes a dump of the Nix database to standard +output. It can be loaded into an empty Nix store using `--load-db`. This +is useful for making backups and when migrating to different database +schemas. + +By default, `--dump-db` will dump the entire Nix database. When one or +more store paths is passed, only the subset of the Nix database for +those store paths is dumped. As with `--export`, the user is responsible +for passing all the store paths for a closure. See `--export` for an +example. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/dump.md b/doc/manual/src/command-ref/nix-store/dump.md new file mode 100644 index 000000000..62656d599 --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/dump.md @@ -0,0 +1,40 @@ +# Name + +`nix-store --dump` - write a single path to a Nix Archive + +## Synopsis + +`nix-store` `--dump` *path* + +## Description + +The operation `--dump` produces a NAR (Nix ARchive) file containing the +contents of the file system tree rooted at *path*. The archive is +written to standard output. + +A NAR archive is like a TAR or Zip archive, but it contains only the +information that Nix considers important. For instance, timestamps are +elided because all files in the Nix store have their timestamp set to 0 +anyway. Likewise, all permissions are left out except for the execute +bit, because all files in the Nix store have 444 or 555 permission. + +Also, a NAR archive is *canonical*, meaning that “equal” paths always +produce the same NAR archive. For instance, directory entries are +always sorted so that the actual on-disk order doesn’t influence the +result. This means that the cryptographic hash of a NAR dump of a +path is usable as a fingerprint of the contents of the path. Indeed, +the hashes of store paths stored in Nix’s database (see `nix-store -q +--hash`) are SHA-256 hashes of the NAR dump of each store path. + +NAR archives support filenames of unlimited length and 64-bit file +sizes. They can contain regular files, directories, and symbolic links, +but not other types of files (such as device nodes). + +A Nix archive can be unpacked using `nix-store +--restore`. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/export.md b/doc/manual/src/command-ref/nix-store/export.md new file mode 100644 index 000000000..aeea38636 --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/export.md @@ -0,0 +1,41 @@ +# Name + +`nix-store --export` - export store paths to a Nix Archive + +## Synopsis + +`nix-store` `--export` *paths…* + +## Description + +The operation `--export` writes a serialisation of the specified store +paths to standard output in a format that can be imported into another +Nix store with `nix-store --import`. This is like `nix-store +--dump`, except that the NAR archive produced by that command doesn’t +contain the necessary meta-information to allow it to be imported into +another Nix store (namely, the set of references of the path). + +This command does not produce a *closure* of the specified paths, so if +a store path references other store paths that are missing in the target +Nix store, the import will fail. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} + +# Examples + +To copy a whole closure, do something +like: + +```console +$ nix-store --export $(nix-store -qR paths) > out +``` + +To import the whole closure again, run: + +```console +$ nix-store --import < out +``` diff --git a/doc/manual/src/command-ref/nix-store/gc.md b/doc/manual/src/command-ref/nix-store/gc.md new file mode 100644 index 000000000..7be0d559a --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/gc.md @@ -0,0 +1,72 @@ +# Name + +`nix-store --gc` - run garbage collection + +# Synopsis + +`nix-store` `--gc` [`--print-roots` | `--print-live` | `--print-dead`] [`--max-freed` *bytes*] + +# Description + +Without additional flags, the operation `--gc` performs a garbage +collection on the Nix store. That is, all paths in the Nix store not +reachable via file system references from a set of “roots”, are deleted. + +The following suboperations may be specified: + + - `--print-roots`\ + This operation prints on standard output the set of roots used by + the garbage collector. + + - `--print-live`\ + This operation prints on standard output the set of “live” store + paths, which are all the store paths reachable from the roots. Live + paths should never be deleted, since that would break consistency — + it would become possible that applications are installed that + reference things that are no longer present in the store. + + - `--print-dead`\ + This operation prints out on standard output the set of “dead” store + paths, which is just the opposite of the set of live paths: any path + in the store that is not live (with respect to the roots) is dead. + +By default, all unreachable paths are deleted. The following options +control what gets deleted and in what order: + + - `--max-freed` *bytes*\ + Keep deleting paths until at least *bytes* bytes have been deleted, + then stop. The argument *bytes* can be followed by the + multiplicative suffix `K`, `M`, `G` or `T`, denoting KiB, MiB, GiB + or TiB units. + +The behaviour of the collector is also influenced by the +`keep-outputs` and `keep-derivations` settings in the Nix +configuration file. + +By default, the collector prints the total number of freed bytes when it +finishes (or when it is interrupted). With `--print-dead`, it prints the +number of bytes that would be freed. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} + +# Examples + +To delete all unreachable paths, just do: + +```console +$ nix-store --gc +deleting `/nix/store/kq82idx6g0nyzsp2s14gfsc38npai7lf-cairo-1.0.4.tar.gz.drv' +... +8825586 bytes freed (8.42 MiB) +``` + +To delete at least 100 MiBs of unreachable paths: + +```console +$ nix-store --gc --max-freed $((100 * 1024 * 1024)) +``` + diff --git a/doc/manual/src/command-ref/nix-store/generate-binary-cache-key.md b/doc/manual/src/command-ref/nix-store/generate-binary-cache-key.md new file mode 100644 index 000000000..8085d877b --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/generate-binary-cache-key.md @@ -0,0 +1,29 @@ +# Name + +`nix-store --generate-binary-cache-key` - generate key pair to use for a binary cache + +## Synopsis + +`nix-store` `--generate-binary-cache-key` *key-name* *secret-key-file* *public-key-file* + +## Description + +This command generates an [Ed25519 key pair](http://ed25519.cr.yp.to/) +that can be used to create a signed binary cache. It takes three +mandatory parameters: + +1. A key name, such as `cache.example.org-1`, that is used to look up + keys on the client when it verifies signatures. It can be anything, + but it’s suggested to use the host name of your cache (e.g. + `cache.example.org`) with a suffix denoting the number of the key + (to be incremented every time you need to revoke a key). + +2. The file name where the secret key is to be stored. + +3. The file name where the public key is to be stored. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/import.md b/doc/manual/src/command-ref/nix-store/import.md new file mode 100644 index 000000000..2711316a7 --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/import.md @@ -0,0 +1,21 @@ +# Name + +`nix-store --import` - import Nix Archive into the store + +# Synopsis + +`nix-store` `--import` + +# Description + +The operation `--import` reads a serialisation of a set of store paths +produced by `nix-store --export` from standard input and adds those +store paths to the Nix store. Paths that already exist in the Nix store +are ignored. If a path refers to another path that doesn’t exist in the +Nix store, the import fails. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/load-db.md b/doc/manual/src/command-ref/nix-store/load-db.md new file mode 100644 index 000000000..e2f438ed6 --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/load-db.md @@ -0,0 +1,18 @@ +# Name + +`nix-store --load-db` - import Nix database + +# Synopsis + +`nix-store` `--load-db` + +# Description + +The operation `--load-db` reads a dump of the Nix database created by +`--dump-db` from standard input and loads it into the Nix database. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/opt-common.md b/doc/manual/src/command-ref/nix-store/opt-common.md new file mode 100644 index 000000000..bf6566555 --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/opt-common.md @@ -0,0 +1,36 @@ +# Options + +The following options are allowed for all `nix-store` operations, but may not always have an effect. + +- [`--add-root`](#opt-add-root) *path* + + Causes the result of a realisation (`--realise` and + `--force-realise`) to be registered as a root of the garbage + collector. *path* will be created as a symlink to the resulting + store path. In addition, a uniquely named symlink to *path* will + be created in `/nix/var/nix/gcroots/auto/`. For instance, + + ```console + $ nix-store --add-root /home/eelco/bla/result -r ... + + $ ls -l /nix/var/nix/gcroots/auto + lrwxrwxrwx 1 ... 2005-03-13 21:10 dn54lcypm8f8... -> /home/eelco/bla/result + + $ ls -l /home/eelco/bla/result + lrwxrwxrwx 1 ... 2005-03-13 21:10 /home/eelco/bla/result -> /nix/store/1r11343n6qd4...-f-spot-0.0.10 + ``` + + Thus, when `/home/eelco/bla/result` is removed, the GC root in the + `auto` directory becomes a dangling symlink and will be ignored by + the collector. + + > **Warning** + > + > Note that it is not possible to move or rename GC roots, since + > the symlink in the `auto` directory will still point to the old + > location. + + If there are multiple results, then multiple symlinks will be + created by sequentially numbering symlinks beyond the first one + (e.g., `foo`, `foo-2`, `foo-3`, and so on). + diff --git a/doc/manual/src/command-ref/nix-store/optimise.md b/doc/manual/src/command-ref/nix-store/optimise.md new file mode 100644 index 000000000..dc392aeb8 --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/optimise.md @@ -0,0 +1,40 @@ +# Name + +`nix-store --optimise` - reduce disk space usage + +## Synopsis + +`nix-store` `--optimise` + +## Description + +The operation `--optimise` reduces Nix store disk space usage by finding +identical files in the store and hard-linking them to each other. It +typically reduces the size of the store by something like 25-35%. Only +regular files and symlinks are hard-linked in this manner. Files are +considered identical when they have the same NAR archive serialisation: +that is, regular files must have the same contents and permission +(executable or non-executable), and symlinks must have the same +contents. + +After completion, or when the command is interrupted, a report on the +achieved savings is printed on standard error. + +Use `-vv` or `-vvv` to get some progress indication. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} + +## Example + +```console +$ nix-store --optimise +hashing files in `/nix/store/qhqx7l2f1kmwihc9bnxs7rc159hsxnf3-gcc-4.1.1' +... +541838819 bytes (516.74 MiB) freed by hard-linking 54143 files; +there are 114486 files with equal contents out of 215894 files in total +``` + diff --git a/doc/manual/src/command-ref/nix-store/print-env.md b/doc/manual/src/command-ref/nix-store/print-env.md new file mode 100644 index 000000000..bd2084ef6 --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/print-env.md @@ -0,0 +1,31 @@ +# Name + +`nix-store --print-env` - print the build environment of a derivation + +## Synopsis + +`nix-store` `--print-env` *drvpath* + +## Description + +The operation `--print-env` prints out the environment of a derivation +in a format that can be evaluated by a shell. The command line arguments +of the builder are placed in the variable `_args`. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} + +## Example + +```console +$ nix-store --print-env $(nix-instantiate '' -A firefox) +… +export src; src='/nix/store/plpj7qrwcz94z2psh6fchsi7s8yihc7k-firefox-12.0.source.tar.bz2' +export stdenv; stdenv='/nix/store/7c8asx3yfrg5dg1gzhzyq2236zfgibnn-stdenv' +export system; system='x86_64-linux' +export _args; _args='-e /nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25c-default-builder.sh' +``` + diff --git a/doc/manual/src/command-ref/nix-store/query.md b/doc/manual/src/command-ref/nix-store/query.md new file mode 100644 index 000000000..9f7dbd3e8 --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/query.md @@ -0,0 +1,220 @@ +# Name + +`nix-store --query` - display information about store paths + +# Synopsis + +`nix-store` {`--query` | `-q`} + {`--outputs` | `--requisites` | `-R` | `--references` | + `--referrers` | `--referrers-closure` | `--deriver` | `-d` | + `--graph` | `--tree` | `--binding` *name* | `-b` *name* | `--hash` | + `--size` | `--roots`} + [`--use-output`] [`-u`] [`--force-realise`] [`-f`] + *paths…* + +# Description + +The operation `--query` displays various bits of information about the +store paths . The queries are described below. At most one query can be +specified. The default query is `--outputs`. + +The paths *paths* may also be symlinks from outside of the Nix store, to +the Nix store. In that case, the query is applied to the target of the +symlink. + +# Common query options + + - `--use-output`; `-u`\ + For each argument to the query that is a [store derivation], apply the + query to the output path of the derivation instead. + + - `--force-realise`; `-f`\ + Realise each argument to the query first (see [`nix-store --realise`](./realise.md)). + +[store derivation]: @docroot@/glossary.md#gloss-store-derivation + +# Queries + + - `--outputs`\ + Prints out the [output paths] of the store + derivations *paths*. These are the paths that will be produced when + the derivation is built. + + [output paths]: ../../glossary.md#gloss-output-path + + - `--requisites`; `-R`\ + Prints out the [closure] of the store path *paths*. + + [closure]: ../../glossary.md#gloss-closure + + This query has one option: + + - `--include-outputs` + Also include the existing output paths of [store derivation]s, + and their closures. + + This query can be used to implement various kinds of deployment. A + *source deployment* is obtained by distributing the closure of a + store derivation. A *binary deployment* is obtained by distributing + the closure of an output path. A *cache deployment* (combined + source/binary deployment, including binaries of build-time-only + dependencies) is obtained by distributing the closure of a store + derivation and specifying the option `--include-outputs`. + + - `--references`\ + Prints the set of [references] of the store paths + *paths*, that is, their immediate dependencies. (For *all* + dependencies, use `--requisites`.) + + [references]: ../../glossary.md#gloss-reference + + - `--referrers`\ + Prints the set of *referrers* of the store paths *paths*, that is, + the store paths currently existing in the Nix store that refer to + one of *paths*. Note that contrary to the references, the set of + referrers is not constant; it can change as store paths are added or + removed. + + - `--referrers-closure`\ + Prints the closure of the set of store paths *paths* under the + referrers relation; that is, all store paths that directly or + indirectly refer to one of *paths*. These are all the path currently + in the Nix store that are dependent on *paths*. + + - `--deriver`; `-d`\ + Prints the [deriver] of the store paths *paths*. If + the path has no deriver (e.g., if it is a source file), or if the + deriver is not known (e.g., in the case of a binary-only + deployment), the string `unknown-deriver` is printed. + + [deriver]: ../../glossary.md#gloss-deriver + + - `--graph`\ + Prints the references graph of the store paths *paths* in the format + of the `dot` tool of AT\&T's [Graphviz + package](http://www.graphviz.org/). This can be used to visualise + dependency graphs. To obtain a build-time dependency graph, apply + this to a store derivation. To obtain a runtime dependency graph, + apply it to an output path. + + - `--tree`\ + Prints the references graph of the store paths *paths* as a nested + ASCII tree. References are ordered by descending closure size; this + tends to flatten the tree, making it more readable. The query only + recurses into a store path when it is first encountered; this + prevents a blowup of the tree representation of the graph. + + - `--graphml`\ + Prints the references graph of the store paths *paths* in the + [GraphML](http://graphml.graphdrawing.org/) file format. This can be + used to visualise dependency graphs. To obtain a build-time + dependency graph, apply this to a [store derivation]. To obtain a + runtime dependency graph, apply it to an output path. + + - `--binding` *name*; `-b` *name*\ + Prints the value of the attribute *name* (i.e., environment + variable) of the [store derivation]s *paths*. It is an error for a + derivation to not have the specified attribute. + + - `--hash`\ + Prints the SHA-256 hash of the contents of the store paths *paths* + (that is, the hash of the output of `nix-store --dump` on the given + paths). Since the hash is stored in the Nix database, this is a fast + operation. + + - `--size`\ + Prints the size in bytes of the contents of the store paths *paths* + — to be precise, the size of the output of `nix-store --dump` on + the given paths. Note that the actual disk space required by the + store paths may be higher, especially on filesystems with large + cluster sizes. + + - `--roots`\ + Prints the garbage collector roots that point, directly or + indirectly, at the store paths *paths*. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} + +# Examples + +Print the closure (runtime dependencies) of the `svn` program in the +current user environment: + +```console +$ nix-store -qR $(which svn) +/nix/store/5mbglq5ldqld8sj57273aljwkfvj22mc-subversion-1.1.4 +/nix/store/9lz9yc6zgmc0vlqmn2ipcpkjlmbi51vv-glibc-2.3.4 +... +``` + +Print the build-time dependencies of `svn`: + +```console +$ nix-store -qR $(nix-store -qd $(which svn)) +/nix/store/02iizgn86m42q905rddvg4ja975bk2i4-grep-2.5.1.tar.bz2.drv +/nix/store/07a2bzxmzwz5hp58nf03pahrv2ygwgs3-gcc-wrapper.sh +/nix/store/0ma7c9wsbaxahwwl04gbw3fcd806ski4-glibc-2.3.4.drv +... lots of other paths ... +``` + +The difference with the previous example is that we ask the closure of +the derivation (`-qd`), not the closure of the output path that contains +`svn`. + +Show the build-time dependencies as a tree: + +```console +$ nix-store -q --tree $(nix-store -qd $(which svn)) +/nix/store/7i5082kfb6yjbqdbiwdhhza0am2xvh6c-subversion-1.1.4.drv ++---/nix/store/d8afh10z72n8l1cr5w42366abiblgn54-builder.sh ++---/nix/store/fmzxmpjx2lh849ph0l36snfj9zdibw67-bash-3.0.drv +| +---/nix/store/570hmhmx3v57605cqg9yfvvyh0nnb8k8-bash +| +---/nix/store/p3srsbd8dx44v2pg6nbnszab5mcwx03v-builder.sh +... +``` + +Show all paths that depend on the same OpenSSL library as `svn`: + +```console +$ nix-store -q --referrers $(nix-store -q --binding openssl $(nix-store -qd $(which svn))) +/nix/store/23ny9l9wixx21632y2wi4p585qhva1q8-sylpheed-1.0.0 +/nix/store/5mbglq5ldqld8sj57273aljwkfvj22mc-subversion-1.1.4 +/nix/store/dpmvp969yhdqs7lm2r1a3gng7pyq6vy4-subversion-1.1.3 +/nix/store/l51240xqsgg8a7yrbqdx1rfzyv6l26fx-lynx-2.8.5 +``` + +Show all paths that directly or indirectly depend on the Glibc (C +library) used by `svn`: + +```console +$ nix-store -q --referrers-closure $(ldd $(which svn) | grep /libc.so | awk '{print $3}') +/nix/store/034a6h4vpz9kds5r6kzb9lhh81mscw43-libgnomeprintui-2.8.2 +/nix/store/15l3yi0d45prm7a82pcrknxdh6nzmxza-gawk-3.1.4 +... +``` + +Note that `ldd` is a command that prints out the dynamic libraries used +by an ELF executable. + +Make a picture of the runtime dependency graph of the current user +environment: + +```console +$ nix-store -q --graph ~/.nix-profile | dot -Tps > graph.ps +$ gv graph.ps +``` + +Show every garbage collector root that points to a store path that +depends on `svn`: + +```console +$ nix-store -q --roots $(which svn) +/nix/var/nix/profiles/default-81-link +/nix/var/nix/profiles/default-82-link +/home/eelco/.local/state/nix/profiles/profile-97-link +``` + diff --git a/doc/manual/src/command-ref/nix-store/read-log.md b/doc/manual/src/command-ref/nix-store/read-log.md new file mode 100644 index 000000000..4a88e9382 --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/read-log.md @@ -0,0 +1,38 @@ +# Name + +`nix-store --read-log` - print build log + +# Synopsis + +`nix-store` {`--read-log` | `-l`} *paths…* + +# Description + +The operation `--read-log` prints the build log of the specified store +paths on standard output. The build log is whatever the builder of a +derivation wrote to standard output and standard error. If a store path +is not a derivation, the deriver of the store path is used. + +Build logs are kept in `/nix/var/log/nix/drvs`. However, there is no +guarantee that a build log is available for any particular store path. +For instance, if the path was downloaded as a pre-built binary through a +substitute, then the log is unavailable. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} + +# Example + +```console +$ nix-store -l $(which ktorrent) +building /nix/store/dhc73pvzpnzxhdgpimsd9sw39di66ph1-ktorrent-2.2.1 +unpacking sources +unpacking source archive /nix/store/p8n1jpqs27mgkjw07pb5269717nzf5f8-ktorrent-2.2.1.tar.gz +ktorrent-2.2.1/ +ktorrent-2.2.1/NEWS +... +``` + diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md new file mode 100644 index 000000000..f61a20100 --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -0,0 +1,118 @@ +# Name + +`nix-store --realise` - realise specified store paths + +# Synopsis + +`nix-store` {`--realise` | `-r`} *paths…* [`--dry-run`] + +# Description + +The operation `--realise` essentially “builds” the specified store +paths. Realisation is a somewhat overloaded term: + + - If the store path is a *derivation*, realisation ensures that the + output paths of the derivation are [valid] (i.e., + the output path and its closure exist in the file system). This + can be done in several ways. First, it is possible that the + outputs are already valid, in which case we are done + immediately. Otherwise, there may be [substitutes] + that produce the outputs (e.g., by downloading them). Finally, the + outputs can be produced by running the build task described + by the derivation. + + - If the store path is not a derivation, realisation ensures that the + specified path is valid (i.e., it and its closure exist in the file + system). If the path is already valid, we are done immediately. + Otherwise, the path and any missing paths in its closure may be + produced through substitutes. If there are no (successful) + substitutes, realisation fails. + +[valid]: @docroot@/glossary.md#gloss-validity +[substitutes]: @docroot@/glossary.md#gloss-substitute + +The output path of each derivation is printed on standard output. (For +non-derivations argument, the argument itself is printed.) + +The following flags are available: + + - `--dry-run`\ + Print on standard error a description of what packages would be + built or downloaded, without actually performing the operation. + + - `--ignore-unknown`\ + If a non-derivation path does not have a substitute, then silently + ignore it. + + - `--check`\ + This option allows you to check whether a derivation is + deterministic. It rebuilds the specified derivation and checks + whether the result is bitwise-identical with the existing outputs, + printing an error if that’s not the case. The outputs of the + specified derivation must already exist. When used with `-K`, if an + output path is not identical to the corresponding output from the + previous build, the new output path is left in + `/nix/store/name.check.` + +Special exit codes: + + - `100`\ + Generic build failure, the builder process returned with a non-zero + exit code. + + - `101`\ + Build timeout, the build was aborted because it did not complete + within the specified `timeout`. + + - `102`\ + Hash mismatch, the build output was rejected because it does not + match the [`outputHash` attribute of the + derivation](@docroot@/language/advanced-attributes.md). + + - `104`\ + Not deterministic, the build succeeded in check mode but the + resulting output is not binary reproducible. + +With the `--keep-going` flag it's possible for multiple failures to +occur, in this case the 1xx status codes are or combined using binary +or. + + 1100100 + ^^^^ + |||`- timeout + ||`-- output hash mismatch + |`--- build failure + `---- not deterministic + + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} + +# Examples + +This operation is typically used to build [store derivation]s produced by +[`nix-instantiate`](@docroot@/command-ref/nix-instantiate.md): + +[store derivation]: @docroot@/glossary.md#gloss-store-derivation + +```console +$ nix-store -r $(nix-instantiate ./test.nix) +/nix/store/31axcgrlbfsxzmfff1gyj1bf62hvkby2-aterm-2.3.1 +``` + +This is essentially what [`nix-build`](@docroot@/command-ref/nix-build.md) does. + +To test whether a previously-built derivation is deterministic: + +```console +$ nix-build '' -A hello --check -K +``` + +Use [`nix-store --read-log`](./read-log.md) to show the stderr and stdout of a build: + +```console +$ nix-store --read-log $(nix-instantiate ./test.nix) +``` diff --git a/doc/manual/src/command-ref/nix-store/repair-path.md b/doc/manual/src/command-ref/nix-store/repair-path.md new file mode 100644 index 000000000..9c3d9f7cd --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/repair-path.md @@ -0,0 +1,35 @@ +# Name + +`nix --repair-path` - re-download path from substituter + +# Synopsis + +`nix-store` `--repair-path` *paths…* + +# Description + +The operation `--repair-path` attempts to “repair” the specified paths +by redownloading them using the available substituters. If no +substitutes are available, then repair is not possible. + +> **Warning** +> +> During repair, there is a very small time window during which the old +> path (if it exists) is moved out of the way and replaced with the new +> path. If repair is interrupted in between, then the system may be left +> in a broken state (e.g., if the path contains a critical system +> component like the GNU C Library). + +# Example + +```console +$ nix-store --verify-path /nix/store/dj7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13 +path `/nix/store/dj7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13' was modified! + expected hash `2db57715ae90b7e31ff1f2ecb8c12ec1cc43da920efcbe3b22763f36a1861588', + got `481c5aa5483ebc97c20457bb8bca24deea56550d3985cda0027f67fe54b808e4' + +$ nix-store --repair-path /nix/store/dj7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13 +fetching path `/nix/store/d7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13'... +… +``` + diff --git a/doc/manual/src/command-ref/nix-store/restore.md b/doc/manual/src/command-ref/nix-store/restore.md new file mode 100644 index 000000000..fcba43df4 --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/restore.md @@ -0,0 +1,18 @@ +# Name + +`nix-store --restore` - extract a Nix archive + +## Synopsis + +`nix-store` `--restore` *path* + +## Description + +The operation `--restore` unpacks a NAR archive to *path*, which must +not already exist. The archive is read from standard input. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/serve.md b/doc/manual/src/command-ref/nix-store/serve.md new file mode 100644 index 000000000..0f90f65ae --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/serve.md @@ -0,0 +1,38 @@ +# Name + +`nix-store --serve` - serve local Nix store over SSH + +# Synopsis + +`nix-store` `--serve` [`--write`] + +# Description + +The operation `--serve` provides access to the Nix store over stdin and +stdout, and is intended to be used as a means of providing Nix store +access to a restricted ssh user. + +The following flags are available: + + - `--write`\ + Allow the connected client to request the realization of + derivations. In effect, this can be used to make the host act as a + remote builder. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} + +# Examples + +To turn a host into a build server, the `authorized_keys` file can be +used to provide build access to a given SSH public key: + +```console +$ cat <>/root/.ssh/authorized_keys +command="nice -n20 nix-store --serve --write" ssh-rsa AAAAB3NzaC1yc2EAAAA... +EOF +``` + diff --git a/doc/manual/src/command-ref/nix-store/verify-path.md b/doc/manual/src/command-ref/nix-store/verify-path.md new file mode 100644 index 000000000..59ffe92a3 --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/verify-path.md @@ -0,0 +1,29 @@ +# Name + +`nix-store --verify-path` - check path contents against Nix database + +## Synopsis + +`nix-store` `--verify-path` *paths…* + +## Description + +The operation `--verify-path` compares the contents of the given store +paths to their cryptographic hashes stored in Nix’s database. For every +changed path, it prints a warning message. The exit status is 0 if no +path has changed, and 1 otherwise. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} + +## Example + +To verify the integrity of the `svn` command and all its dependencies: + +```console +$ nix-store --verify-path $(nix-store -qR $(which svn)) +``` + diff --git a/doc/manual/src/command-ref/nix-store/verify.md b/doc/manual/src/command-ref/nix-store/verify.md new file mode 100644 index 000000000..2695b3361 --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/verify.md @@ -0,0 +1,36 @@ +# Name + +`nix-store --verify` - check Nix database for consistency + +# Synopsis + +`nix-store` `--verify` [`--check-contents`] [`--repair`] + +# Description + +The operation `--verify` verifies the internal consistency of the Nix +database, and the consistency between the Nix database and the Nix +store. Any inconsistencies encountered are automatically repaired. +Inconsistencies are generally the result of the Nix store or database +being modified by non-Nix tools, or of bugs in Nix itself. + +This operation has the following options: + + - `--check-contents`\ + Checks that the contents of every valid store path has not been + altered by computing a SHA-256 hash of the contents and comparing it + with the hash stored in the Nix database at build time. Paths that + have been modified are printed out. For large stores, + `--check-contents` is obviously quite slow. + + - `--repair`\ + If any valid path is missing from the store, or (if + `--check-contents` is given) the contents of a valid path has been + modified, then try to repair the path by redownloading it. See + `nix-store --repair-path` for details. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index e612c416f..a23b87e4e 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -2,13 +2,13 @@ Most Nix commands accept the following command-line options: - - [`--help`]{#opt-help}\ + - [`--help`](#opt-help)\ Prints out a summary of the command syntax and exits. - - [`--version`]{#opt-version}\ + - [`--version`](#opt-version)\ Prints out the Nix version number on standard output and exits. - - [`--verbose`]{#opt-verbose} / `-v`\ + - [`--verbose`](#opt-verbose) / `-v`\ Increases the level of verbosity of diagnostic messages printed on standard error. For each Nix operation, the information printed on standard output is well-defined; any diagnostic information is @@ -37,14 +37,14 @@ Most Nix commands accept the following command-line options: - 5\ “Vomit”: print vast amounts of debug information. - - [`--quiet`]{#opt-quiet}\ + - [`--quiet`](#opt-quiet)\ Decreases the level of verbosity of diagnostic messages printed on standard error. This is the inverse option to `-v` / `--verbose`. This option may be specified repeatedly. See the previous verbosity levels list. - - [`--log-format`]{#opt-log-format} *format*\ + - [`--log-format`](#opt-log-format) *format*\ This option can be used to change the output of the log format, with *format* being one of: @@ -66,14 +66,14 @@ Most Nix commands accept the following command-line options: - bar-with-logs\ Display the raw logs, with the progress bar at the bottom. - - [`--no-build-output`]{#opt-no-build-output} / `-Q`\ + - [`--no-build-output`](#opt-no-build-output) / `-Q`\ By default, output written by builders to standard output and standard error is echoed to the Nix command's standard error. This option suppresses this behaviour. Note that the builder's standard output and error are always written to a log file in `prefix/nix/var/log/nix`. - - [`--max-jobs`]{#opt-max-jobs} / `-j` *number*\ + - [`--max-jobs`](#opt-max-jobs) / `-j` *number*\ Sets the maximum number of build jobs that Nix will perform in parallel to the specified number. Specify `auto` to use the number of CPUs in the system. The default is specified by the `max-jobs` @@ -83,7 +83,7 @@ Most Nix commands accept the following command-line options: Setting it to `0` disallows building on the local machine, which is useful when you want builds to happen only on remote builders. - - [`--cores`]{#opt-cores}\ + - [`--cores`](#opt-cores)\ Sets the value of the `NIX_BUILD_CORES` environment variable in the invocation of builders. Builders can use this variable at their discretion to control the maximum amount of parallelism. For @@ -94,18 +94,18 @@ Most Nix commands accept the following command-line options: means that the builder should use all available CPU cores in the system. - - [`--max-silent-time`]{#opt-max-silent-time}\ + - [`--max-silent-time`](#opt-max-silent-time)\ Sets the maximum number of seconds that a builder can go without producing any data on standard output or standard error. The default is specified by the `max-silent-time` configuration setting. `0` means no time-out. - - [`--timeout`]{#opt-timeout}\ + - [`--timeout`](#opt-timeout)\ Sets the maximum number of seconds that a builder can run. The default is specified by the `timeout` configuration setting. `0` means no timeout. - - [`--keep-going`]{#opt-keep-going} / `-k`\ + - [`--keep-going`](#opt-keep-going) / `-k`\ Keep going in case of failed builds, to the greatest extent possible. That is, if building an input of some derivation fails, Nix will still build the other inputs, but not the derivation @@ -113,13 +113,13 @@ Most Nix commands accept the following command-line options: for builds of substitutes), possibly killing builds in progress (in case of parallel or distributed builds). - - [`--keep-failed`]{#opt-keep-failed} / `-K`\ + - [`--keep-failed`](#opt-keep-failed) / `-K`\ Specifies that in case of a build failure, the temporary directory (usually in `/tmp`) in which the build takes place should not be deleted. The path of the build directory is printed as an informational message. - - [`--fallback`]{#opt-fallback}\ + - [`--fallback`](#opt-fallback)\ Whenever Nix attempts to build a derivation for which substitutes are known for each output path, but realising the output paths through the substitutes fails, fall back on building the derivation. @@ -134,18 +134,18 @@ Most Nix commands accept the following command-line options: failure in obtaining the substitutes to lead to a full build from source (with the related consumption of resources). - - [`--readonly-mode`]{#opt-readonly-mode}\ + - [`--readonly-mode`](#opt-readonly-mode)\ When this option is used, no attempt is made to open the Nix database. Most Nix operations do need database access, so those operations will fail. - - [`--arg`]{#opt-arg} *name* *value*\ + - [`--arg`](#opt-arg) *name* *value*\ This option is accepted by `nix-env`, `nix-instantiate`, `nix-shell` and `nix-build`. When evaluating Nix expressions, the expression evaluator will automatically try to call functions that it encounters. It can automatically call functions for which every argument has a [default - value](../language/constructs.md#functions) (e.g., + value](@docroot@/language/constructs.md#functions) (e.g., `{ argName ? defaultValue }: ...`). With `--arg`, you can also call functions that have arguments without a default value (or override a default value). That is, if the evaluator encounters a @@ -164,26 +164,26 @@ Most Nix commands accept the following command-line options: So if you call this Nix expression (e.g., when you do `nix-env -iA pkgname`), the function will be called automatically using the - value [`builtins.currentSystem`](../language/builtins.md) for + value [`builtins.currentSystem`](@docroot@/language/builtins.md) for the `system` argument. You can override this using `--arg`, e.g., `nix-env -iA pkgname --arg system \"i686-freebsd\"`. (Note that since the argument is a Nix string literal, you have to escape the quotes.) - - [`--argstr`]{#opt-argstr} *name* *value*\ + - [`--argstr`](#opt-argstr) *name* *value*\ This option is like `--arg`, only the value is not a Nix expression but a string. So instead of `--arg system \"i686-linux\"` (the outer quotes are to keep the shell happy) you can say `--argstr system i686-linux`. - - [`--attr`]{#opt-attr} / `-A` *attrPath*\ + - [`--attr`](#opt-attr) / `-A` *attrPath*\ Select an attribute from the top-level Nix expression being evaluated. (`nix-env`, `nix-instantiate`, `nix-build` and `nix-shell` only.) The *attribute path* *attrPath* is a sequence of attribute names separated by dots. For instance, given a top-level Nix expression *e*, the attribute path `xorg.xorgserver` would cause the expression `e.xorg.xorgserver` to be used. See - [`nix-env --install`](nix-env.md#operation---install) for some + [`nix-env --install`](@docroot@/command-ref/nix-env/install.md) for some concrete examples. In addition to attribute names, you can also specify array indices. @@ -191,7 +191,7 @@ Most Nix commands accept the following command-line options: attribute of the fourth element of the array in the `foo` attribute of the top-level expression. - - [`--expr`]{#opt-expr} / `-E`\ + - [`--expr`](#opt-expr) / `-E`\ Interpret the command line arguments as a list of Nix expressions to be parsed and evaluated, rather than as a list of file names of Nix expressions. (`nix-instantiate`, `nix-build` and `nix-shell` only.) @@ -202,17 +202,17 @@ Most Nix commands accept the following command-line options: use, give your expression to the `nix-shell -p` convenience flag instead. - - [`-I`]{#opt-I} *path*\ + - [`-I`](#opt-I) *path*\ Add a path to the Nix expression search path. This option may be given multiple times. See the `NIX_PATH` environment variable for information on the semantics of the Nix search path. Paths added through `-I` take precedence over `NIX_PATH`. - - [`--option`]{#opt-option} *name* *value*\ + - [`--option`](#opt-option) *name* *value*\ Set the Nix configuration option *name* to *value*. This overrides settings in the Nix configuration file (see nix.conf5). - - [`--repair`]{#opt-repair}\ + - [`--repair`](#opt-repair)\ Fix corrupted or missing store paths by redownloading or rebuilding them. Note that this is slow because it requires computing a cryptographic hash of the contents of every path in the closure of diff --git a/doc/manual/src/contributing/experimental-features.md b/doc/manual/src/contributing/experimental-features.md new file mode 100644 index 000000000..f1db22751 --- /dev/null +++ b/doc/manual/src/contributing/experimental-features.md @@ -0,0 +1,91 @@ +This section describes the notion of *experimental features*, and how it fits into the big picture of the development of Nix. + +# What are experimental features? + +Experimental features are considered unstable, which means that they can be changed or removed at any time. +Users must explicitly enable them by toggling the associated [experimental feature flags](@docroot@/command-ref/conf-file.md#conf-experimental-features). +This allows accessing unstable functionality without unwittingly relying on it. + +Experimental feature flags were first introduced in [Nix 2.4](@docroot@/release-notes/rl-2.4.md). +Before that, Nix did have experimental features, but they were not guarded by flags and were merely documented as unstable. +This was a source of confusion and controversy. + +# When should a new feature be marked experimental? + +A change in the Nix codebase should be guarded by an experimental feature flag if it is considered likely to be reverted or adapted in a backwards-incompatible manner after gathering more experience with it in practice. + +Examples: + +- Changes to the Nix language, such as new built-ins, syntactic or semantic changes, etc. +- Changes to the command-line interface + +# Lifecycle of an experimental feature + +Experimental features have to be treated on a case-by-case basis. +However, the standard workflow for an experimental feature is as follows: + +- A new feature is implemented in a *pull request* + - It is guarded by an experimental feature flag that is disabled by default +- The pull request is merged, the *experimental* feature ends up in a release + - Using the feature requires explicitly enabling it, signifying awareness of the potential risks + - Being experimental, the feature can still be changed arbitrarily +- The feature can be *removed* + - The associated experimental feature flag is also removed +- The feature can be declared *stable* + - The associated experimental feature flag is removed + - There should be enough evidence of users having tried the feature, such as feedback, fixed bugs, demonstrations of how it is put to use + - Maintainers must feel confident that: + - The feature is designed and implemented sensibly, that it is fit for purpose + - Potential interactions are well-understood + - Stabilising the feature will not incur an outsized maintenance burden in the future + +The following diagram illustrates the process: + +``` + .------. + | idea | + '------' + | + discussion, design, implementation + | + | .-------. + | | | + v v | + .--------------. review + | pull request | | + '--------------' | + | ^ | | + | | '-------' + .---' '----. + | | + merge user feedback, + | (breaking) changes + | | + '---. .----' + | | + v | + +--------------+ + .---| experimental |----. + | +--------------+ | + | | +decision to stabilise decision against + | keeping the feature + | | + v v + +--------+ +---------+ + | stable | | removed | + +--------+ +---------+ +``` + +# Relation to the RFC process + +Experimental features and [RFCs](https://github.com/NixOS/rfcs/) both allow approaching substantial changes while minimizing the risk. +However they serve different purposes: + +- An experimental feature enables developers to iterate on and deliver a new idea without committing to it or requiring a costly long-running fork. + It is primarily an issue of *implementation*, targeting Nix developers and early testers. +- The goal of an RFC is to make explicit all the implications of a change: + Explain why it is wanted, which new use-cases it enables, which interface changes it requires, etc. + It is primarily an issue of *design* and *communication*, targeting the broader community. + +This means that experimental features and RFCs are orthogonal mechanisms, and can be used independently or together as needed. diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 3869c37a4..ca69f076a 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -389,3 +389,35 @@ If a broken link occurs in a snippet that was inserted into multiple generated f If the `@docroot@` literal appears in an error message from the `mdbook-linkcheck` tool, the `@docroot@` replacement needs to be applied to the generated source file that mentions it. See existing `@docroot@` logic in the [Makefile]. Regular markdown files used for the manual have a base path of their own and they can use relative paths instead of `@docroot@`. + +## API documentation + +Doxygen API documentation is [available +online](https://hydra.nixos.org/job/nix/master/internal-api-docs/latest/download-by-type/doc/internal-api-docs). You +can also build and view it yourself: + +```console +# nix build .#hydraJobs.internal-api-docs +# xdg-open ./result/share/doc/nix/internal-api/html/index.html +``` + +or inside a `nix develop` shell by running: + +``` +# make internal-api-html +# xdg-open ./outputs/doc/share/doc/nix/internal-api/html/index.html +``` + +## Coverage analysis + +A coverage analysis report is [available +online](https://hydra.nixos.org/job/nix/master/coverage/latest/download-by-type/report/coverage). You +can build it yourself: + +``` +# nix build .#hydraJobs.coverage +# xdg-open ./result/coverage/index.html +``` + +Metrics about the change in line/function coverage over time are also +[available](https://hydra.nixos.org/job/nix/master/coverage#tabs-charts). diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index c916af4bc..4eedb2e93 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -15,7 +15,7 @@ Example: `/nix/store/g946hcz4c8mdvq2g8vxx42z51qb71rvp-git-2.38.1.drv` - See [`nix show-derivation`](./command-ref/new-cli/nix3-show-derivation.md) (experimental) for displaying the contents of store derivations. + See [`nix derivation show`](./command-ref/new-cli/nix3-derivation-show.md) (experimental) for displaying the contents of store derivations. [store derivation]: #gloss-store-derivation @@ -31,7 +31,7 @@ This means either running the `builder` executable as specified in the corresponding [derivation] or fetching a pre-built [store object] from a [substituter]. - See [`nix-build`](./command-ref/nix-build.md) and [`nix-store --realise`](./command-ref/nix-store.md#operation---realise). + See [`nix-build`](./command-ref/nix-build.md) and [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md). See [`nix build`](./command-ref/new-cli/nix3-build.md) (experimental). @@ -54,7 +54,7 @@ invoked, the Nix store can be referred to as a "_local_" or a "_remote_" one: - + A *local store* exists on the filesystem of + + A [local store]{#gloss-local-store} exists on the filesystem of the machine where Nix is invoked. You can use other local stores by passing the `--store` flag to the `nix` command. Local stores can be used for building derivations. @@ -65,17 +65,17 @@ served by the `nix-serve` Perl script. [store]: #gloss-store + [local store]: #gloss-local-store - [chroot store]{#gloss-chroot-store}\ - A local store whose canonical path is anything other than `/nix/store`. + A [local store] whose canonical path is anything other than `/nix/store`. - [binary cache]{#gloss-binary-cache}\ A *binary cache* is a Nix store which uses a different format: its metadata and signatures are kept in `.narinfo` files rather than in a - Nix database. This different format simplifies serving store objects - over the network, but cannot host builds. Examples of binary caches - include S3 buckets and the [NixOS binary - cache](https://cache.nixos.org). + [Nix database]. This different format simplifies serving store objects + over the network, but cannot host builds. Examples of binary caches + include S3 buckets and the [NixOS binary cache](https://cache.nixos.org). - [store path]{#gloss-store-path}\ The location of a [store object] in the file system, i.e., an @@ -108,7 +108,7 @@ [fixed-output derivations](#gloss-fixed-output-derivation). - [substitute]{#gloss-substitute}\ - A substitute is a command invocation stored in the Nix database that + A substitute is a command invocation stored in the [Nix database] that describes how to build a store object, bypassing the normal build mechanism (i.e., derivations). Typically, the substitute builds the store object by downloading a pre-built version of the store object @@ -127,6 +127,14 @@ builder can rely on external inputs such as the network or the system time) but the Nix model assumes it. + - Nix database{#gloss-nix-database}\ + An SQlite database to track [reference]s between [store object]s. + This is an implementation detail of the [local store]. + + Default location: `/nix/var/nix/db`. + + [Nix database]: #gloss-nix-database + - [Nix expression]{#gloss-nix-expression}\ A high-level description of software packages and compositions thereof. Deploying software using Nix entails writing Nix @@ -175,9 +183,9 @@ - [validity]{#gloss-validity}\ A store path is valid if all [store object]s in its [closure] can be read from the [store]. - For a local store, this means: + For a [local store], this means: - The store path leads to an existing [store object] in that [store]. - - The store path is listed in the Nix database as being valid. + - The store path is listed in the [Nix database] as being valid. - All paths in the store path's [closure] are valid. [validity]: #gloss-validity diff --git a/doc/manual/src/installation/env-variables.md b/doc/manual/src/installation/env-variables.md index fb8155a80..db98f52ff 100644 --- a/doc/manual/src/installation/env-variables.md +++ b/doc/manual/src/installation/env-variables.md @@ -42,14 +42,11 @@ export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt > You must not add the export and then do the install, as the Nix > installer will detect the presence of Nix configuration, and abort. -## `NIX_SSL_CERT_FILE` with macOS and the Nix daemon +If you use the Nix daemon, you should also add the following to +`/etc/nix/nix.conf`: -On macOS you must specify the environment variable for the Nix daemon -service, then restart it: - -```console -$ sudo launchctl setenv NIX_SSL_CERT_FILE /etc/ssl/my-certificate-bundle.crt -$ sudo launchctl kickstart -k system/org.nixos.nix-daemon +``` +ssl-cert-file = /etc/ssl/my-certificate-bundle.crt ``` ## Proxy Environment Variables diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 5a63236e5..3e8c48890 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -198,8 +198,7 @@ Derivations can declare some infrequently used optional attributes. - `"recursive"`\ The hash is computed over the NAR archive dump of the output - (i.e., the result of [`nix-store - --dump`](../command-ref/nix-store.md#operation---dump)). In + (i.e., the result of [`nix-store --dump`](@docroot@/command-ref/nix-store/dump.md)). In this case, the output can be anything, including a directory tree. diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index 90b325597..a07d976ad 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -43,8 +43,8 @@ If the attribute doesn’t exist, return *value* if provided, otherwise abort ev An attribute path is a dot-separated list of attribute names. An attribute name can be an identifier or a string. -> *attrpath* = *name* [ `.` *name* ]... -> *name* = *identifier* | *string* +> *attrpath* = *name* [ `.` *name* ]... \ +> *name* = *identifier* | *string* \ > *identifier* ~ `[a-zA-Z_][a-zA-Z0-9_'-]*` [Attribute selection]: #attribute-selection diff --git a/doc/manual/src/package-management/s3-substituter.md b/doc/manual/src/package-management/s3-substituter.md index 30f2b2e11..d8a1d9105 100644 --- a/doc/manual/src/package-management/s3-substituter.md +++ b/doc/manual/src/package-management/s3-substituter.md @@ -1,41 +1,11 @@ # Serving a Nix store via S3 -Nix has built-in support for storing and fetching store paths from +Nix has [built-in support](@docroot@/command-ref/new-cli/nix3-help-stores.md#s3-binary-cache-store) +for storing and fetching store paths from Amazon S3 and S3-compatible services. This uses the same *binary* cache mechanism that Nix usually uses to fetch prebuilt binaries from [cache.nixos.org](https://cache.nixos.org/). -The following options can be specified as URL parameters to the S3 URL: - - - `profile`\ - The name of the AWS configuration profile to use. By default Nix - will use the `default` profile. - - - `region`\ - The region of the S3 bucket. `us–east-1` by default. - - If your bucket is not in `us–east-1`, you should always explicitly - specify the region parameter. - - - `endpoint`\ - The URL to your S3-compatible service, for when not using Amazon S3. - Do not specify this value if you're using Amazon S3. - - > **Note** - > - > This endpoint must support HTTPS and will use path-based - > addressing instead of virtual host based addressing. - - - `scheme`\ - The scheme used for S3 requests, `https` (default) or `http`. This - option allows you to disable HTTPS for binary caches which don't - support it. - - > **Note** - > - > HTTPS should be used if the cache might contain sensitive - > information. - In this example we will use the bucket named `example-nix-cache`. ## Anonymous Reads to your S3-compatible binary cache diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index b8d14e605..5b62836bf 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -26,4 +26,33 @@ ```shell-session $ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^* ``` - provides information about each of its outputs. \ No newline at end of file + provides information about each of its outputs. + +* The experimental command `nix describe-stores` has been removed. + +* Nix stores and their settings are now documented in [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md). + +* Documentation for operations of `nix-store` and `nix-env` are now available on separate pages of the manual. + They include all common options that can be specified and common environment variables that affect these commands. + + These pages can be viewed offline with `man` using + + * `man nix-store-` and `man nix-env-` + * `nix-store --help --` and `nix-env --help --`. + +* Nix when used as a client now checks whether the store (the server) trusts the client. + (The store always had to check whether it trusts the client, but now the client is informed of the store's decision.) + This is useful for scripting interactions with (non-legacy-ssh) remote Nix stores. + + `nix store ping` and `nix doctor` now display this information. + +* A new command `nix derivation add` is created, to allow adding derivations to the store without involving the Nix language. + It exists to round out our collection of basic utility/plumbing commands, and allow for a low barrier-to-entry way of experimenting with alternative front-ends to the Nix Store. + It uses the same JSON layout as `nix show-derivation`, and is its inverse. + +* `nix show-derivation` has been renamed to `nix derivation show`. + This matches `nix derivation add`, and avoids bloating the top-level namespace. + The old name is still kept as an alias for compatibility, however. + +* The `nix derivation {add,show}` JSON format now includes the derivation name as a top-level field. + This is useful in general, but especially necessary for the `add` direction, as otherwise we would need to pass in the name out of band for certain cases. diff --git a/doc/manual/utils.nix b/doc/manual/utils.nix index d0643ef46..5eacce0dd 100644 --- a/doc/manual/utils.nix +++ b/doc/manual/utils.nix @@ -38,4 +38,46 @@ rec { filterAttrs = pred: set: listToAttrs (concatMap (name: let v = set.${name}; in if pred name v then [(nameValuePair name v)] else []) (attrNames set)); + + showSetting = { useAnchors }: name: { description, documentDefault, defaultValue, aliases, value }: + let + result = squash '' + - ${if useAnchors + then ''[`${name}`](#conf-${name})'' + else ''`${name}`''} + + ${indent " " body} + ''; + + # separate body to cleanly handle indentation + body = '' + ${description} + + **Default:** ${showDefault documentDefault defaultValue} + + ${showAliases aliases} + ''; + + showDefault = documentDefault: defaultValue: + if documentDefault then + # a StringMap value type is specified as a string, but + # this shows the value type. The empty stringmap is `null` in + # JSON, but that converts to `{ }` here. + if defaultValue == "" || defaultValue == [] || isAttrs defaultValue + then "*empty*" + else if isBool defaultValue then + if defaultValue then "`true`" else "`false`" + else "`${toString defaultValue}`" + else "*machine-specific*"; + + showAliases = aliases: + if aliases == [] then "" else + "**Deprecated alias:** ${(concatStringsSep ", " (map (s: "`${s}`") aliases))}"; + + indent = prefix: s: + concatStringsSep "\n" (map (x: if x == "" then x else "${prefix}${x}") (splitLines s)); + + in result; + + showSettings = args: settingsInfo: concatStrings (attrValues (mapAttrs (showSetting args) settingsInfo)); } diff --git a/docker.nix b/docker.nix index 203a06b53..52199af66 100644 --- a/docker.nix +++ b/docker.nix @@ -8,6 +8,7 @@ , extraPkgs ? [] , maxLayers ? 100 , nixConf ? {} +, flake-registry ? null }: let defaultPkgs = with pkgs; [ @@ -247,7 +248,16 @@ let mkdir -p $out/bin $out/usr/bin ln -s ${pkgs.coreutils}/bin/env $out/usr/bin/env ln -s ${pkgs.bashInteractive}/bin/bash $out/bin/sh - ''; + + '' + (lib.optionalString (flake-registry != null) '' + nixCacheDir="/root/.cache/nix" + mkdir -p $out$nixCacheDir + globalFlakeRegistryPath="$nixCacheDir/flake-registry.json" + ln -s ${flake-registry}/flake-registry.json $out$globalFlakeRegistryPath + mkdir -p $out/nix/var/nix/gcroots/auto + rootName=$(${pkgs.nix}/bin/nix --extra-experimental-features nix-command hash file --type sha1 --base32 <(echo -n $globalFlakeRegistryPath)) + ln -s $globalFlakeRegistryPath $out/nix/var/nix/gcroots/auto/$rootName + ''); in pkgs.dockerTools.buildLayeredImageWithNixDb { diff --git a/flake.nix b/flake.nix index af0ce5022..a4ee80b32 100644 --- a/flake.nix +++ b/flake.nix @@ -219,6 +219,7 @@ enableParallelBuilding = true; + configureFlags = testConfigureFlags; # otherwise configure fails dontBuild = true; doInstallCheck = true; @@ -319,12 +320,18 @@ }; let canRunInstalled = currentStdenv.buildPlatform.canExecute currentStdenv.hostPlatform; + + sourceByRegexInverted = rxs: origSrc: final.lib.cleanSourceWith { + filter = (path: type: + let relPath = final.lib.removePrefix (toString origSrc + "/") (toString path); + in ! lib.any (re: builtins.match re relPath != null) rxs); + src = origSrc; + }; in currentStdenv.mkDerivation (finalAttrs: { name = "nix-${version}"; inherit version; - src = self; - + src = sourceByRegexInverted [ "tests/nixos/.*" "tests/installer/.*" ] self; VERSION_SUFFIX = versionSuffix; outputs = [ "out" "dev" "doc" ]; @@ -466,8 +473,6 @@ }; in { - inherit nixpkgsFor; - # A Nixpkgs overlay that overrides the 'nix' and # 'nix.perl-bindings' packages. overlays.default = overlayFor (p: p.stdenv); @@ -577,6 +582,8 @@ tests.nix-copy-closure = runNixOSTestFor "x86_64-linux" ./tests/nixos/nix-copy-closure.nix; + tests.nix-copy = runNixOSTestFor "x86_64-linux" ./tests/nixos/nix-copy.nix; + tests.nssPreload = runNixOSTestFor "x86_64-linux" ./tests/nixos/nss-preload.nix; tests.githubFlakes = runNixOSTestFor "x86_64-linux" ./tests/nixos/github-flakes.nix; diff --git a/local.mk b/local.mk index 6a7074e8e..6951c179e 100644 --- a/local.mk +++ b/local.mk @@ -1,6 +1,8 @@ clean-files += Makefile.config -GLOBAL_CXXFLAGS += -Wno-deprecated-declarations +GLOBAL_CXXFLAGS += -Wno-deprecated-declarations -Werror=switch +# Allow switch-enum to be overridden for files that do not support it, usually because of dependency headers. +ERROR_SWITCH_ENUM = -Werror=switch-enum $(foreach i, config.h $(wildcard src/lib*/*.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) diff --git a/maintainers/README.md b/maintainers/README.md index 476a5f51e..618bfb4e4 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -56,7 +56,7 @@ Meeting notes are collected on a [collaborative scratchpad](https://pad.lassul.u The team uses a [GitHub project board](https://github.com/orgs/NixOS/projects/19/views/1) for tracking its work. -Issues on the board progress through the following states: +Items on the board progress through the following states: - No Status @@ -69,6 +69,7 @@ Issues on the board progress through the following states: 2. [security](https://github.com/NixOS/nix/labels/security) 3. [regression](https://github.com/NixOS/nix/labels/regression) 4. [bug](https://github.com/NixOS/nix/issues?q=is%3Aopen+label%3Abug+sort%3Areactions-%2B1-desc) + 5. [tests of existing functionality](https://github.com/NixOS/nix/issues?q=is%3Aopen+label%3Atests+-label%3Afeature+sort%3Areactions-%2B1-desc) - [oldest pull requests](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+sort%3Acreated-asc) - [most popular pull requests](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+sort%3Areactions-%2B1-desc) @@ -79,6 +80,9 @@ Issues on the board progress through the following states: If there is disagreement on the general idea behind an issue or pull request, it is moved to _To discuss_, otherwise to _In review_. + To ensure process quality and reliability, all non-trivial pull requests must be triaged before merging. + What constitutes a trivial pull request is up to maintainers' judgement. + - To discuss Pull requests and issues that are deemed important and controversial are discussed by the team during discussion meetings. @@ -91,7 +95,7 @@ Issues on the board progress through the following states: Contributors who took the time to implement concrete change proposals should not wait indefinitely. - - Prioritise fixing bugs over documentation, improvements or new features + - Prioritise fixing bugs and testing over documentation, improvements or new features The team values stability and accessibility higher than raw functionality. diff --git a/misc/launchd/org.nixos.nix-daemon.plist.in b/misc/launchd/org.nixos.nix-daemon.plist.in index 5fa489b20..e1470cf99 100644 --- a/misc/launchd/org.nixos.nix-daemon.plist.in +++ b/misc/launchd/org.nixos.nix-daemon.plist.in @@ -4,8 +4,6 @@ EnvironmentVariables - NIX_SSL_CERT_FILE - /nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt OBJC_DISABLE_INITIALIZE_FORK_SAFETY YES diff --git a/mk/patterns.mk b/mk/patterns.mk index 86a724806..c81150260 100644 --- a/mk/patterns.mk +++ b/mk/patterns.mk @@ -1,10 +1,10 @@ $(buildprefix)%.o: %.cc @mkdir -p "$(dir $@)" - $(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep, $@) -MP $(buildprefix)%.o: %.cpp @mkdir -p "$(dir $@)" - $(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep, $@) -MP $(buildprefix)%.o: %.c @mkdir -p "$(dir $@)" diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 63e3e3fa9..cfc4baaca 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -305,7 +305,7 @@ connected: std::set missingRealisations; StorePathSet missingPaths; - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) && !drv.type().hasKnownOutputPaths()) { + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && !drv.type().hasKnownOutputPaths()) { for (auto & outputName : wantedOutputs) { auto thisOutputHash = outputHashes.at(outputName); auto thisOutputId = DrvOutput{ thisOutputHash, outputName }; @@ -337,7 +337,7 @@ connected: for (auto & realisation : missingRealisations) { // Should hold, because if the feature isn't enabled the set // of missing realisations should be empty - settings.requireExperimentalFeature(Xp::CaDerivations); + experimentalFeatureSettings.require(Xp::CaDerivations); store->registerDrvOutput(realisation); } diff --git a/src/libcmd/command-installable-value.cc b/src/libcmd/command-installable-value.cc new file mode 100644 index 000000000..7e0c15eb8 --- /dev/null +++ b/src/libcmd/command-installable-value.cc @@ -0,0 +1,11 @@ +#include "command-installable-value.hh" + +namespace nix { + +void InstallableValueCommand::run(ref store, ref installable) +{ + auto installableValue = InstallableValue::require(installable); + run(store, installableValue); +} + +} diff --git a/src/libcmd/command-installable-value.hh b/src/libcmd/command-installable-value.hh new file mode 100644 index 000000000..7880d4119 --- /dev/null +++ b/src/libcmd/command-installable-value.hh @@ -0,0 +1,23 @@ +#pragma once +///@file + +#include "installable-value.hh" +#include "command.hh" + +namespace nix { + +/** + * An InstallableCommand where the single positional argument must be an + * InstallableValue in particular. + */ +struct InstallableValueCommand : InstallableCommand +{ + /** + * Entry point to this command + */ + virtual void run(ref store, ref installable) = 0; + + void run(ref store, ref installable) override; +}; + +} diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 874ca3249..96236b987 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -1,6 +1,7 @@ #pragma once +///@file -#include "installables.hh" +#include "installable-value.hh" #include "args.hh" #include "common-eval-args.hh" #include "path.hh" @@ -18,6 +19,7 @@ class EvalState; struct Pos; class Store; +static constexpr Command::Category catHelp = -1; static constexpr Command::Category catSecondary = 100; static constexpr Command::Category catUtility = 101; static constexpr Command::Category catNixInstallation = 102; diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 908127b4d..5b6477c82 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -136,7 +136,11 @@ MixEvalArgs::MixEvalArgs() addFlag({ .longName = "eval-store", - .description = "The Nix store to use for evaluations.", + .description = + R"( + The [URL of the Nix store](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + to use for evaluation, i.e. to store derivations (`.drv` files) and inputs referenced by them. + )", .category = category, .labels = {"store-url"}, .handler = {&evalStoreUrl}, @@ -166,7 +170,7 @@ Path lookupFileArg(EvalState & state, std::string_view s) } else if (hasPrefix(s, "flake:")) { - settings.requireExperimentalFeature(Xp::Flakes); + experimentalFeatureSettings.require(Xp::Flakes); auto flakeRef = parseFlakeRef(std::string(s.substr(6)), {}, true, false); auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first.storePath; return state.store->toRealPath(storePath); diff --git a/src/libcmd/common-eval-args.hh b/src/libcmd/common-eval-args.hh index 1ec800613..b69db11dd 100644 --- a/src/libcmd/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "args.hh" diff --git a/src/libcmd/editor-for.hh b/src/libcmd/editor-for.hh index 8fbd08792..c8c4e9d9b 100644 --- a/src/libcmd/editor-for.hh +++ b/src/libcmd/editor-for.hh @@ -1,11 +1,14 @@ #pragma once +///@file #include "types.hh" namespace nix { -/* Helper function to generate args that invoke $EDITOR on - filename:lineno. */ +/** + * Helper function to generate args that invoke $EDITOR on + * filename:lineno. + */ Strings editorFor(const Path & file, uint32_t line); } diff --git a/src/libcmd/installable-attr-path.cc b/src/libcmd/installable-attr-path.cc index d9377f0d6..cf513126d 100644 --- a/src/libcmd/installable-attr-path.cc +++ b/src/libcmd/installable-attr-path.cc @@ -87,6 +87,10 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths() .drvPath = drvPath, .outputs = outputs, }, + .info = make_ref(ExtraPathInfoValue::Value { + /* FIXME: reconsider backwards compatibility above + so we can fill in this info. */ + }), }); return res; diff --git a/src/libcmd/installable-attr-path.hh b/src/libcmd/installable-attr-path.hh index c06132ec8..e9f0c33da 100644 --- a/src/libcmd/installable-attr-path.hh +++ b/src/libcmd/installable-attr-path.hh @@ -1,3 +1,6 @@ +#pragma once +///@file + #include "globals.hh" #include "installable-value.hh" #include "outputs-spec.hh" diff --git a/src/libcmd/installable-derived-path.cc b/src/libcmd/installable-derived-path.cc index 729dc7d31..6ecf54b7c 100644 --- a/src/libcmd/installable-derived-path.cc +++ b/src/libcmd/installable-derived-path.cc @@ -10,7 +10,10 @@ std::string InstallableDerivedPath::what() const DerivedPathsWithInfo InstallableDerivedPath::toDerivedPaths() { - return {{.path = derivedPath, .info = {} }}; + return {{ + .path = derivedPath, + .info = make_ref(), + }}; } std::optional InstallableDerivedPath::getStorePath() diff --git a/src/libcmd/installable-derived-path.hh b/src/libcmd/installable-derived-path.hh index 042878b91..e0b4f18b3 100644 --- a/src/libcmd/installable-derived-path.hh +++ b/src/libcmd/installable-derived-path.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "installables.hh" diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index 7b0cc376d..a3352af76 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -101,7 +101,8 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() return {{ .path = DerivedPath::Opaque { .path = std::move(storePath), - } + }, + .info = make_ref(), }}; } @@ -113,7 +114,8 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() return {{ .path = DerivedPath::Opaque { .path = std::move(*storePath), - } + }, + .info = make_ref(), }}; } else throw Error("flake output attribute '%s' evaluates to the string '%s' which is not a store path", attrPath, s); @@ -160,13 +162,16 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() }, }, extendedOutputsSpec.raw()), }, - .info = { - .priority = priority, - .originalRef = flakeRef, - .resolvedRef = getLockedFlake()->flake.lockedRef, - .attrPath = attrPath, - .extendedOutputsSpec = extendedOutputsSpec, - } + .info = make_ref( + ExtraPathInfoValue::Value { + .priority = priority, + .attrPath = attrPath, + .extendedOutputsSpec = extendedOutputsSpec, + }, + ExtraPathInfoFlake::Flake { + .originalRef = flakeRef, + .resolvedRef = getLockedFlake()->flake.lockedRef, + }), }}; } @@ -212,6 +217,7 @@ std::shared_ptr InstallableFlake::getLockedFlake() const { if (!_lockedFlake) { flake::LockFlags lockFlagsApplyConfig = lockFlags; + // FIXME why this side effect? lockFlagsApplyConfig.applyNixConfig = true; _lockedFlake = std::make_shared(lockFlake(*state, flakeRef, lockFlagsApplyConfig)); } @@ -229,7 +235,7 @@ FlakeRef InstallableFlake::nixpkgsFlakeRef() const } } - return Installable::nixpkgsFlakeRef(); + return InstallableValue::nixpkgsFlakeRef(); } } diff --git a/src/libcmd/installable-flake.hh b/src/libcmd/installable-flake.hh index c75765086..afe64d977 100644 --- a/src/libcmd/installable-flake.hh +++ b/src/libcmd/installable-flake.hh @@ -1,9 +1,34 @@ #pragma once +///@file #include "installable-value.hh" namespace nix { +/** + * Extra info about a \ref DerivedPath "derived path" that ultimately + * come from a Flake. + * + * Invariant: every ExtraPathInfo gotten from an InstallableFlake should + * be possible to downcast to an ExtraPathInfoFlake. + */ +struct ExtraPathInfoFlake : ExtraPathInfoValue +{ + /** + * Extra struct to get around C++ designated initializer limitations + */ + struct Flake { + FlakeRef originalRef; + FlakeRef resolvedRef; + }; + + Flake flake; + + ExtraPathInfoFlake(Value && v, Flake && f) + : ExtraPathInfoValue(std::move(v)), flake(f) + { } +}; + struct InstallableFlake : InstallableValue { FlakeRef flakeRef; @@ -33,8 +58,10 @@ struct InstallableFlake : InstallableValue std::pair toValue(EvalState & state) override; - /* Get a cursor to every attrpath in getActualAttrPaths() - that exists. However if none exists, throw an exception. */ + /** + * Get a cursor to every attrpath in getActualAttrPaths() that + * exists. However if none exists, throw an exception. + */ std::vector> getCursors(EvalState & state) override; diff --git a/src/libcmd/installable-value.cc b/src/libcmd/installable-value.cc new file mode 100644 index 000000000..3a7ede4e2 --- /dev/null +++ b/src/libcmd/installable-value.cc @@ -0,0 +1,44 @@ +#include "installable-value.hh" +#include "eval-cache.hh" + +namespace nix { + +std::vector> +InstallableValue::getCursors(EvalState & state) +{ + auto evalCache = + std::make_shared(std::nullopt, state, + [&]() { return toValue(state).first; }); + return {evalCache->getRoot()}; +} + +ref +InstallableValue::getCursor(EvalState & state) +{ + /* Although getCursors should return at least one element, in case it doesn't, + bound check to avoid an undefined behavior for vector[0] */ + return getCursors(state).at(0); +} + +static UsageError nonValueInstallable(Installable & installable) +{ + return UsageError("installable '%s' does not correspond to a Nix language value", installable.what()); +} + +InstallableValue & InstallableValue::require(Installable & installable) +{ + auto * castedInstallable = dynamic_cast(&installable); + if (!castedInstallable) + throw nonValueInstallable(installable); + return *castedInstallable; +} + +ref InstallableValue::require(ref installable) +{ + auto castedInstallable = installable.dynamic_pointer_cast(); + if (!castedInstallable) + throw nonValueInstallable(*installable); + return ref { castedInstallable }; +} + +} diff --git a/src/libcmd/installable-value.hh b/src/libcmd/installable-value.hh index c6cdc4797..bfb3bfeed 100644 --- a/src/libcmd/installable-value.hh +++ b/src/libcmd/installable-value.hh @@ -1,14 +1,108 @@ #pragma once +///@file #include "installables.hh" +#include "flake/flake.hh" namespace nix { +struct DrvInfo; +struct SourceExprCommand; + +namespace eval_cache { class EvalCache; class AttrCursor; } + +struct App +{ + std::vector context; + Path program; + // FIXME: add args, sandbox settings, metadata, ... +}; + +struct UnresolvedApp +{ + App unresolved; + App resolve(ref evalStore, ref store); +}; + +/** + * Extra info about a \ref DerivedPath "derived path" that ultimately + * come from a Nix language value. + * + * Invariant: every ExtraPathInfo gotten from an InstallableValue should + * be possible to downcast to an ExtraPathInfoValue. + */ +struct ExtraPathInfoValue : ExtraPathInfo +{ + /** + * Extra struct to get around C++ designated initializer limitations + */ + struct Value { + /** + * An optional priority for use with "build envs". See Package + */ + std::optional priority; + + /** + * The attribute path associated with this value. The idea is + * that an installable referring to a value typically refers to + * a larger value, from which we project a smaller value out + * with this. + */ + std::string attrPath; + + /** + * \todo merge with DerivedPath's 'outputs' field? + */ + ExtendedOutputsSpec extendedOutputsSpec; + }; + + Value value; + + ExtraPathInfoValue(Value && v) + : value(v) + { } + + virtual ~ExtraPathInfoValue() = default; +}; + +/** + * An Installable which corresponds a Nix langauge value, in addition to + * a collection of \ref DerivedPath "derived paths". + */ struct InstallableValue : Installable { ref state; InstallableValue(ref state) : state(state) {} + + virtual ~InstallableValue() { } + + virtual std::pair toValue(EvalState & state) = 0; + + /** + * Get a cursor to each value this Installable could refer to. + * However if none exists, throw exception instead of returning + * empty vector. + */ + virtual std::vector> + getCursors(EvalState & state); + + /** + * Get the first and most preferred cursor this Installable could + * refer to, or throw an exception if none exists. + */ + virtual ref + getCursor(EvalState & state); + + UnresolvedApp toApp(EvalState & state); + + virtual FlakeRef nixpkgsFlakeRef() const + { + return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}}); + } + + static InstallableValue & require(Installable & installable); + static ref require(ref installable); }; } diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 5cbf26b88..32ae46d9f 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -102,6 +102,28 @@ MixFlakeOptions::MixFlakeOptions() }} }); + addFlag({ + .longName = "reference-lock-file", + .description = "Read the given lock file instead of `flake.lock` within the top-level flake.", + .category = category, + .labels = {"flake-lock-path"}, + .handler = {[&](std::string lockFilePath) { + lockFlags.referenceLockFilePath = lockFilePath; + }}, + .completer = completePath + }); + + addFlag({ + .longName = "output-lock-file", + .description = "Write the given lock file instead of `flake.lock` within the top-level flake.", + .category = category, + .labels = {"flake-lock-path"}, + .handler = {[&](std::string lockFilePath) { + lockFlags.outputLockFilePath = lockFilePath; + }}, + .completer = completePath + }); + addFlag({ .longName = "inputs-from", .description = "Use the inputs of the specified flake as registry entries.", @@ -332,7 +354,7 @@ void completeFlakeRefWithFragment( void completeFlakeRef(ref store, std::string_view prefix) { - if (!settings.isExperimentalFeatureEnabled(Xp::Flakes)) + if (!experimentalFeatureSettings.isEnabled(Xp::Flakes)) return; if (prefix == "") @@ -364,23 +386,6 @@ DerivedPathWithInfo Installable::toDerivedPath() return std::move(buildables[0]); } -std::vector> -Installable::getCursors(EvalState & state) -{ - auto evalCache = - std::make_shared(std::nullopt, state, - [&]() { return toValue(state).first; }); - return {evalCache->getRoot()}; -} - -ref -Installable::getCursor(EvalState & state) -{ - /* Although getCursors should return at least one element, in case it doesn't, - bound check to avoid an undefined behavior for vector[0] */ - return getCursors(state).at(0); -} - static StorePath getDeriver( ref store, const Installable & i, @@ -534,7 +539,7 @@ std::vector, BuiltPathWithResult>> Installable::build struct Aux { - ExtraPathInfo info; + ref info; ref installable; }; diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index 6c2922d89..42d6c7c7c 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -1,12 +1,11 @@ #pragma once +///@file #include "util.hh" #include "path.hh" #include "outputs-spec.hh" #include "derived-path.hh" -#include "eval.hh" #include "store-api.hh" -#include "flake/flake.hh" #include "build-result.hh" #include @@ -14,113 +13,144 @@ namespace nix { struct DrvInfo; -struct SourceExprCommand; - -namespace eval_cache { class EvalCache; class AttrCursor; } - -struct App -{ - std::vector context; - Path program; - // FIXME: add args, sandbox settings, metadata, ... -}; - -struct UnresolvedApp -{ - App unresolved; - App resolve(ref evalStore, ref store); -}; enum class Realise { - /* Build the derivation. Postcondition: the - derivation outputs exist. */ + /** + * Build the derivation. + * + * Postcondition: the derivation outputs exist. + */ Outputs, - /* Don't build the derivation. Postcondition: the store derivation - exists. */ + /** + * Don't build the derivation. + * + * Postcondition: the store derivation exists. + */ Derivation, - /* Evaluate in dry-run mode. Postcondition: nothing. */ - // FIXME: currently unused, but could be revived if we can - // evaluate derivations in-memory. + /** + * Evaluate in dry-run mode. + * + * Postcondition: nothing. + * + * \todo currently unused, but could be revived if we can evaluate + * derivations in-memory. + */ Nothing }; -/* How to handle derivations in commands that operate on store paths. */ +/** + * How to handle derivations in commands that operate on store paths. + */ enum class OperateOn { - /* Operate on the output path. */ + /** + * Operate on the output path. + */ Output, - /* Operate on the .drv path. */ + /** + * Operate on the .drv path. + */ Derivation }; +/** + * Extra info about a DerivedPath + * + * Yes, this is empty, but that is intended. It will be sub-classed by + * the subclasses of Installable to allow those to provide more info. + * Certain commands will make use of this info. + */ struct ExtraPathInfo { - std::optional priority; - std::optional originalRef; - std::optional resolvedRef; - std::optional attrPath; - // FIXME: merge with DerivedPath's 'outputs' field? - std::optional extendedOutputsSpec; + virtual ~ExtraPathInfo() = default; }; -/* A derived path with any additional info that commands might - need from the derivation. */ +/** + * A DerivedPath with \ref ExtraPathInfo "any additional info" that + * commands might need from the derivation. + */ struct DerivedPathWithInfo { DerivedPath path; - ExtraPathInfo info; + ref info; }; +/** + * Like DerivedPathWithInfo but extending BuiltPath with \ref + * ExtraPathInfo "extra info" and also possibly the \ref BuildResult + * "result of building". + */ struct BuiltPathWithResult { BuiltPath path; - ExtraPathInfo info; + ref info; std::optional result; }; +/** + * Shorthand, for less typing and helping us keep the choice of + * collection in sync. + */ typedef std::vector DerivedPathsWithInfo; struct Installable; + +/** + * Shorthand, for less typing and helping us keep the choice of + * collection in sync. + */ typedef std::vector> Installables; +/** + * Installables are the main positional arguments for the Nix + * Command-line. + * + * This base class is very flexible, and just assumes and the + * Installable refers to a collection of \ref DerivedPath "derived paths" with + * \ref ExtraPathInfo "extra info". + */ struct Installable { virtual ~Installable() { } + /** + * What Installable is this? + * + * Prints back valid CLI syntax that would result in this same + * installable. It doesn't need to be exactly what the user wrote, + * just something that means the same thing. + */ virtual std::string what() const = 0; + /** + * Get the collection of \ref DerivedPathWithInfo "derived paths + * with info" that this \ref Installable instalallable denotes. + * + * This is the main method of this class + */ virtual DerivedPathsWithInfo toDerivedPaths() = 0; + /** + * A convenience wrapper of the above for when we expect an + * installable to produce a single \ref DerivedPath "derived path" + * only. + * + * If no or multiple \ref DerivedPath "derived paths" are produced, + * and error is raised. + */ DerivedPathWithInfo toDerivedPath(); - UnresolvedApp toApp(EvalState & state); - - virtual std::pair toValue(EvalState & state) - { - throw Error("argument '%s' cannot be evaluated", what()); - } - - /* Return a value only if this installable is a store path or a - symlink to it. */ + /** + * Return a value only if this installable is a store path or a + * symlink to it. + * + * \todo should we move this to InstallableDerivedPath? It is only + * supposed to work there anyways. Can always downcast. + */ virtual std::optional getStorePath() { return {}; } - /* Get a cursor to each value this Installable could refer to. However - if none exists, throw exception instead of returning empty vector. */ - virtual std::vector> - getCursors(EvalState & state); - - /* Get the first and most preferred cursor this Installable could refer - to, or throw an exception if none exists. */ - virtual ref - getCursor(EvalState & state); - - virtual FlakeRef nixpkgsFlakeRef() const - { - return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}}); - } - static std::vector build( ref evalStore, ref store, diff --git a/src/libcmd/legacy.hh b/src/libcmd/legacy.hh index f503b0da3..357500a4d 100644 --- a/src/libcmd/legacy.hh +++ b/src/libcmd/legacy.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include diff --git a/src/libcmd/markdown.hh b/src/libcmd/markdown.hh index 78320fcf5..a04d32a4f 100644 --- a/src/libcmd/markdown.hh +++ b/src/libcmd/markdown.hh @@ -1,3 +1,6 @@ +#pragma once +///@file + #include "types.hh" namespace nix { diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index e3afb1531..57848a5d3 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -252,7 +252,9 @@ void NixRepl::mainLoop() el_hist_size = 1000; #endif read_history(historyFile.c_str()); + auto oldRepl = curRepl; curRepl = this; + Finally restoreRepl([&] { curRepl = oldRepl; }); #ifndef READLINE rl_set_complete_func(completionCallback); rl_set_list_possib_func(listPossibleCallback); @@ -1024,6 +1026,8 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m str << v.fpoint; break; + case nThunk: + case nExternal: default: str << ANSI_RED "«unknown»" ANSI_NORMAL; break; diff --git a/src/libcmd/repl.hh b/src/libcmd/repl.hh index dfccc93e7..731c8e6db 100644 --- a/src/libcmd/repl.hh +++ b/src/libcmd/repl.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "eval.hh" diff --git a/src/libexpr/attr-path.hh b/src/libexpr/attr-path.hh index 117e0051b..b2bfb5d04 100644 --- a/src/libexpr/attr-path.hh +++ b/src/libexpr/attr-path.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "eval.hh" @@ -16,7 +17,9 @@ std::pair findAlongAttrPath( Bindings & autoArgs, Value & vIn); -/* Heuristic to find the filename and lineno or a nix value. */ +/** + * Heuristic to find the filename and lineno or a nix value. + */ std::pair findPackageFilename(EvalState & state, Value & v, std::string what); std::vector parseAttrPath(EvalState & state, std::string_view s); diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh index dcc73b506..31215f880 100644 --- a/src/libexpr/attr-set.hh +++ b/src/libexpr/attr-set.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "nixexpr.hh" #include "symbol-table.hh" @@ -12,7 +13,9 @@ namespace nix { class EvalState; struct Value; -/* Map one attribute name to its value. */ +/** + * Map one attribute name to its value. + */ struct Attr { /* the placement of `name` and `pos` in this struct is important. @@ -36,10 +39,12 @@ static_assert(sizeof(Attr) == 2 * sizeof(uint32_t) + sizeof(Value *), "avoid introducing any padding into Attr if at all possible, and do not " "introduce new fields that need not be present for almost every instance."); -/* Bindings contains all the attributes of an attribute set. It is defined - by its size and its capacity, the capacity being the number of Attr - elements allocated after this structure, while the size corresponds to - the number of elements already inserted in this structure. */ +/** + * Bindings contains all the attributes of an attribute set. It is defined + * by its size and its capacity, the capacity being the number of Attr + * elements allocated after this structure, while the size corresponds to + * the number of elements already inserted in this structure. + */ class Bindings { public: @@ -94,7 +99,9 @@ public: size_t capacity() { return capacity_; } - /* Returns the attributes in lexicographically sorted order. */ + /** + * Returns the attributes in lexicographically sorted order. + */ std::vector lexicographicOrder(const SymbolTable & symbols) const { std::vector res; @@ -111,9 +118,11 @@ public: friend class EvalState; }; -/* A wrapper around Bindings that ensures that its always in sorted - order at the end. The only way to consume a BindingsBuilder is to - call finish(), which sorts the bindings. */ +/** + * A wrapper around Bindings that ensures that its always in sorted + * order at the end. The only way to consume a BindingsBuilder is to + * call finish(), which sorts the bindings. + */ class BindingsBuilder { Bindings * bindings; diff --git a/src/libexpr/eval-cache.hh b/src/libexpr/eval-cache.hh index c93e55b93..46c4999c8 100644 --- a/src/libexpr/eval-cache.hh +++ b/src/libexpr/eval-cache.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "sync.hh" #include "hash.hh" @@ -109,8 +110,10 @@ public: ref getAttr(std::string_view name); - /* Get an attribute along a chain of attrsets. Note that this does - not auto-call functors or functions. */ + /** + * Get an attribute along a chain of attrsets. Note that this does + * not auto-call functors or functions. + */ OrSuggestions> findAlongAttrPath(const std::vector & attrPath, bool force = false); std::string getString(); @@ -129,7 +132,9 @@ public: Value & forceValue(); - /* Force creation of the .drv file in the Nix store. */ + /** + * Force creation of the .drv file in the Nix store. + */ StorePath forceDerivation(); }; diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index f0da688db..a988fa40c 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -1,10 +1,13 @@ #pragma once +///@file #include "eval.hh" namespace nix { -/* Note: Various places expect the allocated memory to be zeroed. */ +/** + * Note: Various places expect the allocated memory to be zeroed. + */ [[gnu::always_inline]] inline void * allocBytes(size_t n) { diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 2721b6733..18cfd9531 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -8,6 +8,7 @@ #include "eval-inline.hh" #include "filetransfer.hh" #include "function-trace.hh" +#include "profiles.hh" #include #include @@ -172,7 +173,17 @@ void Value::print(const SymbolTable & symbols, std::ostream & str, case tFloat: str << fpoint; break; + case tBlackhole: + // Although we know for sure that it's going to be an infinite recursion + // when this value is accessed _in the current context_, it's likely + // that the user will misinterpret a simpler «infinite recursion» output + // as a definitive statement about the value, while in fact it may be + // a valid value after `builtins.trace` and perhaps some other steps + // have completed. + str << "«potential infinite recursion»"; + break; default: + printError("Nix evaluator internal error: Value::print(): invalid value type %1%", internalType); abort(); } } @@ -228,6 +239,9 @@ std::string_view showType(ValueType type) std::string showType(const Value & v) { + // Allow selecting a subset of enum values + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wswitch-enum" switch (v.internalType) { case tString: return v.string.context ? "a string with context" : "a string"; case tPrimOp: @@ -241,16 +255,21 @@ std::string showType(const Value & v) default: return std::string(showType(v.type())); } + #pragma GCC diagnostic pop } PosIdx Value::determinePos(const PosIdx pos) const { + // Allow selecting a subset of enum values + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wswitch-enum" switch (internalType) { case tAttrs: return attrs->pos; case tLambda: return lambda.fun->pos; case tApp: return app.left->determinePos(pos); default: return pos; } + #pragma GCC diagnostic pop } bool Value::isTrivial() const @@ -325,6 +344,22 @@ static Symbol getName(const AttrName & name, EvalState & state, Env & env) } } +#if HAVE_BOEHMGC +/* Disable GC while this object lives. Used by CoroutineContext. + * + * Boehm keeps a count of GC_disable() and GC_enable() calls, + * and only enables GC when the count matches. + */ +class BoehmDisableGC { +public: + BoehmDisableGC() { + GC_disable(); + }; + ~BoehmDisableGC() { + GC_enable(); + }; +}; +#endif static bool gcInitialised = false; @@ -349,6 +384,15 @@ void initGC() StackAllocator::defaultAllocator = &boehmGCStackAllocator; + +#if NIX_BOEHM_PATCH_VERSION != 1 + printTalkative("Unpatched BoehmGC, disabling GC inside coroutines"); + /* Used to disable GC when entering coroutines on macOS */ + create_coro_gc_hook = []() -> std::shared_ptr { + return std::make_shared(); + }; +#endif + /* Set the initial heap size to something fairly big (25% of physical RAM, up to a maximum of 384 MiB) so that in most cases we don't need to garbage collect at all. (Collection has a @@ -2326,6 +2370,7 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v case nFloat: return v1.fpoint == v2.fpoint; + case nThunk: // Must not be left by forceValue default: error("cannot compare %1% with %2%", showType(v1), showType(v2)).withTrace(pos, errorCtx).debugThrow(); } @@ -2491,8 +2536,8 @@ Strings EvalSettings::getDefaultNixPath() if (!evalSettings.restrictEval && !evalSettings.pureEval) { add(settings.useXDGBaseDirectories ? getStateDir() + "/nix/defexpr/channels" : getHome() + "/.nix-defexpr/channels"); - add(settings.nixStateDir + "/profiles/per-user/root/channels/nixpkgs", "nixpkgs"); - add(settings.nixStateDir + "/profiles/per-user/root/channels"); + add(rootChannelsDir() + "/nixpkgs", "nixpkgs"); + add(rootChannelsDir()); } return res; diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index e4d5906bd..b3b985683 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "attr-set.hh" #include "types.hh" @@ -42,7 +43,10 @@ struct PrimOp struct Env { Env * up; - unsigned short prevWith:14; // nr of levels up to next `with' environment + /** + * Number of of levels up to next `with` environment + */ + unsigned short prevWith:14; enum { Plain = 0, HasWithExpr, HasWithAttrs } type:2; Value * values[0]; }; @@ -55,8 +59,10 @@ std::unique_ptr mapStaticEnvBindings(const SymbolTable & st, const Stati void copyContext(const Value & v, PathSet & context); -/* Cache for calls to addToStore(); maps source paths to the store - paths. */ +/** + * Cache for calls to addToStore(); maps source paths to the store + * paths. + */ typedef std::map SrcToStore; @@ -68,7 +74,9 @@ typedef std::pair SearchPathElem; typedef std::list SearchPath; -/* Initialise the Boehm GC, if applicable. */ +/** + * Initialise the Boehm GC, if applicable. + */ void initGC(); @@ -143,26 +151,36 @@ public: sOutputSpecified; Symbol sDerivationNix; - /* If set, force copying files to the Nix store even if they - already exist there. */ + /** + * If set, force copying files to the Nix store even if they + * already exist there. + */ RepairFlag repair; - /* The allowed filesystem paths in restricted or pure evaluation - mode. */ + /** + * The allowed filesystem paths in restricted or pure evaluation + * mode. + */ std::optional allowedPaths; Bindings emptyBindings; - /* Store used to materialise .drv files. */ + /** + * Store used to materialise .drv files. + */ const ref store; - /* Store used to build stuff. */ + /** + * Store used to build stuff. + */ const ref buildStore; RootValue vCallFlake = nullptr; RootValue vImportedDrvToDerivation = nullptr; - /* Debugger */ + /** + * Debugger + */ void (* debugRepl)(ref es, const ValMap & extraEnv); bool debugStop; bool debugQuit; @@ -218,7 +236,9 @@ public: private: SrcToStore srcToStore; - /* A cache from path names to parse trees. */ + /** + * A cache from path names to parse trees. + */ #if HAVE_BOEHMGC typedef std::map, traceable_allocator>> FileParseCache; #else @@ -226,7 +246,9 @@ private: #endif FileParseCache fileParseCache; - /* A cache from path names to values. */ + /** + * A cache from path names to values. + */ #if HAVE_BOEHMGC typedef std::map, traceable_allocator>> FileEvalCache; #else @@ -238,17 +260,25 @@ private: std::map> searchPathResolved; - /* Cache used by checkSourcePath(). */ + /** + * Cache used by checkSourcePath(). + */ std::unordered_map resolvedPaths; - /* Cache used by prim_match(). */ + /** + * Cache used by prim_match(). + */ std::shared_ptr regexCache; #if HAVE_BOEHMGC - /* Allocation cache for GC'd Value objects. */ + /** + * Allocation cache for GC'd Value objects. + */ std::shared_ptr valueAllocCache; - /* Allocation cache for size-1 Env objects. */ + /** + * Allocation cache for size-1 Env objects. + */ std::shared_ptr env1AllocCache; #endif @@ -264,47 +294,65 @@ public: SearchPath getSearchPath() { return searchPath; } - /* Allow access to a path. */ + /** + * Allow access to a path. + */ void allowPath(const Path & path); - /* Allow access to a store path. Note that this gets remapped to - the real store path if `store` is a chroot store. */ + /** + * Allow access to a store path. Note that this gets remapped to + * the real store path if `store` is a chroot store. + */ void allowPath(const StorePath & storePath); - /* Allow access to a store path and return it as a string. */ + /** + * Allow access to a store path and return it as a string. + */ void allowAndSetStorePathString(const StorePath & storePath, Value & v); - /* Check whether access to a path is allowed and throw an error if - not. Otherwise return the canonicalised path. */ + /** + * Check whether access to a path is allowed and throw an error if + * not. Otherwise return the canonicalised path. + */ Path checkSourcePath(const Path & path); void checkURI(const std::string & uri); - /* When using a diverted store and 'path' is in the Nix store, map - 'path' to the diverted location (e.g. /nix/store/foo is mapped - to /home/alice/my-nix/nix/store/foo). However, this is only - done if the context is not empty, since otherwise we're - probably trying to read from the actual /nix/store. This is - intended to distinguish between import-from-derivation and - sources stored in the actual /nix/store. */ + /** + * When using a diverted store and 'path' is in the Nix store, map + * 'path' to the diverted location (e.g. /nix/store/foo is mapped + * to /home/alice/my-nix/nix/store/foo). However, this is only + * done if the context is not empty, since otherwise we're + * probably trying to read from the actual /nix/store. This is + * intended to distinguish between import-from-derivation and + * sources stored in the actual /nix/store. + */ Path toRealPath(const Path & path, const PathSet & context); - /* Parse a Nix expression from the specified file. */ + /** + * Parse a Nix expression from the specified file. + */ Expr * parseExprFromFile(const Path & path); Expr * parseExprFromFile(const Path & path, std::shared_ptr & staticEnv); - /* Parse a Nix expression from the specified string. */ + /** + * Parse a Nix expression from the specified string. + */ Expr * parseExprFromString(std::string s, const Path & basePath, std::shared_ptr & staticEnv); Expr * parseExprFromString(std::string s, const Path & basePath); Expr * parseStdin(); - /* Evaluate an expression read from the given file to normal - form. Optionally enforce that the top-level expression is - trivial (i.e. doesn't require arbitrary computation). */ + /** + * Evaluate an expression read from the given file to normal + * form. Optionally enforce that the top-level expression is + * trivial (i.e. doesn't require arbitrary computation). + */ void evalFile(const Path & path, Value & v, bool mustBeTrivial = false); - /* Like `evalFile`, but with an already parsed expression. */ + /** + * Like `evalFile`, but with an already parsed expression. + */ void cacheFile( const Path & path, const Path & resolvedPath, @@ -314,37 +362,52 @@ public: void resetFileCache(); - /* Look up a file in the search path. */ + /** + * Look up a file in the search path. + */ Path findFile(const std::string_view path); Path findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos); - /* If the specified search path element is a URI, download it. */ + /** + * If the specified search path element is a URI, download it. + */ std::pair resolveSearchPathElem(const SearchPathElem & elem); - /* Evaluate an expression to normal form, storing the result in - value `v'. */ + /** + * Evaluate an expression to normal form + * + * @param [out] v The resulting is stored here. + */ void eval(Expr * e, Value & v); - /* Evaluation the expression, then verify that it has the expected - type. */ + /** + * Evaluation the expression, then verify that it has the expected + * type. + */ inline bool evalBool(Env & env, Expr * e); inline bool evalBool(Env & env, Expr * e, const PosIdx pos, std::string_view errorCtx); inline void evalAttrs(Env & env, Expr * e, Value & v, const PosIdx pos, std::string_view errorCtx); - /* If `v' is a thunk, enter it and overwrite `v' with the result - of the evaluation of the thunk. If `v' is a delayed function - application, call the function and overwrite `v' with the - result. Otherwise, this is a no-op. */ + /** + * If `v` is a thunk, enter it and overwrite `v` with the result + * of the evaluation of the thunk. If `v` is a delayed function + * application, call the function and overwrite `v` with the + * result. Otherwise, this is a no-op. + */ inline void forceValue(Value & v, const PosIdx pos); template inline void forceValue(Value & v, Callable getPos); - /* Force a value, then recursively force list elements and - attributes. */ + /** + * Force a value, then recursively force list elements and + * attributes. + */ void forceValueDeep(Value & v); - /* Force `v', and then verify that it has the expected type. */ + /** + * Force `v`, and then verify that it has the expected type. + */ NixInt forceInt(Value & v, const PosIdx pos, std::string_view errorCtx); NixFloat forceFloat(Value & v, const PosIdx pos, std::string_view errorCtx); bool forceBool(Value & v, const PosIdx pos, std::string_view errorCtx); @@ -355,7 +418,10 @@ public: inline void forceAttrs(Value & v, Callable getPos, std::string_view errorCtx); inline void forceList(Value & v, const PosIdx pos, std::string_view errorCtx); - void forceFunction(Value & v, const PosIdx pos, std::string_view errorCtx); // either lambda or primop + /** + * @param v either lambda or primop + */ + void forceFunction(Value & v, const PosIdx pos, std::string_view errorCtx); std::string_view forceString(Value & v, const PosIdx pos, std::string_view errorCtx); std::string_view forceString(Value & v, PathSet & context, const PosIdx pos, std::string_view errorCtx); std::string_view forceStringNoCtx(Value & v, const PosIdx pos, std::string_view errorCtx); @@ -366,17 +432,23 @@ public: void addErrorTrace(Error & e, const PosIdx pos, const char * s, const std::string & s2, bool frame = false) const; public: - /* Return true iff the value `v' denotes a derivation (i.e. a - set with attribute `type = "derivation"'). */ + /** + * @return true iff the value `v` denotes a derivation (i.e. a + * set with attribute `type = "derivation"`). + */ bool isDerivation(Value & v); std::optional tryAttrsToString(const PosIdx pos, Value & v, PathSet & context, bool coerceMore = false, bool copyToStore = true); - /* String coercion. Converts strings, paths and derivations to a - string. If `coerceMore' is set, also converts nulls, integers, - booleans and lists to a string. If `copyToStore' is set, - referenced paths are copied to the Nix store as a side effect. */ + /** + * String coercion. + * + * Converts strings, paths and derivations to a + * string. If `coerceMore` is set, also converts nulls, integers, + * booleans and lists to a string. If `copyToStore` is set, + * referenced paths are copied to the Nix store as a side effect. + */ BackedStringView coerceToString(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx, bool coerceMore = false, bool copyToStore = true, @@ -384,21 +456,31 @@ public: StorePath copyPathToStore(PathSet & context, const Path & path); - /* Path coercion. Converts strings, paths and derivations to a - path. The result is guaranteed to be a canonicalised, absolute - path. Nothing is copied to the store. */ + /** + * Path coercion. + * + * Converts strings, paths and derivations to a + * path. The result is guaranteed to be a canonicalised, absolute + * path. Nothing is copied to the store. + */ Path coerceToPath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx); - /* Like coerceToPath, but the result must be a store path. */ + /** + * Like coerceToPath, but the result must be a store path. + */ StorePath coerceToStorePath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx); public: - /* The base environment, containing the builtin functions and - values. */ + /** + * The base environment, containing the builtin functions and + * values. + */ Env & baseEnv; - /* The same, but used during parsing to resolve variables. */ + /** + * The same, but used during parsing to resolve variables. + */ std::shared_ptr staticBaseEnv; // !!! should be private private: @@ -448,8 +530,10 @@ private: public: - /* Do a deep equality test between two values. That is, list - elements and attributes are compared recursively. */ + /** + * Do a deep equality test between two values. That is, list + * elements and attributes are compared recursively. + */ bool eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx); bool isFunctor(Value & fun); @@ -463,11 +547,15 @@ public: callFunction(fun, 1, args, vRes, pos); } - /* Automatically call a function for which each argument has a - default value or has a binding in the `args' map. */ + /** + * Automatically call a function for which each argument has a + * default value or has a binding in the `args` map. + */ void autoCallFunction(Bindings & args, Value & fun, Value & res); - /* Allocation primitives. */ + /** + * Allocation primitives. + */ inline Value * allocValue(); inline Env & allocEnv(size_t size); @@ -487,10 +575,13 @@ public: void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx); - /* Print statistics. */ + /** + * Print statistics. + */ void printStats(); - /* Realise the given context, and return a mapping from the placeholders + /** + * Realise the given context, and return a mapping from the placeholders * used to construct the associated value to their final store path */ [[nodiscard]] StringMap realiseContext(const PathSet & context); @@ -550,11 +641,15 @@ struct DebugTraceStacker { DebugTrace trace; }; -/* Return a string representing the type of the value `v'. */ +/** + * @return A string representing the type of the value `v`. + */ std::string_view showType(ValueType type); std::string showType(const Value & v); -/* If `path' refers to a directory, then append "/default.nix". */ +/** + * If `path` refers to a directory, then append "/default.nix". + */ Path resolveExprPath(Path path); struct InvalidPathError : EvalError diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 336eb274d..ac396236f 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -125,6 +125,9 @@ static FlakeInput parseFlakeInput(EvalState & state, follows.insert(follows.begin(), lockRootPath.begin(), lockRootPath.end()); input.follows = follows; } else { + // Allow selecting a subset of enum values + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wswitch-enum" switch (attr.value->type()) { case nString: attrs.emplace(state.symbols[attr.name], attr.value->string.s); @@ -139,6 +142,7 @@ static FlakeInput parseFlakeInput(EvalState & state, throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected", state.symbols[attr.name], showType(*attr.value)); } + #pragma GCC diagnostic pop } } catch (Error & e) { e.addTrace( @@ -320,7 +324,7 @@ LockedFlake lockFlake( const FlakeRef & topRef, const LockFlags & lockFlags) { - settings.requireExperimentalFeature(Xp::Flakes); + experimentalFeatureSettings.require(Xp::Flakes); FlakeCache flakeCache; @@ -334,10 +338,14 @@ LockedFlake lockFlake( } try { + if (!fetchSettings.allowDirty && lockFlags.referenceLockFilePath) { + throw Error("reference lock file was provided, but the `allow-dirty` setting is set to false"); + } // FIXME: symlink attack auto oldLockFile = LockFile::read( - flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock"); + lockFlags.referenceLockFilePath.value_or( + flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock")); debug("old lock file: %s", oldLockFile); @@ -619,13 +627,20 @@ LockedFlake lockFlake( debug("new lock file: %s", newLockFile); + auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"; + auto sourcePath = topRef.input.getSourcePath(); + auto outputLockFilePath = sourcePath ? std::optional{*sourcePath + "/" + relPath} : std::nullopt; + if (lockFlags.outputLockFilePath) { + outputLockFilePath = lockFlags.outputLockFilePath; + } + /* Check whether we need to / can write the new lock file. */ - if (!(newLockFile == oldLockFile)) { + if (newLockFile != oldLockFile || lockFlags.outputLockFilePath) { auto diff = LockFile::diff(oldLockFile, newLockFile); if (lockFlags.writeLockFile) { - if (auto sourcePath = topRef.input.getSourcePath()) { + if (outputLockFilePath) { if (auto unlockedInput = newLockFile.isUnlocked()) { if (fetchSettings.warnDirty) warn("will not write lock file of flake '%s' because it has an unlocked input ('%s')", topRef, *unlockedInput); @@ -633,25 +648,24 @@ LockedFlake lockFlake( if (!lockFlags.updateLockFile) throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef); - auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"; - - auto path = *sourcePath + "/" + relPath; - - bool lockFileExists = pathExists(path); + bool lockFileExists = pathExists(*outputLockFilePath); if (lockFileExists) { auto s = chomp(diff); if (s.empty()) - warn("updating lock file '%s'", path); + warn("updating lock file '%s'", *outputLockFilePath); else - warn("updating lock file '%s':\n%s", path, s); + warn("updating lock file '%s':\n%s", *outputLockFilePath, s); } else - warn("creating lock file '%s'", path); + warn("creating lock file '%s'", *outputLockFilePath); - newLockFile.write(path); + newLockFile.write(*outputLockFilePath); std::optional commitMessage = std::nullopt; if (lockFlags.commitLockFile) { + if (lockFlags.outputLockFilePath) { + throw Error("--commit-lock-file and --output-lock-file are currently incompatible"); + } std::string cm; cm = fetchSettings.commitLockFileSummary.get(); diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index 10301d8aa..c1d1b71e5 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "flakeref.hh" @@ -17,7 +18,8 @@ struct FlakeInput; typedef std::map FlakeInputs; -/* FlakeInput is the 'Flake'-level parsed form of the "input" entries +/** + * FlakeInput is the 'Flake'-level parsed form of the "input" entries * in the flake file. * * A FlakeInput is normally constructed by the 'parseFlakeInput' @@ -41,7 +43,12 @@ typedef std::map FlakeInputs; struct FlakeInput { std::optional ref; - bool isFlake = true; // true = process flake to get outputs, false = (fetched) static source path + /** + * true = process flake to get outputs + * + * false = (fetched) static source path + */ + bool isFlake = true; std::optional follows; FlakeInputs overrides; }; @@ -55,23 +62,42 @@ struct ConfigFile void apply(); }; -/* The contents of a flake.nix file. */ +/** + * The contents of a flake.nix file. + */ struct Flake { - FlakeRef originalRef; // the original flake specification (by the user) - FlakeRef resolvedRef; // registry references and caching resolved to the specific underlying flake - FlakeRef lockedRef; // the specific local store result of invoking the fetcher - bool forceDirty = false; // pretend that 'lockedRef' is dirty + /** + * The original flake specification (by the user) + */ + FlakeRef originalRef; + /** + * registry references and caching resolved to the specific underlying flake + */ + FlakeRef resolvedRef; + /** + * the specific local store result of invoking the fetcher + */ + FlakeRef lockedRef; + /** + * pretend that 'lockedRef' is dirty + */ + bool forceDirty = false; std::optional description; std::shared_ptr sourceInfo; FlakeInputs inputs; - ConfigFile config; // 'nixConfig' attribute + /** + * 'nixConfig' attribute + */ + ConfigFile config; ~Flake(); }; Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool allowLookup); -/* Fingerprint of a locked flake; used as a cache key. */ +/** + * Fingerprint of a locked flake; used as a cache key. + */ typedef Hash Fingerprint; struct LockedFlake @@ -84,44 +110,72 @@ struct LockedFlake struct LockFlags { - /* Whether to ignore the existing lock file, creating a new one - from scratch. */ + /** + * Whether to ignore the existing lock file, creating a new one + * from scratch. + */ bool recreateLockFile = false; - /* Whether to update the lock file at all. If set to false, if any - change to the lock file is needed (e.g. when an input has been - added to flake.nix), you get a fatal error. */ + /** + * Whether to update the lock file at all. If set to false, if any + * change to the lock file is needed (e.g. when an input has been + * added to flake.nix), you get a fatal error. + */ bool updateLockFile = true; - /* Whether to write the lock file to disk. If set to true, if the - any changes to the lock file are needed and the flake is not - writable (i.e. is not a local Git working tree or similar), you - get a fatal error. If set to false, Nix will use the modified - lock file in memory only, without writing it to disk. */ + /** + * Whether to write the lock file to disk. If set to true, if the + * any changes to the lock file are needed and the flake is not + * writable (i.e. is not a local Git working tree or similar), you + * get a fatal error. If set to false, Nix will use the modified + * lock file in memory only, without writing it to disk. + */ bool writeLockFile = true; - /* Whether to use the registries to lookup indirect flake - references like 'nixpkgs'. */ + /** + * Whether to use the registries to lookup indirect flake + * references like 'nixpkgs'. + */ std::optional useRegistries = std::nullopt; - /* Whether to apply flake's nixConfig attribute to the configuration */ + /** + * Whether to apply flake's nixConfig attribute to the configuration + */ bool applyNixConfig = false; - /* Whether unlocked flake references (i.e. those without a Git - revision or similar) without a corresponding lock are - allowed. Unlocked flake references with a lock are always - allowed. */ + /** + * Whether unlocked flake references (i.e. those without a Git + * revision or similar) without a corresponding lock are + * allowed. Unlocked flake references with a lock are always + * allowed. + */ bool allowUnlocked = true; - /* Whether to commit changes to flake.lock. */ + /** + * Whether to commit changes to flake.lock. + */ bool commitLockFile = false; - /* Flake inputs to be overridden. */ + /** + * The path to a lock file to read instead of the `flake.lock` file in the top-level flake + */ + std::optional referenceLockFilePath; + + /** + * The path to a lock file to write to instead of the `flake.lock` file in the top-level flake + */ + std::optional outputLockFilePath; + + /** + * Flake inputs to be overridden. + */ std::map inputOverrides; - /* Flake inputs to be updated. This means that any existing lock - for those inputs will be ignored. */ + /** + * Flake inputs to be updated. This means that any existing lock + * for those inputs will be ignored. + */ std::set inputUpdates; }; diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index c4142fc20..a7c9208c0 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "hash.hh" @@ -13,7 +14,8 @@ class Store; typedef std::string FlakeId; -/* A flake reference specifies how to fetch a flake or raw source +/** + * A flake reference specifies how to fetch a flake or raw source * (e.g. from a Git repository). It is created from a URL-like syntax * (e.g. 'github:NixOS/patchelf'), an attrset representation (e.g. '{ * type="github"; owner = "NixOS"; repo = "patchelf"; }'), or a local @@ -32,14 +34,17 @@ typedef std::string FlakeId; * be lazy), but the fetcher can be invoked at any time via the * FlakeRef to ensure the store is populated with this input. */ - struct FlakeRef { - /* Fetcher-specific representation of the input, sufficient to - perform the fetch operation. */ + /** + * Fetcher-specific representation of the input, sufficient to + * perform the fetch operation. + */ fetchers::Input input; - /* sub-path within the fetched input that represents this input */ + /** + * sub-path within the fetched input that represents this input + */ Path subdir; bool operator==(const FlakeRef & other) const; diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh index 02e9bdfbc..0ac731b5d 100644 --- a/src/libexpr/flake/lockfile.hh +++ b/src/libexpr/flake/lockfile.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "flakeref.hh" @@ -15,9 +16,11 @@ typedef std::vector InputPath; struct LockedNode; -/* A node in the lock file. It has outgoing edges to other nodes (its - inputs). Only the root node has this type; all other nodes have - type LockedNode. */ +/** + * A node in the lock file. It has outgoing edges to other nodes (its + * inputs). Only the root node has this type; all other nodes have + * type LockedNode. + */ struct Node : std::enable_shared_from_this { typedef std::variant, InputPath> Edge; @@ -27,7 +30,9 @@ struct Node : std::enable_shared_from_this virtual ~Node() { } }; -/* A non-root node in the lock file. */ +/** + * A non-root node in the lock file. + */ struct LockedNode : Node { FlakeRef lockedRef, originalRef; @@ -62,7 +67,9 @@ struct LockFile void write(const Path & path) const; - /* Check whether this lock file has any unlocked inputs. */ + /** + * Check whether this lock file has any unlocked inputs. + */ std::optional isUnlocked() const; bool operator ==(const LockFile & other) const; @@ -73,7 +80,9 @@ struct LockFile static std::string diff(const LockFile & oldLocks, const LockFile & newLocks); - /* Check that every 'follows' input target exists. */ + /** + * Check that every 'follows' input target exists. + */ void check(); }; diff --git a/src/libexpr/function-trace.hh b/src/libexpr/function-trace.hh index e9a2526bd..91439b0aa 100644 --- a/src/libexpr/function-trace.hh +++ b/src/libexpr/function-trace.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "eval.hh" diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh index bbd2d3c47..584d64ac1 100644 --- a/src/libexpr/get-drvs.hh +++ b/src/libexpr/get-drvs.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "eval.hh" #include "path.hh" @@ -25,7 +26,10 @@ private: mutable std::string outputName; Outputs outputs; - bool failed = false; // set if we get an AssertionError + /** + * Set if we get an AssertionError + */ + bool failed = false; Bindings * attrs = nullptr, * meta = nullptr; @@ -34,7 +38,10 @@ private: bool checkMeta(Value & v); public: - std::string attrPath; /* path towards the derivation */ + /** + * path towards the derivation + */ + std::string attrPath; DrvInfo(EvalState & state) : state(&state) { }; DrvInfo(EvalState & state, std::string attrPath, Bindings * attrs); @@ -46,8 +53,10 @@ public: StorePath requireDrvPath() const; StorePath queryOutPath() const; std::string queryOutputName() const; - /** Return the unordered map of output names to (optional) output paths. - * The "outputs to install" are determined by `meta.outputsToInstall`. */ + /** + * Return the unordered map of output names to (optional) output paths. + * The "outputs to install" are determined by `meta.outputsToInstall`. + */ Outputs queryOutputs(bool withPaths = true, bool onlyOutputsToInstall = false); StringSet queryMetaNames(); @@ -79,8 +88,10 @@ typedef std::list DrvInfos; #endif -/* If value `v' denotes a derivation, return a DrvInfo object - describing it. Otherwise return nothing. */ +/** + * If value `v` denotes a derivation, return a DrvInfo object + * describing it. Otherwise return nothing. + */ std::optional getDerivation(EvalState & state, Value & v, bool ignoreAssertionFailures); diff --git a/src/libexpr/json-to-value.hh b/src/libexpr/json-to-value.hh index 84bec4eba..3b8ec000f 100644 --- a/src/libexpr/json-to-value.hh +++ b/src/libexpr/json-to-value.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "eval.hh" diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 2171e769b..d243b9cec 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -46,3 +46,5 @@ $(foreach i, $(wildcard src/libexpr/flake/*.hh), \ $(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh $(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh + +src/libexpr/primops/fromTOML.o: ERROR_SWITCH_ENUM = diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 4a81eaa47..c2f817c9a 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -21,7 +22,9 @@ MakeError(UndefinedVarError, Error); MakeError(MissingArgumentError, EvalError); MakeError(RestrictedPathError, Error); -/* Position objects. */ +/** + * Position objects. + */ struct Pos { uint32_t line; @@ -132,7 +135,9 @@ class EvalState; struct StaticEnv; -/* An attribute path is a sequence of attribute names. */ +/** + * An attribute path is a sequence of attribute names. + */ struct AttrName { Symbol symbol; @@ -212,11 +217,11 @@ struct ExprVar : Expr or function argument) or from a "with". */ bool fromWith; - /* In the former case, the value is obtained by going `level' + /* In the former case, the value is obtained by going `level` levels up from the current environment and getting the - `displ'th value in that environment. In the latter case, the - value is obtained by getting the attribute named `name' from - the set stored in the environment that is `level' levels up + `displ`th value in that environment. In the latter case, the + value is obtained by getting the attribute named `name` from + the set stored in the environment that is `level` levels up from the current one.*/ Level level; Displacement displ; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 0f75ed9a0..97e615c37 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -469,7 +469,7 @@ expr_simple new ExprString(std::move(path))}); } | URI { - static bool noURLLiterals = settings.isExperimentalFeatureEnabled(Xp::NoUrlLiterals); + static bool noURLLiterals = experimentalFeatureSettings.isEnabled(Xp::NoUrlLiterals); if (noURLLiterals) throw ParseError({ .msg = hintfmt("URL literals are disabled"), @@ -816,7 +816,7 @@ std::pair EvalState::resolveSearchPathElem(const SearchPathEl } else if (hasPrefix(elem.second, "flake:")) { - settings.requireExperimentalFeature(Xp::Flakes); + experimentalFeatureSettings.require(Xp::Flakes); auto flakeRef = parseFlakeRef(elem.second.substr(6), {}, true, false); debug("fetching flake search path element '%s''", elem.second); auto storePath = flakeRef.resolve(store).fetchTree(store).first.storePath; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index fb7fc3ddb..f1bce2fb6 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -254,9 +254,16 @@ static RegisterPrimOp primop_import({ .args = {"path"}, // TODO turn "normal path values" into link below .doc = R"( - Load, parse and return the Nix expression in the file *path*. If - *path* is a directory, the file ` default.nix ` in that directory - is loaded. Evaluation aborts if the file doesn’t exist or contains + Load, parse and return the Nix expression in the file *path*. + + The value *path* can be a path, a string, or an attribute set with an + `__toString` attribute or a `outPath` attribute (as derivations or flake + inputs typically have). + + If *path* is a directory, the file `default.nix` in that directory + is loaded. + + Evaluation aborts if the file doesn’t exist or contains an incorrect Nix expression. `import` implements Nix’s module system: you can put any Nix expression (such as a set or a function) in a separate file, and use it from Nix expressions in @@ -570,6 +577,9 @@ struct CompareValues return v1->integer < v2->fpoint; if (v1->type() != v2->type()) state.error("cannot compare %s with %s", showType(*v1), showType(*v2)).debugThrow(); + // Allow selecting a subset of enum values + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wswitch-enum" switch (v1->type()) { case nInt: return v1->integer < v2->integer; @@ -592,6 +602,7 @@ struct CompareValues } default: state.error("cannot compare %s with %s; values of that type are incomparable", showType(*v1), showType(*v2)).debugThrow(); + #pragma GCC diagnostic pop } } catch (Error & e) { if (!errorCtx.empty()) @@ -1141,13 +1152,13 @@ drvName, Bindings * attrs, Value & v) if (i->name == state.sContentAddressed) { contentAddressed = state.forceBool(*i->value, noPos, context_below); if (contentAddressed) - settings.requireExperimentalFeature(Xp::CaDerivations); + experimentalFeatureSettings.require(Xp::CaDerivations); } else if (i->name == state.sImpure) { isImpure = state.forceBool(*i->value, noPos, context_below); if (isImpure) - settings.requireExperimentalFeature(Xp::ImpureDerivations); + experimentalFeatureSettings.require(Xp::ImpureDerivations); } /* The `args' attribute is special: it supplies the @@ -4114,7 +4125,7 @@ void EvalState::createBaseEnv() if (RegisterPrimOp::primOps) for (auto & primOp : *RegisterPrimOp::primOps) if (!primOp.experimentalFeature - || settings.isExperimentalFeatureEnabled(*primOp.experimentalFeature)) + || experimentalFeatureSettings.isEnabled(*primOp.experimentalFeature)) { addPrimOp({ .fun = primOp.fun, diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh index 1cfb4356b..4ae73fe1f 100644 --- a/src/libexpr/primops.hh +++ b/src/libexpr/primops.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "eval.hh" @@ -22,9 +23,11 @@ struct RegisterPrimOp typedef std::vector PrimOps; static PrimOps * primOps; - /* You can register a constant by passing an arity of 0. fun - will get called during EvalState initialization, so there - may be primops not yet added and builtins is not yet sorted. */ + /** + * You can register a constant by passing an arity of 0. fun + * will get called during EvalState initialization, so there + * may be primops not yet added and builtins is not yet sorted. + */ RegisterPrimOp( std::string name, size_t arity, @@ -37,10 +40,14 @@ struct RegisterPrimOp may wish to use them in limited contexts without globally enabling them. */ -/* Load a ValueInitializer from a DSO and return whatever it initializes */ +/** + * Load a ValueInitializer from a DSO and return whatever it initializes + */ void prim_importNative(EvalState & state, const PosIdx pos, Value * * args, Value & v); -/* Execute a program and parse its output */ +/** + * Execute a program and parse its output + */ void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v); } diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 0f54fedde..0d0e00fa5 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -190,7 +190,7 @@ static void fetchTree( static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - settings.requireExperimentalFeature(Xp::Flakes); + experimentalFeatureSettings.require(Xp::Flakes); fetchTree(state, pos, args, v, std::nullopt, FetchTreeParams { .allowNameArgument = false }); } diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh index 288c15602..967a186dd 100644 --- a/src/libexpr/symbol-table.hh +++ b/src/libexpr/symbol-table.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -9,15 +10,11 @@ namespace nix { -/* Symbol table used by the parser and evaluator to represent and look - up identifiers and attributes efficiently. SymbolTable::create() - converts a string into a symbol. Symbols have the property that - they can be compared efficiently (using an equality test), - because the symbol table stores only one copy of each string. */ - -/* This class mainly exists to give us an operator<< for ostreams. We could also - return plain strings from SymbolTable, but then we'd have to wrap every - instance of a symbol that is fmt()ed, which is inconvenient and error-prone. */ +/** + * This class mainly exists to give us an operator<< for ostreams. We could also + * return plain strings from SymbolTable, but then we'd have to wrap every + * instance of a symbol that is fmt()ed, which is inconvenient and error-prone. + */ class SymbolStr { friend class SymbolTable; @@ -46,6 +43,11 @@ public: friend std::ostream & operator <<(std::ostream & os, const SymbolStr & symbol); }; +/** + * Symbols have the property that they can be compared efficiently + * (using an equality test), because the symbol table stores only one + * copy of each string. + */ class Symbol { friend class SymbolTable; @@ -65,6 +67,10 @@ public: bool operator!=(const Symbol other) const { return id != other.id; } }; +/** + * Symbol table used by the parser and evaluator to represent and look + * up identifiers and attributes efficiently. + */ class SymbolTable { private: @@ -73,6 +79,9 @@ private: public: + /** + * converts a string into a symbol. + */ Symbol create(std::string_view s) { // Most symbols are looked up more than once, so we trade off insertion performance diff --git a/src/libexpr/tests/libexpr.hh b/src/libexpr/tests/libexpr.hh index 8534d9567..69c932f05 100644 --- a/src/libexpr/tests/libexpr.hh +++ b/src/libexpr/tests/libexpr.hh @@ -1,3 +1,6 @@ +#pragma once +///@file + #include #include diff --git a/src/libexpr/tests/value/context.hh b/src/libexpr/tests/value/context.hh index 54d21760e..c0bc97ba3 100644 --- a/src/libexpr/tests/value/context.hh +++ b/src/libexpr/tests/value/context.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include diff --git a/src/libexpr/value-to-json.hh b/src/libexpr/value-to-json.hh index 22f26b790..713356c7f 100644 --- a/src/libexpr/value-to-json.hh +++ b/src/libexpr/value-to-json.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "nixexpr.hh" #include "eval.hh" diff --git a/src/libexpr/value-to-xml.hh b/src/libexpr/value-to-xml.hh index 506f32b6b..ace7ead0f 100644 --- a/src/libexpr/value-to-xml.hh +++ b/src/libexpr/value-to-xml.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "nixexpr.hh" #include "eval.hh" diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 508dbe218..7739f99df 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include @@ -35,9 +36,11 @@ typedef enum { tFloat } InternalType; -// This type abstracts over all actual value types in the language, -// grouping together implementation details like tList*, different function -// types, and types in non-normal form (so thunks and co.) +/** + * This type abstracts over all actual value types in the language, + * grouping together implementation details like tList*, different function + * types, and types in non-normal form (so thunks and co.) + */ typedef enum { nThunk, nInt, @@ -69,38 +72,51 @@ class XMLWriter; typedef int64_t NixInt; typedef double NixFloat; -/* External values must descend from ExternalValueBase, so that +/** + * External values must descend from ExternalValueBase, so that * type-agnostic nix functions (e.g. showType) can be implemented */ class ExternalValueBase { friend std::ostream & operator << (std::ostream & str, const ExternalValueBase & v); protected: - /* Print out the value */ + /** + * Print out the value + */ virtual std::ostream & print(std::ostream & str) const = 0; public: - /* Return a simple string describing the type */ + /** + * Return a simple string describing the type + */ virtual std::string showType() const = 0; - /* Return a string to be used in builtins.typeOf */ + /** + * Return a string to be used in builtins.typeOf + */ virtual std::string typeOf() const = 0; - /* Coerce the value to a string. Defaults to uncoercable, i.e. throws an + /** + * Coerce the value to a string. Defaults to uncoercable, i.e. throws an * error. */ virtual std::string coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const; - /* Compare to another value of the same type. Defaults to uncomparable, + /** + * Compare to another value of the same type. Defaults to uncomparable, * i.e. always false. */ virtual bool operator ==(const ExternalValueBase & b) const; - /* Print the value as JSON. Defaults to unconvertable, i.e. throws an error */ + /** + * Print the value as JSON. Defaults to unconvertable, i.e. throws an error + */ virtual nlohmann::json printValueAsJSON(EvalState & state, bool strict, PathSet & context, bool copyToStore = true) const; - /* Print the value as XML. Defaults to unevaluated */ + /** + * Print the value as XML. Defaults to unevaluated + */ virtual void printValueAsXML(EvalState & state, bool strict, bool location, XMLWriter & doc, PathSet & context, PathSet & drvsSeen, const PosIdx pos) const; @@ -145,26 +161,28 @@ public: NixInt integer; bool boolean; - /* Strings in the evaluator carry a so-called `context' which - is a list of strings representing store paths. This is to - allow users to write things like + /** + * Strings in the evaluator carry a so-called `context` which + * is a list of strings representing store paths. This is to + * allow users to write things like - "--with-freetype2-library=" + freetype + "/lib" + * "--with-freetype2-library=" + freetype + "/lib" - where `freetype' is a derivation (or a source to be copied - to the store). If we just concatenated the strings without - keeping track of the referenced store paths, then if the - string is used as a derivation attribute, the derivation - will not have the correct dependencies in its inputDrvs and - inputSrcs. + * where `freetype` is a derivation (or a source to be copied + * to the store). If we just concatenated the strings without + * keeping track of the referenced store paths, then if the + * string is used as a derivation attribute, the derivation + * will not have the correct dependencies in its inputDrvs and + * inputSrcs. - The semantics of the context is as follows: when a string - with context C is used as a derivation attribute, then the - derivations in C will be added to the inputDrvs of the - derivation, and the other store paths in C will be added to - the inputSrcs of the derivations. + * The semantics of the context is as follows: when a string + * with context C is used as a derivation attribute, then the + * derivations in C will be added to the inputDrvs of the + * derivation, and the other store paths in C will be added to + * the inputSrcs of the derivations. - For canonicity, the store paths should be in sorted order. */ + * For canonicity, the store paths should be in sorted order. + */ struct { const char * s; const char * * context; // must be in sorted order @@ -196,8 +214,10 @@ public: NixFloat fpoint; }; - // Returns the normal type of a Value. This only returns nThunk if the - // Value hasn't been forceValue'd + /** + * Returns the normal type of a Value. This only returns nThunk if + * the Value hasn't been forceValue'd + */ inline ValueType type() const { switch (internalType) { @@ -216,8 +236,10 @@ public: abort(); } - /* After overwriting an app node, be sure to clear pointers in the - Value to ensure that the target isn't kept alive unnecessarily. */ + /** + * After overwriting an app node, be sure to clear pointers in the + * Value to ensure that the target isn't kept alive unnecessarily. + */ inline void clearValue() { app.left = app.right = 0; @@ -365,9 +387,11 @@ public: PosIdx determinePos(const PosIdx pos) const; - /* Check whether forcing this value requires a trivial amount of - computation. In particular, function applications are - non-trivial. */ + /** + * Check whether forcing this value requires a trivial amount of + * computation. In particular, function applications are + * non-trivial. + */ bool isTrivial() const; NixStringContext getContext(const Store &); @@ -413,7 +437,9 @@ typedef std::map ValueVectorMap; #endif -/* A value allocated in traceable memory. */ +/** + * A value allocated in traceable memory. + */ typedef std::shared_ptr RootValue; RootValue allocRootValue(Value * v); diff --git a/src/libexpr/value/context.hh b/src/libexpr/value/context.hh index 721563cba..8719602d8 100644 --- a/src/libexpr/value/context.hh +++ b/src/libexpr/value/context.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "util.hh" #include "comparator.hh" @@ -27,34 +28,37 @@ public: class Store; -/* Plain opaque path to some store object. - - Encoded as just the path: ‘’. -*/ +/** + * Plain opaque path to some store object. + * + * Encoded as just the path: ‘’. + */ struct NixStringContextElem_Opaque { StorePath path; GENERATE_CMP(NixStringContextElem_Opaque, me->path); }; -/* Path to a derivation and its entire build closure. - - The path doesn't just refer to derivation itself and its closure, but - also all outputs of all derivations in that closure (including the - root derivation). - - Encoded in the form ‘=’. -*/ +/** + * Path to a derivation and its entire build closure. + * + * The path doesn't just refer to derivation itself and its closure, but + * also all outputs of all derivations in that closure (including the + * root derivation). + * + * Encoded in the form ‘=’. + */ struct NixStringContextElem_DrvDeep { StorePath drvPath; GENERATE_CMP(NixStringContextElem_DrvDeep, me->drvPath); }; -/* Derivation output. - - Encoded in the form ‘!!’. -*/ +/** + * Derivation output. + * + * Encoded in the form ‘!!’. + */ struct NixStringContextElem_Built { StorePath drvPath; std::string output; @@ -83,11 +87,12 @@ struct NixStringContextElem : _NixStringContextElem_Raw { return static_cast(*this); } - /* Decode a context string, one of: - - ‘’ - - ‘=’ - - ‘!!’ - */ + /** + * Decode a context string, one of: + * - ‘’ + * - ‘=’ + * - ‘!!’ + */ static NixStringContextElem parse(const Store & store, std::string_view s); std::string to_string(const Store & store) const; }; diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh index e41037633..1a14bb023 100644 --- a/src/libfetchers/attrs.hh +++ b/src/libfetchers/attrs.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" diff --git a/src/libfetchers/cache.hh b/src/libfetchers/cache.hh index 3763ee2a6..ae398d040 100644 --- a/src/libfetchers/cache.hh +++ b/src/libfetchers/cache.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "fetchers.hh" diff --git a/src/libfetchers/fetch-settings.hh b/src/libfetchers/fetch-settings.hh index 7049dea30..6108a179c 100644 --- a/src/libfetchers/fetch-settings.hh +++ b/src/libfetchers/fetch-settings.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "config.hh" @@ -75,21 +76,25 @@ struct FetchSettings : public Config Path or URI of the global flake registry. When empty, disables the global flake registry. - )"}; + )", + {}, true, Xp::Flakes}; Setting useRegistries{this, true, "use-registries", - "Whether to use flake registries to resolve flake references."}; + "Whether to use flake registries to resolve flake references.", + {}, true, Xp::Flakes}; Setting acceptFlakeConfig{this, false, "accept-flake-config", - "Whether to accept nix configuration from a flake without prompting."}; + "Whether to accept nix configuration from a flake without prompting.", + {}, true, Xp::Flakes}; Setting commitLockFileSummary{ this, "", "commit-lockfile-summary", R"( The commit summary to use when committing changed flake lock files. If empty, the summary is generated based on the action performed. - )"}; + )", + {}, true, Xp::Flakes}; }; // FIXME: don't use a global variable. diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 95c0f5974..498ad7e4d 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "hash.hh" @@ -20,14 +21,14 @@ struct Tree struct InputScheme; -/* The Input object is generated by a specific fetcher, based on the +/** + * The Input object is generated by a specific fetcher, based on the * user-supplied input attribute in the flake.nix file, and contains * the information that the specific fetcher needs to perform the * actual fetch. The Input object is most commonly created via the * "fromURL()" or "fromAttrs()" static functions which are provided * the url or attrset specified in the flake file. */ - struct Input { friend struct InputScheme; @@ -37,7 +38,9 @@ struct Input bool locked = false; bool direct = true; - /* path of the parent of this input, used for relative path resolution */ + /** + * path of the parent of this input, used for relative path resolution + */ std::optional parent; public: @@ -55,27 +58,35 @@ public: Attrs toAttrs() const; - /* Check whether this is a "direct" input, that is, not - one that goes through a registry. */ + /** + * Check whether this is a "direct" input, that is, not + * one that goes through a registry. + */ bool isDirect() const { return direct; } - /* Check whether this is a "locked" input, that is, - one that contains a commit hash or content hash. */ + /** + * Check whether this is a "locked" input, that is, + * one that contains a commit hash or content hash. + */ bool isLocked() const { return locked; } - /* Check whether the input carries all necessary info required - for cache insertion and substitution. - These fields are used to uniquely identify cached trees - within the "tarball TTL" window without necessarily - indicating that the input's origin is unchanged. */ + /** + * Check whether the input carries all necessary info required + * for cache insertion and substitution. + * These fields are used to uniquely identify cached trees + * within the "tarball TTL" window without necessarily + * indicating that the input's origin is unchanged. + */ bool hasAllInfo() const; bool operator ==(const Input & other) const; bool contains(const Input & other) const; - /* Fetch the input into the Nix store, returning the location in - the Nix store and the locked input. */ + /** + * Fetch the input into the Nix store, returning the location in + * the Nix store and the locked input. + */ std::pair fetch(ref store) const; Input applyOverrides( @@ -104,7 +115,8 @@ public: }; -/* The InputScheme represents a type of fetcher. Each fetcher +/** + * The InputScheme represents a type of fetcher. Each fetcher * registers with nix at startup time. When processing an input for a * flake, each scheme is given an opportunity to "recognize" that * input from the url or attributes in the flake file's specification diff --git a/src/libfetchers/registry.hh b/src/libfetchers/registry.hh index 260a2c460..f57ab1e6b 100644 --- a/src/libfetchers/registry.hh +++ b/src/libfetchers/registry.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "fetchers.hh" diff --git a/src/libmain/common-args.hh b/src/libmain/common-args.hh index f180d83ce..e7ed0d934 100644 --- a/src/libmain/common-args.hh +++ b/src/libmain/common-args.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "args.hh" diff --git a/src/libmain/loggers.hh b/src/libmain/loggers.hh index f3c759193..e5721420c 100644 --- a/src/libmain/loggers.hh +++ b/src/libmain/loggers.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index 024259584..6600ec177 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -72,6 +72,7 @@ private: uint64_t corruptedPaths = 0, untrustedPaths = 0; bool active = true; + bool paused = false; bool haveUpdate = true; }; @@ -120,6 +121,18 @@ public: updateThread.join(); } + void pause() override { + state_.lock()->paused = true; + writeToStderr("\r\e[K"); + } + + void resume() override { + state_.lock()->paused = false; + writeToStderr("\r\e[K"); + state_.lock()->haveUpdate = true; + updateCV.notify_one(); + } + bool isVerbose() override { return printBuildLogs; @@ -339,7 +352,7 @@ public: auto nextWakeup = std::chrono::milliseconds::max(); state.haveUpdate = false; - if (!state.active) return nextWakeup; + if (state.paused || !state.active) return nextWakeup; std::string line; diff --git a/src/libmain/progress-bar.hh b/src/libmain/progress-bar.hh index 3a76f8448..c3c6e3833 100644 --- a/src/libmain/progress-bar.hh +++ b/src/libmain/progress-bar.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "logging.hh" diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 1715374a6..7a9e83c6c 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "util.hh" #include "args.hh" @@ -24,7 +25,9 @@ public: int handleExceptions(const std::string & programName, std::function fun); -/* Don't forget to call initPlugins() after settings are initialized! */ +/** + * Don't forget to call initPlugins() after settings are initialized! + */ void initNix(); void parseCmdLine(int argc, char * * argv, @@ -35,7 +38,9 @@ void parseCmdLine(const std::string & programName, const Strings & args, void printVersion(const std::string & programName); -/* Ugh. No better place to put this. */ +/** + * Ugh. No better place to put this. + */ void printGCWarning(); class Store; @@ -74,11 +79,15 @@ struct LegacyArgs : public MixCommonArgs }; -/* Show the manual page for the specified program. */ +/** + * Show the manual page for the specified program. + */ void showManPage(const std::string & name); -/* The constructor of this class starts a pager if stdout is a - terminal and $PAGER is set. Stdout is redirected to the pager. */ +/** + * The constructor of this class starts a pager if stdout is a + * terminal and $PAGER is set. Stdout is redirected to the pager. + */ class RunPager { public: @@ -109,28 +118,34 @@ struct PrintFreed }; -/* Install a SIGSEGV handler to detect stack overflows. */ +/** + * Install a SIGSEGV handler to detect stack overflows. + */ void detectStackOverflow(); -/* Pluggable behavior to run in case of a stack overflow. - - Default value: defaultStackOverflowHandler. - - This is called by the handler installed by detectStackOverflow(). - - This gives Nix library consumers a limit opportunity to report the error - condition. The handler should exit the process. - See defaultStackOverflowHandler() for a reference implementation. - - NOTE: Use with diligence, because this runs in the signal handler, with very - limited stack space and a potentially a corrupted heap, all while the failed - thread is blocked indefinitely. All functions called must be reentrant. */ +/** + * Pluggable behavior to run in case of a stack overflow. + * + * Default value: defaultStackOverflowHandler. + * + * This is called by the handler installed by detectStackOverflow(). + * + * This gives Nix library consumers a limit opportunity to report the error + * condition. The handler should exit the process. + * See defaultStackOverflowHandler() for a reference implementation. + * + * NOTE: Use with diligence, because this runs in the signal handler, with very + * limited stack space and a potentially a corrupted heap, all while the failed + * thread is blocked indefinitely. All functions called must be reentrant. + */ extern std::function stackOverflowHandler; -/* The default, robust implementation of stackOverflowHandler. - - Prints an error message directly to stderr using a syscall instead of the - logger. Exits the process immediately after. */ +/** + * The default, robust implementation of stackOverflowHandler. + * + * Prints an error message directly to stderr using a syscall instead of the + * logger. Exits the process immediately after. + */ void defaultStackOverflowHandler(siginfo_t * info, void * ctx); } diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index abd92a83c..49f271d24 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "crypto.hh" #include "store-api.hh" @@ -16,19 +17,40 @@ struct BinaryCacheStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; - const Setting compression{(StoreConfig*) this, "xz", "compression", "NAR compression method ('xz', 'bzip2', 'gzip', 'zstd', or 'none')"}; - const Setting writeNARListing{(StoreConfig*) this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"}; - const Setting writeDebugInfo{(StoreConfig*) this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"}; - const Setting secretKeyFile{(StoreConfig*) this, "", "secret-key", "path to secret key used to sign the binary cache"}; - const Setting localNarCache{(StoreConfig*) this, "", "local-nar-cache", "path to a local cache of NARs"}; + const Setting compression{(StoreConfig*) this, "xz", "compression", + "NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`)."}; + + const Setting writeNARListing{(StoreConfig*) this, false, "write-nar-listing", + "Whether to write a JSON file that lists the files in each NAR."}; + + const Setting writeDebugInfo{(StoreConfig*) this, false, "index-debug-info", + R"( + Whether to index DWARF debug info files by build ID. This allows [`dwarffs`](https://github.com/edolstra/dwarffs) to + fetch debug info on demand + )"}; + + const Setting secretKeyFile{(StoreConfig*) this, "", "secret-key", + "Path to the secret key used to sign the binary cache."}; + + const Setting localNarCache{(StoreConfig*) this, "", "local-nar-cache", + "Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`."}; + const Setting parallelCompression{(StoreConfig*) this, false, "parallel-compression", - "enable multi-threading compression for NARs, available for xz and zstd only currently"}; + "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`."}; + const Setting compressionLevel{(StoreConfig*) this, -1, "compression-level", - "specify 'preset level' of compression to be used with NARs: " - "meaning and accepted range of values depends on compression method selected, " - "other than -1 which we reserve to indicate Nix defaults should be used"}; + R"( + The *preset level* to be used when compressing NARs. + The meaning and accepted values depend on the compression method selected. + `-1` specifies that the default compression level should be used. + )"}; }; + +/** + * @note subclasses must implement at least one of the two + * virtual getFile() methods. + */ class BinaryCacheStore : public virtual BinaryCacheStoreConfig, public virtual Store, public virtual LogStore @@ -58,14 +80,15 @@ public: std::string && data, const std::string & mimeType); - /* Note: subclasses must implement at least one of the two - following getFile() methods. */ - - /* Dump the contents of the specified file to a sink. */ + /** + * Dump the contents of the specified file to a sink. + */ virtual void getFile(const std::string & path, Sink & sink); - /* Fetch the specified file and call the specified callback with - the result. A subclass may implement this asynchronously. */ + /** + * Fetch the specified file and call the specified callback with + * the result. A subclass may implement this asynchronously. + */ virtual void getFile( const std::string & path, Callback> callback) noexcept; diff --git a/src/libstore/build-result.hh b/src/libstore/build-result.hh index a5749cf33..27d1a1b6c 100644 --- a/src/libstore/build-result.hh +++ b/src/libstore/build-result.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "realisation.hh" #include "derived-path.hh" @@ -11,9 +12,12 @@ namespace nix { struct BuildResult { - /* Note: don't remove status codes, and only add new status codes - at the end of the list, to prevent client/server - incompatibilities in the nix-store --serve protocol. */ + /** + * @note This is directly used in the nix-store --serve protocol. + * That means we need to worry about compatability across versions. + * Therefore, don't remove status codes, and only add new status + * codes at the end of the list. + */ enum Status { Built = 0, Substituted, @@ -21,8 +25,10 @@ struct BuildResult PermanentFailure, InputRejected, OutputRejected, - TransientFailure, // possibly transient - CachedFailure, // no longer used + /// possibly transient + TransientFailure, + /// no longer used + CachedFailure, TimedOut, MiscFailure, DependencyFailed, @@ -32,7 +38,12 @@ struct BuildResult NoSubstituters, } status = MiscFailure; - // FIXME: include entire ErrorInfo object. + /** + * Information about the error if the build failed. + * + * @todo This should be an entire ErrorInfo object, not just a + * string, for richer information. + */ std::string errorMsg; std::string toString() const { @@ -52,33 +63,46 @@ struct BuildResult case LogLimitExceeded: return "LogLimitExceeded"; case NotDeterministic: return "NotDeterministic"; case ResolvesToAlreadyValid: return "ResolvesToAlreadyValid"; + case NoSubstituters: return "NoSubstituters"; default: return "Unknown"; }; }(); return strStatus + ((errorMsg == "") ? "" : " : " + errorMsg); } - /* How many times this build was performed. */ + /** + * How many times this build was performed. + */ unsigned int timesBuilt = 0; - /* If timesBuilt > 1, whether some builds did not produce the same - result. (Note that 'isNonDeterministic = false' does not mean - the build is deterministic, just that we don't have evidence of - non-determinism.) */ + /** + * If timesBuilt > 1, whether some builds did not produce the same + * result. (Note that 'isNonDeterministic = false' does not mean + * the build is deterministic, just that we don't have evidence of + * non-determinism.) + */ bool isNonDeterministic = false; - /* The derivation we built or the store path we substituted. */ + /** + * The derivation we built or the store path we substituted. + */ DerivedPath path; - /* For derivations, a mapping from the names of the wanted outputs - to actual paths. */ + /** + * For derivations, a mapping from the names of the wanted outputs + * to actual paths. + */ DrvOutputs builtOutputs; - /* The start/stop times of the build (or one of the rounds, if it - was repeated). */ + /** + * The start/stop times of the build (or one of the rounds, if it + * was repeated). + */ time_t startTime = 0, stopTime = 0; - /* User and system CPU time the build took. */ + /** + * User and system CPU time the build took. + */ std::optional cpuUser, cpuSystem; bool success() diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 38b73d531..26faf8c8e 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -199,10 +199,10 @@ void DerivationGoal::haveDerivation() parsedDrv = std::make_unique(drvPath, *drv); if (!drv->type().hasKnownOutputPaths()) - settings.requireExperimentalFeature(Xp::CaDerivations); + experimentalFeatureSettings.require(Xp::CaDerivations); if (!drv->type().isPure()) { - settings.requireExperimentalFeature(Xp::ImpureDerivations); + experimentalFeatureSettings.require(Xp::ImpureDerivations); for (auto & [outputName, output] : drv->outputs) { auto randomPath = StorePath::random(outputPathName(drv->name, outputName)); @@ -336,7 +336,7 @@ void DerivationGoal::gaveUpOnSubstitution() for (auto & i : dynamic_cast(drv.get())->inputDrvs) { /* Ensure that pure, non-fixed-output derivations don't depend on impure derivations. */ - if (settings.isExperimentalFeatureEnabled(Xp::ImpureDerivations) && drv->type().isPure() && !drv->type().isFixed()) { + if (experimentalFeatureSettings.isEnabled(Xp::ImpureDerivations) && drv->type().isPure() && !drv->type().isFixed()) { auto inputDrv = worker.evalStore.readDerivation(i.first); if (!inputDrv.type().isPure()) throw Error("pure derivation '%s' depends on impure derivation '%s'", @@ -477,7 +477,7 @@ void DerivationGoal::inputsRealised() ca.fixed /* Can optionally resolve if fixed, which is good for avoiding unnecessary rebuilds. */ - ? settings.isExperimentalFeatureEnabled(Xp::CaDerivations) + ? experimentalFeatureSettings.isEnabled(Xp::CaDerivations) /* Must resolve if floating and there are any inputs drvs. */ : true); @@ -488,7 +488,7 @@ void DerivationGoal::inputsRealised() }, drvType.raw()); if (resolveDrv && !fullDrv.inputDrvs.empty()) { - settings.requireExperimentalFeature(Xp::CaDerivations); + experimentalFeatureSettings.require(Xp::CaDerivations); /* We are be able to resolve this derivation based on the now-known results of dependencies. If so, we become a @@ -911,7 +911,11 @@ void DerivationGoal::buildDone() msg += line; msg += "\n"; } - msg += fmt("For full logs, run '" ANSI_BOLD "nix log %s" ANSI_NORMAL "'.", + auto nixLogCommand = experimentalFeatureSettings.isEnabled(Xp::NixCommand) + ? "nix log" + : "nix-store -l"; + msg += fmt("For full logs, run '" ANSI_BOLD "%s %s" ANSI_NORMAL "'.", + nixLogCommand, worker.store.printStorePath(drvPath)); } @@ -1352,7 +1356,7 @@ std::pair DerivationGoal::checkPathValidity() }; } auto drvOutput = DrvOutput{info.outputHash, i.first}; - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { if (auto real = worker.store.queryRealisation(drvOutput)) { info.known = { .path = real->outPath, diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 707e38b4b..3a6f0c2d9 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "parsed-derivations.hh" #include "lock.hh" @@ -15,8 +16,10 @@ struct HookInstance; typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; -/* Unless we are repairing, we don't both to test validity and just assume it, - so the choices are `Absent` or `Valid`. */ +/** + * Unless we are repairing, we don't both to test validity and just assume it, + * so the choices are `Absent` or `Valid`. + */ enum struct PathStatus { Corrupt, Absent, @@ -26,11 +29,15 @@ enum struct PathStatus { struct InitialOutputStatus { StorePath path; PathStatus status; - /* Valid in the store, and additionally non-corrupt if we are repairing */ + /** + * Valid in the store, and additionally non-corrupt if we are repairing + */ bool isValid() const { return status == PathStatus::Valid; } - /* Merely present, allowed to be corrupt */ + /** + * Merely present, allowed to be corrupt + */ bool isPresent() const { return status == PathStatus::Corrupt || status == PathStatus::Valid; @@ -45,59 +52,87 @@ struct InitialOutput { struct DerivationGoal : public Goal { - /* Whether to use an on-disk .drv file. */ + /** + * Whether to use an on-disk .drv file. + */ bool useDerivation; - /* The path of the derivation. */ + /** The path of the derivation. */ StorePath drvPath; - /* The goal for the corresponding resolved derivation */ + /** + * The goal for the corresponding resolved derivation + */ std::shared_ptr resolvedDrvGoal; - /* The specific outputs that we need to build. Empty means all of - them. */ + /** + * The specific outputs that we need to build. Empty means all of + * them. + */ OutputsSpec wantedOutputs; - /* Mapping from input derivations + output names to actual store - paths. This is filled in by waiteeDone() as each dependency - finishes, before inputsRealised() is reached, */ + /** + * Mapping from input derivations + output names to actual store + * paths. This is filled in by waiteeDone() as each dependency + * finishes, before inputsRealised() is reached. + */ std::map, StorePath> inputDrvOutputs; - /* Whether additional wanted outputs have been added. */ + /** + * Whether additional wanted outputs have been added. + */ bool needRestart = false; - /* Whether to retry substituting the outputs after building the - inputs. This is done in case of an incomplete closure. */ + /** + * Whether to retry substituting the outputs after building the + * inputs. This is done in case of an incomplete closure. + */ bool retrySubstitution = false; - /* Whether we've retried substitution, in which case we won't try - again. */ + /** + * Whether we've retried substitution, in which case we won't try + * again. + */ bool retriedSubstitution = false; - /* The derivation stored at drvPath. */ + /** + * The derivation stored at drvPath. + */ std::unique_ptr drv; std::unique_ptr parsedDrv; - /* The remainder is state held during the build. */ + /** + * The remainder is state held during the build. + */ - /* Locks on (fixed) output paths. */ + /** + * Locks on (fixed) output paths. + */ PathLocks outputLocks; - /* All input paths (that is, the union of FS closures of the - immediate input paths). */ + /** + * All input paths (that is, the union of FS closures of the + * immediate input paths). + */ StorePathSet inputPaths; std::map initialOutputs; - /* File descriptor for the log file. */ + /** + * File descriptor for the log file. + */ AutoCloseFD fdLogFile; std::shared_ptr logFileSink, logSink; - /* Number of bytes received from the builder's stdout/stderr. */ + /** + * Number of bytes received from the builder's stdout/stderr. + */ unsigned long logSize; - /* The most recent log lines. */ + /** + * The most recent log lines. + */ std::list logTail; std::string currentLogLine; @@ -105,10 +140,14 @@ struct DerivationGoal : public Goal std::string currentHookLine; - /* The build hook. */ + /** + * The build hook. + */ std::unique_ptr hook; - /* The sort of derivation we are building. */ + /** + * The sort of derivation we are building. + */ DerivationType derivationType; typedef void (DerivationGoal::*GoalState)(); @@ -120,12 +159,16 @@ struct DerivationGoal : public Goal std::unique_ptr act; - /* Activity that denotes waiting for a lock. */ + /** + * Activity that denotes waiting for a lock. + */ std::unique_ptr actLock; std::map builderActivities; - /* The remote machine on which we're building. */ + /** + * The remote machine on which we're building. + */ std::string machineName; DerivationGoal(const StorePath & drvPath, @@ -142,10 +185,14 @@ struct DerivationGoal : public Goal void work() override; - /* Add wanted outputs to an already existing derivation goal. */ + /** + * Add wanted outputs to an already existing derivation goal. + */ void addWantedOutputs(const OutputsSpec & outputs); - /* The states. */ + /** + * The states. + */ void getDerivation(); void loadDerivation(); void haveDerivation(); @@ -159,28 +206,42 @@ struct DerivationGoal : public Goal void resolvedFinished(); - /* Is the build hook willing to perform the build? */ + /** + * Is the build hook willing to perform the build? + */ HookReply tryBuildHook(); virtual int getChildStatus(); - /* Check that the derivation outputs all exist and register them - as valid. */ + /** + * Check that the derivation outputs all exist and register them + * as valid. + */ virtual DrvOutputs registerOutputs(); - /* Open a log file and a pipe to it. */ + /** + * Open a log file and a pipe to it. + */ Path openLogFile(); - /* Sign the newly built realisation if the store allows it */ + /** + * Sign the newly built realisation if the store allows it + */ virtual void signRealisation(Realisation&) {} - /* Close the log file. */ + /** + * Close the log file. + */ void closeLogFile(); - /* Close the read side of the logger pipe. */ + /** + * Close the read side of the logger pipe. + */ virtual void closeReadPipes(); - /* Cleanup hooks for buildDone() */ + /** + * Cleanup hooks for buildDone() + */ virtual void cleanupHookFinally(); virtual void cleanupPreChildKill(); virtual void cleanupPostChildKill(); @@ -190,30 +251,40 @@ struct DerivationGoal : public Goal virtual bool isReadDesc(int fd); - /* Callback used by the worker to write to the log. */ + /** + * Callback used by the worker to write to the log. + */ void handleChildOutput(int fd, std::string_view data) override; void handleEOF(int fd) override; void flushLine(); - /* Wrappers around the corresponding Store methods that first consult the - derivation. This is currently needed because when there is no drv file - there also is no DB entry. */ + /** + * Wrappers around the corresponding Store methods that first consult the + * derivation. This is currently needed because when there is no drv file + * there also is no DB entry. + */ std::map> queryPartialDerivationOutputMap(); OutputPathMap queryDerivationOutputMap(); - /* Update 'initialOutputs' to determine the current status of the - outputs of the derivation. Also returns a Boolean denoting - whether all outputs are valid and non-corrupt, and a - 'DrvOutputs' structure containing the valid and wanted - outputs. */ + /** + * Update 'initialOutputs' to determine the current status of the + * outputs of the derivation. Also returns a Boolean denoting + * whether all outputs are valid and non-corrupt, and a + * 'DrvOutputs' structure containing the valid and wanted + * outputs. + */ std::pair checkPathValidity(); - /* Aborts if any output is not valid or corrupt, and otherwise - returns a 'DrvOutputs' structure containing the wanted - outputs. */ + /** + * Aborts if any output is not valid or corrupt, and otherwise + * returns a 'DrvOutputs' structure containing the wanted + * outputs. + */ DrvOutputs assertPathValidity(); - /* Forcibly kill the child process, if any. */ + /** + * Forcibly kill the child process, if any. + */ virtual void killChild(); void repairClosure(); diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh index e4b044790..697ddb283 100644 --- a/src/libstore/build/drv-output-substitution-goal.hh +++ b/src/libstore/build/drv-output-substitution-goal.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" #include "goal.hh" @@ -10,24 +11,34 @@ namespace nix { class Worker; -// Substitution of a derivation output. -// This is done in three steps: -// 1. Fetch the output info from a substituter -// 2. Substitute the corresponding output path -// 3. Register the output info +/** + * Substitution of a derivation output. + * This is done in three steps: + * 1. Fetch the output info from a substituter + * 2. Substitute the corresponding output path + * 3. Register the output info + */ class DrvOutputSubstitutionGoal : public Goal { - // The drv output we're trying to substitue + /** + * The drv output we're trying to substitue + */ DrvOutput id; - // The realisation corresponding to the given output id. - // Will be filled once we can get it. + /** + * The realisation corresponding to the given output id. + * Will be filled once we can get it. + */ std::shared_ptr outputInfo; - /* The remaining substituters. */ + /** + * The remaining substituters. + */ std::list> subs; - /* The current substituter. */ + /** + * The current substituter. + */ std::shared_ptr sub; struct DownloadState @@ -38,7 +49,9 @@ class DrvOutputSubstitutionGoal : public Goal { std::shared_ptr downloadState; - /* Whether a substituter failed. */ + /** + * Whether a substituter failed. + */ bool substituterFailed = false; public: diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 776eb86bc..f4bf6f38b 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "store-api.hh" @@ -6,11 +7,15 @@ namespace nix { -/* Forward definition. */ +/** + * Forward definition. + */ struct Goal; class Worker; -/* A pointer to a goal. */ +/** + * A pointer to a goal. + */ typedef std::shared_ptr GoalPtr; typedef std::weak_ptr WeakGoalPtr; @@ -18,48 +23,72 @@ struct CompareGoalPtrs { bool operator() (const GoalPtr & a, const GoalPtr & b) const; }; -/* Set of goals. */ +/** + * Set of goals. + */ typedef std::set Goals; typedef std::set> WeakGoals; -/* A map of paths to goals (and the other way around). */ +/** + * A map of paths to goals (and the other way around). + */ typedef std::map WeakGoalMap; struct Goal : public std::enable_shared_from_this { typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode; - /* Backlink to the worker. */ + /** + * Backlink to the worker. + */ Worker & worker; - /* Goals that this goal is waiting for. */ + /** + * Goals that this goal is waiting for. + */ Goals waitees; - /* Goals waiting for this one to finish. Must use weak pointers - here to prevent cycles. */ + /** + * Goals waiting for this one to finish. Must use weak pointers + * here to prevent cycles. + */ WeakGoals waiters; - /* Number of goals we are/were waiting for that have failed. */ + /** + * Number of goals we are/were waiting for that have failed. + */ size_t nrFailed = 0; - /* Number of substitution goals we are/were waiting for that - failed because there are no substituters. */ + /** + * Number of substitution goals we are/were waiting for that + * failed because there are no substituters. + */ size_t nrNoSubstituters = 0; - /* Number of substitution goals we are/were waiting for that - failed because they had unsubstitutable references. */ + /** + * Number of substitution goals we are/were waiting for that + * failed because they had unsubstitutable references. + */ size_t nrIncompleteClosure = 0; - /* Name of this goal for debugging purposes. */ + /** + * Name of this goal for debugging purposes. + */ std::string name; - /* Whether the goal is finished. */ + /** + * Whether the goal is finished. + */ ExitCode exitCode = ecBusy; - /* Build result. */ + /** + * Build result. + */ BuildResult buildResult; - /* Exception containing an error message, if any. */ + /** + * Exception containing an error message, if any. + */ std::optional ex; Goal(Worker & worker, DerivedPath path) @@ -95,9 +124,11 @@ struct Goal : public std::enable_shared_from_this return name; } - /* Callback in case of a timeout. It should wake up its waiters, - get rid of any running child processes that are being monitored - by the worker (important!), etc. */ + /** + * Callback in case of a timeout. It should wake up its waiters, + * get rid of any running child processes that are being monitored + * by the worker (important!), etc. + */ virtual void timedOut(Error && ex) = 0; virtual std::string key() = 0; diff --git a/src/libstore/build/hook-instance.hh b/src/libstore/build/hook-instance.hh index 9e8cff128..d84f62877 100644 --- a/src/libstore/build/hook-instance.hh +++ b/src/libstore/build/hook-instance.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "logging.hh" #include "serialise.hh" @@ -7,16 +8,24 @@ namespace nix { struct HookInstance { - /* Pipes for talking to the build hook. */ + /** + * Pipes for talking to the build hook. + */ Pipe toHook; - /* Pipe for the hook's standard output/error. */ + /** + * Pipe for the hook's standard output/error. + */ Pipe fromHook; - /* Pipe for the builder's standard output/error. */ + /** + * Pipe for the builder's standard output/error. + */ Pipe builderOut; - /* The process ID of the hook. */ + /** + * The process ID of the hook. + */ Pid pid; FdSink sink; diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 3484d2044..58d6901d3 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -413,7 +413,7 @@ void LocalDerivationGoal::startBuilder() ) { #if __linux__ - settings.requireExperimentalFeature(Xp::Cgroups); + experimentalFeatureSettings.require(Xp::Cgroups); auto cgroupFS = getCgroupFS(); if (!cgroupFS) @@ -1415,12 +1415,15 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo virtual void addBuildLog(const StorePath & path, std::string_view log) override { unsupported("addBuildLog"); } + + std::optional isTrustedClient() override + { return NotTrusted; } }; void LocalDerivationGoal::startDaemon() { - settings.requireExperimentalFeature(Xp::RecursiveNix); + experimentalFeatureSettings.require(Xp::RecursiveNix); Store::Params params; params["path-info-cache-size"] = "0"; @@ -1467,7 +1470,7 @@ void LocalDerivationGoal::startDaemon() FdSink to(remote.get()); try { daemon::processConnection(store, from, to, - daemon::NotTrusted, daemon::Recursive); + NotTrusted, daemon::Recursive); debug("terminated daemon connection"); } catch (SysError &) { ignoreException(); @@ -2280,7 +2283,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() bool discardReferences = false; if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { if (auto udr = get(*structuredAttrs, "unsafeDiscardReferences")) { - settings.requireExperimentalFeature(Xp::DiscardReferences); + experimentalFeatureSettings.require(Xp::DiscardReferences); if (auto output = get(*udr, outputName)) { if (!output->is_boolean()) throw Error("attribute 'unsafeDiscardReferences.\"%s\"' of derivation '%s' must be a Boolean", outputName, drvPath.to_string()); @@ -2700,7 +2703,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() }, .outPath = newInfo.path }; - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && drv->type().isPure()) { signRealisation(thisRealisation); diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index c9ecc8828..42d32a31a 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "derivation-goal.hh" #include "local-store.hh" @@ -9,49 +10,75 @@ struct LocalDerivationGoal : public DerivationGoal { LocalStore & getLocalStore(); - /* User selected for running the builder. */ + /** + * User selected for running the builder. + */ std::unique_ptr buildUser; - /* The process ID of the builder. */ + /** + * The process ID of the builder. + */ Pid pid; - /* The cgroup of the builder, if any. */ + /** + * The cgroup of the builder, if any. + */ std::optional cgroup; - /* The temporary directory. */ + /** + * The temporary directory. + */ Path tmpDir; - /* The path of the temporary directory in the sandbox. */ + /** + * The path of the temporary directory in the sandbox. + */ Path tmpDirInSandbox; - /* Master side of the pseudoterminal used for the builder's - standard output/error. */ + /** + * Master side of the pseudoterminal used for the builder's + * standard output/error. + */ AutoCloseFD builderOut; - /* Pipe for synchronising updates to the builder namespaces. */ + /** + * Pipe for synchronising updates to the builder namespaces. + */ Pipe userNamespaceSync; - /* The mount namespace and user namespace of the builder, used to add additional - paths to the sandbox as a result of recursive Nix calls. */ + /** + * The mount namespace and user namespace of the builder, used to add additional + * paths to the sandbox as a result of recursive Nix calls. + */ AutoCloseFD sandboxMountNamespace; AutoCloseFD sandboxUserNamespace; - /* On Linux, whether we're doing the build in its own user - namespace. */ + /** + * On Linux, whether we're doing the build in its own user + * namespace. + */ bool usingUserNamespace = true; - /* Whether we're currently doing a chroot build. */ + /** + * Whether we're currently doing a chroot build. + */ bool useChroot = false; Path chrootRootDir; - /* RAII object to delete the chroot directory. */ + /** + * RAII object to delete the chroot directory. + */ std::shared_ptr autoDelChroot; - /* Whether to run the build in a private network namespace. */ + /** + * Whether to run the build in a private network namespace. + */ bool privateNetwork = false; - /* Stuff we need to pass to initChild(). */ + /** + * Stuff we need to pass to initChild(). + */ struct ChrootPath { Path source; bool optional; @@ -70,30 +97,35 @@ struct LocalDerivationGoal : public DerivationGoal SandboxProfile additionalSandboxProfile; #endif - /* Hash rewriting. */ + /** + * Hash rewriting. + */ StringMap inputRewrites, outputRewrites; typedef map RedirectedOutputs; RedirectedOutputs redirectedOutputs; - /* The outputs paths used during the build. - - - Input-addressed derivations or fixed content-addressed outputs are - sometimes built when some of their outputs already exist, and can not - be hidden via sandboxing. We use temporary locations instead and - rewrite after the build. Otherwise the regular predetermined paths are - put here. - - - Floating content-addressed derivations do not know their final build - output paths until the outputs are hashed, so random locations are - used, and then renamed. The randomness helps guard against hidden - self-references. + /** + * The outputs paths used during the build. + * + * - Input-addressed derivations or fixed content-addressed outputs are + * sometimes built when some of their outputs already exist, and can not + * be hidden via sandboxing. We use temporary locations instead and + * rewrite after the build. Otherwise the regular predetermined paths are + * put here. + * + * - Floating content-addressed derivations do not know their final build + * output paths until the outputs are hashed, so random locations are + * used, and then renamed. The randomness helps guard against hidden + * self-references. */ OutputPathMap scratchOutputs; - /* Path registration info from the previous round, if we're - building multiple times. Since this contains the hash, it - allows us to compare whether two rounds produced the same - result. */ + /** + * Path registration info from the previous round, if we're + * building multiple times. Since this contains the hash, it + * allows us to compare whether two rounds produced the same + * result. + */ std::map prevInfos; uid_t sandboxUid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 1000 : 0) : buildUser->getUID(); } @@ -101,25 +133,37 @@ struct LocalDerivationGoal : public DerivationGoal const static Path homeDir; - /* The recursive Nix daemon socket. */ + /** + * The recursive Nix daemon socket. + */ AutoCloseFD daemonSocket; - /* The daemon main thread. */ + /** + * The daemon main thread. + */ std::thread daemonThread; - /* The daemon worker threads. */ + /** + * The daemon worker threads. + */ std::vector daemonWorkerThreads; - /* Paths that were added via recursive Nix calls. */ + /** + * Paths that were added via recursive Nix calls. + */ StorePathSet addedPaths; - /* Realisations that were added via recursive Nix calls. */ + /** + * Realisations that were added via recursive Nix calls. + */ std::set addedDrvOutputs; - /* Recursive Nix calls are only allowed to build or realize paths - in the original input closure or added via a recursive Nix call - (so e.g. you can't do 'nix-store -r /nix/store/' where - /nix/store/ is some arbitrary path in a binary cache). */ + /** + * Recursive Nix calls are only allowed to build or realize paths + * in the original input closure or added via a recursive Nix call + * (so e.g. you can't do 'nix-store -r /nix/store/' where + * /nix/store/ is some arbitrary path in a binary cache). + */ bool isAllowed(const StorePath & path) { return inputPaths.count(path) || addedPaths.count(path); @@ -137,55 +181,81 @@ struct LocalDerivationGoal : public DerivationGoal virtual ~LocalDerivationGoal() override; - /* Whether we need to perform hash rewriting if there are valid output paths. */ + /** + * Whether we need to perform hash rewriting if there are valid output paths. + */ bool needsHashRewrite(); - /* The additional states. */ + /** + * The additional states. + */ void tryLocalBuild() override; - /* Start building a derivation. */ + /** + * Start building a derivation. + */ void startBuilder(); - /* Fill in the environment for the builder. */ + /** + * Fill in the environment for the builder. + */ void initEnv(); - /* Setup tmp dir location. */ + /** + * Setup tmp dir location. + */ void initTmpDir(); - /* Write a JSON file containing the derivation attributes. */ + /** + * Write a JSON file containing the derivation attributes. + */ void writeStructuredAttrs(); void startDaemon(); void stopDaemon(); - /* Add 'path' to the set of paths that may be referenced by the - outputs, and make it appear in the sandbox. */ + /** + * Add 'path' to the set of paths that may be referenced by the + * outputs, and make it appear in the sandbox. + */ void addDependency(const StorePath & path); - /* Make a file owned by the builder. */ + /** + * Make a file owned by the builder. + */ void chownToBuilder(const Path & path); int getChildStatus() override; - /* Run the builder's process. */ + /** + * Run the builder's process. + */ void runChild(); - /* Check that the derivation outputs all exist and register them - as valid. */ + /** + * Check that the derivation outputs all exist and register them + * as valid. + */ DrvOutputs registerOutputs() override; void signRealisation(Realisation &) override; - /* Check that an output meets the requirements specified by the - 'outputChecks' attribute (or the legacy - '{allowed,disallowed}{References,Requisites}' attributes). */ + /** + * Check that an output meets the requirements specified by the + * 'outputChecks' attribute (or the legacy + * '{allowed,disallowed}{References,Requisites}' attributes). + */ void checkOutputs(const std::map & outputs); - /* Close the read side of the logger pipe. */ + /** + * Close the read side of the logger pipe. + */ void closeReadPipes() override; - /* Cleanup hooks for buildDone() */ + /** + * Cleanup hooks for buildDone() + */ void cleanupHookFinally() override; void cleanupPreChildKill() override; void cleanupPostChildKill() override; @@ -195,24 +265,36 @@ struct LocalDerivationGoal : public DerivationGoal bool isReadDesc(int fd) override; - /* Delete the temporary directory, if we have one. */ + /** + * Delete the temporary directory, if we have one. + */ void deleteTmpDir(bool force); - /* Forcibly kill the child process, if any. */ + /** + * Forcibly kill the child process, if any. + */ void killChild() override; - /* Kill any processes running under the build user UID or in the - cgroup of the build. */ + /** + * Kill any processes running under the build user UID or in the + * cgroup of the build. + */ void killSandbox(bool getStats); - /* Create alternative path calculated from but distinct from the - input, so we can avoid overwriting outputs (or other store paths) - that already exist. */ + /** + * Create alternative path calculated from but distinct from the + * input, so we can avoid overwriting outputs (or other store paths) + * that already exist. + */ StorePath makeFallbackPath(const StorePath & path); - /* Make a path to another based on the output name along with the - derivation hash. */ - /* FIXME add option to randomize, so we can audit whether our - rewrites caught everything */ + + /** + * Make a path to another based on the output name along with the + * derivation hash. + * + * @todo Add option to randomize, so we can audit whether our + * rewrites caught everything + */ StorePath makeFallbackPath(std::string_view outputName); }; diff --git a/src/libstore/build/personality.hh b/src/libstore/build/personality.hh index 30e4f4062..91b730fab 100644 --- a/src/libstore/build/personality.hh +++ b/src/libstore/build/personality.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index a73f8e666..c2b7fc95a 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "lock.hh" #include "store-api.hh" @@ -10,38 +11,58 @@ class Worker; struct PathSubstitutionGoal : public Goal { - /* The store path that should be realised through a substitute. */ + /** + * The store path that should be realised through a substitute. + */ StorePath storePath; - /* The path the substituter refers to the path as. This will be - different when the stores have different names. */ + /** + * The path the substituter refers to the path as. This will be + * different when the stores have different names. + */ std::optional subPath; - /* The remaining substituters. */ + /** + * The remaining substituters. + */ std::list> subs; - /* The current substituter. */ + /** + * The current substituter. + */ std::shared_ptr sub; - /* Whether a substituter failed. */ + /** + * Whether a substituter failed. + */ bool substituterFailed = false; - /* Path info returned by the substituter's query info operation. */ + /** + * Path info returned by the substituter's query info operation. + */ std::shared_ptr info; - /* Pipe for the substituter's standard output. */ + /** + * Pipe for the substituter's standard output. + */ Pipe outPipe; - /* The substituter thread. */ + /** + * The substituter thread. + */ std::thread thr; std::promise promise; - /* Whether to try to repair a valid path. */ + /** + * Whether to try to repair a valid path. + */ RepairFlag repair; - /* Location where we're downloading the substitute. Differs from - storePath when doing a repair. */ + /** + * Location where we're downloading the substitute. Differs from + * storePath when doing a repair. + */ Path destPath; std::unique_ptr> maintainExpectedSubstitutions, @@ -50,7 +71,9 @@ struct PathSubstitutionGoal : public Goal typedef void (PathSubstitutionGoal::*GoalState)(); GoalState state; - /* Content address for recomputing store path */ + /** + * Content address for recomputing store path + */ std::optional ca; void done( @@ -64,16 +87,20 @@ public: void timedOut(Error && ex) override { abort(); }; + /** + * We prepend "a$" to the key name to ensure substitution goals + * happen before derivation goals. + */ std::string key() override { - /* "a$" ensures substitution goals happen before derivation - goals. */ return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath); } void work() override; - /* The states. */ + /** + * The states. + */ void init(); void tryNext(); void gotInfo(); @@ -81,7 +108,9 @@ public: void tryToRun(); void finished(); - /* Callback used by the worker to write to the log. */ + /** + * Callback used by the worker to write to the log. + */ void handleChildOutput(int fd, std::string_view data) override; void handleEOF(int fd) override; diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 6d68d3cf1..48a1a27fa 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "lock.hh" @@ -16,24 +17,29 @@ struct DerivationGoal; struct PathSubstitutionGoal; class DrvOutputSubstitutionGoal; -/* Workaround for not being able to declare a something like - - class PathSubstitutionGoal : public Goal; - - even when Goal is a complete type. - - This is still a static cast. The purpose of exporting it is to define it in - a place where `PathSubstitutionGoal` is concrete, and use it in a place where it - is opaque. */ +/** + * Workaround for not being able to declare a something like + * + * ```c++ + * class PathSubstitutionGoal : public Goal; + * ``` + * even when Goal is a complete type. + * + * This is still a static cast. The purpose of exporting it is to define it in + * a place where `PathSubstitutionGoal` is concrete, and use it in a place where it + * is opaque. + */ GoalPtr upcast_goal(std::shared_ptr subGoal); GoalPtr upcast_goal(std::shared_ptr subGoal); typedef std::chrono::time_point steady_time_point; -/* A mapping used to remember for each child process to what goal it - belongs, and file descriptors for receiving log data and output - path creation commands. */ +/** + * A mapping used to remember for each child process to what goal it + * belongs, and file descriptors for receiving log data and output + * path creation commands. + */ struct Child { WeakGoalPtr goal; @@ -41,14 +47,19 @@ struct Child std::set fds; bool respectTimeouts; bool inBuildSlot; - steady_time_point lastOutput; /* time we last got output on stdout/stderr */ + /** + * Time we last got output on stdout/stderr + */ + steady_time_point lastOutput; steady_time_point timeStarted; }; /* Forward definition. */ struct HookInstance; -/* The worker class. */ +/** + * The worker class. + */ class Worker { private: @@ -56,38 +67,58 @@ private: /* Note: the worker should only have strong pointers to the top-level goals. */ - /* The top-level goals of the worker. */ + /** + * The top-level goals of the worker. + */ Goals topGoals; - /* Goals that are ready to do some work. */ + /** + * Goals that are ready to do some work. + */ WeakGoals awake; - /* Goals waiting for a build slot. */ + /** + * Goals waiting for a build slot. + */ WeakGoals wantingToBuild; - /* Child processes currently running. */ + /** + * Child processes currently running. + */ std::list children; - /* Number of build slots occupied. This includes local builds and - substitutions but not remote builds via the build hook. */ + /** + * Number of build slots occupied. This includes local builds and + * substitutions but not remote builds via the build hook. + */ unsigned int nrLocalBuilds; - /* Maps used to prevent multiple instantiations of a goal for the - same derivation / path. */ + /** + * Maps used to prevent multiple instantiations of a goal for the + * same derivation / path. + */ std::map> derivationGoals; std::map> substitutionGoals; std::map> drvOutputSubstitutionGoals; - /* Goals waiting for busy paths to be unlocked. */ + /** + * Goals waiting for busy paths to be unlocked. + */ WeakGoals waitingForAnyGoal; - /* Goals sleeping for a few seconds (polling a lock). */ + /** + * Goals sleeping for a few seconds (polling a lock). + */ WeakGoals waitingForAWhile; - /* Last time the goals in `waitingForAWhile' where woken up. */ + /** + * Last time the goals in `waitingForAWhile` where woken up. + */ steady_time_point lastWokenUp; - /* Cache for pathContentsGood(). */ + /** + * Cache for pathContentsGood(). + */ std::map pathContentsGoodCache; public: @@ -96,17 +127,25 @@ public: const Activity actDerivations; const Activity actSubstitutions; - /* Set if at least one derivation had a BuildError (i.e. permanent - failure). */ + /** + * Set if at least one derivation had a BuildError (i.e. permanent + * failure). + */ bool permanentFailure; - /* Set if at least one derivation had a timeout. */ + /** + * Set if at least one derivation had a timeout. + */ bool timedOut; - /* Set if at least one derivation fails with a hash mismatch. */ + /** + * Set if at least one derivation fails with a hash mismatch. + */ bool hashMismatch; - /* Set if at least one derivation is not deterministic in check mode. */ + /** + * Set if at least one derivation is not deterministic in check mode. + */ bool checkMismatch; Store & store; @@ -128,16 +167,22 @@ public: uint64_t expectedNarSize = 0; uint64_t doneNarSize = 0; - /* Whether to ask the build hook if it can build a derivation. If - it answers with "decline-permanently", we don't try again. */ + /** + * Whether to ask the build hook if it can build a derivation. If + * it answers with "decline-permanently", we don't try again. + */ bool tryBuildHook = true; Worker(Store & store, Store & evalStore); ~Worker(); - /* Make a goal (with caching). */ + /** + * Make a goal (with caching). + */ - /* derivation goal */ + /** + * derivation goal + */ private: std::shared_ptr makeDerivationGoalCommon( const StorePath & drvPath, const OutputsSpec & wantedOutputs, @@ -150,56 +195,80 @@ public: const StorePath & drvPath, const BasicDerivation & drv, const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal); - /* substitution goal */ + /** + * substitution goal + */ std::shared_ptr makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); std::shared_ptr makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); - /* Remove a dead goal. */ + /** + * Remove a dead goal. + */ void removeGoal(GoalPtr goal); - /* Wake up a goal (i.e., there is something for it to do). */ + /** + * Wake up a goal (i.e., there is something for it to do). + */ void wakeUp(GoalPtr goal); - /* Return the number of local build and substitution processes - currently running (but not remote builds via the build - hook). */ + /** + * Return the number of local build and substitution processes + * currently running (but not remote builds via the build + * hook). + */ unsigned int getNrLocalBuilds(); - /* Registers a running child process. `inBuildSlot' means that - the process counts towards the jobs limit. */ + /** + * Registers a running child process. `inBuildSlot` means that + * the process counts towards the jobs limit. + */ void childStarted(GoalPtr goal, const std::set & fds, bool inBuildSlot, bool respectTimeouts); - /* Unregisters a running child process. `wakeSleepers' should be - false if there is no sense in waking up goals that are sleeping - because they can't run yet (e.g., there is no free build slot, - or the hook would still say `postpone'). */ + /** + * Unregisters a running child process. `wakeSleepers` should be + * false if there is no sense in waking up goals that are sleeping + * because they can't run yet (e.g., there is no free build slot, + * or the hook would still say `postpone`). + */ void childTerminated(Goal * goal, bool wakeSleepers = true); - /* Put `goal' to sleep until a build slot becomes available (which - might be right away). */ + /** + * Put `goal` to sleep until a build slot becomes available (which + * might be right away). + */ void waitForBuildSlot(GoalPtr goal); - /* Wait for any goal to finish. Pretty indiscriminate way to - wait for some resource that some other goal is holding. */ + /** + * Wait for any goal to finish. Pretty indiscriminate way to + * wait for some resource that some other goal is holding. + */ void waitForAnyGoal(GoalPtr goal); - /* Wait for a few seconds and then retry this goal. Used when - waiting for a lock held by another process. This kind of - polling is inefficient, but POSIX doesn't really provide a way - to wait for multiple locks in the main select() loop. */ + /** + * Wait for a few seconds and then retry this goal. Used when + * waiting for a lock held by another process. This kind of + * polling is inefficient, but POSIX doesn't really provide a way + * to wait for multiple locks in the main select() loop. + */ void waitForAWhile(GoalPtr goal); - /* Loop until the specified top-level goals have finished. */ + /** + * Loop until the specified top-level goals have finished. + */ void run(const Goals & topGoals); - /* Wait for input to become available. */ + /** + * Wait for input to become available. + */ void waitForInput(); unsigned int exitStatus(); - /* Check whether the given valid path exists and has the right - contents. */ + /** + * Check whether the given valid path exists and has the right + * contents. + */ bool pathContentsGood(const StorePath & path); void markContentsGood(const StorePath & path); diff --git a/src/libstore/builtins.hh b/src/libstore/builtins.hh index 66597e456..d201fb3ac 100644 --- a/src/libstore/builtins.hh +++ b/src/libstore/builtins.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "derivations.hh" diff --git a/src/libstore/builtins/buildenv.hh b/src/libstore/builtins/buildenv.hh index a018de3af..0923c2adb 100644 --- a/src/libstore/builtins/buildenv.hh +++ b/src/libstore/builtins/buildenv.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "derivations.hh" #include "store-api.hh" diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index f6a6f5140..368a7ec48 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -1,48 +1,85 @@ #pragma once +///@file #include #include "hash.hh" +#include "comparator.hh" namespace nix { +/** + * An enumeration of the ways we can serialize file system objects. + */ enum struct FileIngestionMethod : uint8_t { + /** + * Flat-file hashing. Directly ingest the contents of a single file + */ Flat = false, + /** + * Recursive (or NAR) hashing. Serializes the file-system object in Nix + * Archive format and ingest that + */ Recursive = true }; +/** + * Somewhat obscure, used by \ref Derivation derivations and + * `builtins.toFile` currently. + */ struct TextHash { + /** + * Hash of the contents of the text/file. + */ Hash hash; + + GENERATE_CMP(TextHash, me->hash); }; -/// Pair of a hash, and how the file system was ingested +/** + * For path computed by makeFixedOutputPath. + */ struct FixedOutputHash { + /** + * How the file system objects are serialized + */ FileIngestionMethod method; + /** + * Hash of that serialization + */ Hash hash; + std::string printMethodAlgo() const; + + GENERATE_CMP(FixedOutputHash, me->method, me->hash); }; -/* - We've accumulated several types of content-addressed paths over the years; - fixed-output derivations support multiple hash algorithms and serialisation - methods (flat file vs NAR). Thus, ‘ca’ has one of the following forms: - - * ‘text:sha256:’: For paths - computed by makeTextPath() / addTextToStore(). - - * ‘fixed:::’: For paths computed by - makeFixedOutputPath() / addToStore(). -*/ +/** + * We've accumulated several types of content-addressed paths over the + * years; fixed-output derivations support multiple hash algorithms and + * serialisation methods (flat file vs NAR). Thus, ‘ca’ has one of the + * following forms: + * + * - ‘text:sha256:’: For paths + * computed by Store::makeTextPath() / Store::addTextToStore(). + * + * - ‘fixed:::’: For paths computed by + * Store::makeFixedOutputPath() / Store::addToStore(). + */ typedef std::variant< - TextHash, // for paths computed by makeTextPath() / addTextToStore - FixedOutputHash // for path computed by makeFixedOutputPath + TextHash, + FixedOutputHash > ContentAddress; -/* Compute the prefix to the hash algorithm which indicates how the files were - ingested. */ +/** + * Compute the prefix to the hash algorithm which indicates how the + * files were ingested. + */ std::string makeFileIngestionPrefix(const FileIngestionMethod m); -/* Compute the content-addressability assertion (ValidPathInfo::ca) - for paths created by makeFixedOutputPath() / addToStore(). */ +/** + * Compute the content-addressability assertion (ValidPathInfo::ca) for + * paths created by Store::makeFixedOutputPath() / Store::addToStore(). + */ std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash); std::string renderContentAddress(ContentAddress ca); @@ -65,6 +102,11 @@ struct FixedOutputHashMethod { HashType hashType; }; +/** + * Ways of content addressing but not a complete ContentAddress. + * + * A ContentAddress without a Hash. + */ typedef std::variant< TextHashMethod, FixedOutputHashMethod diff --git a/src/libstore/crypto.hh b/src/libstore/crypto.hh index 03f85c103..35216d470 100644 --- a/src/libstore/crypto.hh +++ b/src/libstore/crypto.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" @@ -11,8 +12,10 @@ struct Key std::string name; std::string key; - /* Construct Key from a string in the format - ‘:’. */ + /** + * Construct Key from a string in the format + * ‘:’. + */ Key(std::string_view s); std::string to_string() const; @@ -28,7 +31,9 @@ struct SecretKey : Key { SecretKey(std::string_view s); - /* Return a detached signature of the given string. */ + /** + * Return a detached signature of the given string. + */ std::string signDetached(std::string_view s) const; PublicKey toPublicKey() const; @@ -52,8 +57,10 @@ private: typedef std::map PublicKeys; -/* Return true iff ‘sig’ is a correct signature over ‘data’ using one - of the given public keys. */ +/** + * @return true iff ‘sig’ is a correct signature over ‘data’ using one + * of the given public keys. + */ bool verifyDetached(const std::string & data, const std::string & sig, const PublicKeys & publicKeys); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 7f8b0f905..63898f8dc 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -231,10 +231,10 @@ struct ClientSettings try { if (name == "ssh-auth-sock") // obsolete ; - else if (name == settings.experimentalFeatures.name) { + else if (name == experimentalFeatureSettings.experimentalFeatures.name) { // We don’t want to forward the experimental features to // the daemon, as that could cause some pretty weird stuff - if (parseFeatures(tokenizeString(value)) != settings.experimentalFeatures.get()) + if (parseFeatures(tokenizeString(value)) != experimentalFeatureSettings.experimentalFeatures.get()) debug("Ignoring the client-specified experimental features"); } else if (name == settings.pluginFiles.name) { if (tokenizeString(value) != settings.pluginFiles.get()) @@ -1032,6 +1032,15 @@ void processConnection( if (GET_PROTOCOL_MINOR(clientVersion) >= 33) to << nixVersion; + if (GET_PROTOCOL_MINOR(clientVersion) >= 35) { + // We and the underlying store both need to trust the client for + // it to be trusted. + auto temp = trusted + ? store->isTrustedClient() + : std::optional { NotTrusted }; + worker_proto::write(*store, to, temp); + } + /* Send startup error messages to the client. */ tunnelLogger->startWork(); diff --git a/src/libstore/daemon.hh b/src/libstore/daemon.hh index 8c765615c..1964c0d99 100644 --- a/src/libstore/daemon.hh +++ b/src/libstore/daemon.hh @@ -1,11 +1,11 @@ #pragma once +///@file #include "serialise.hh" #include "store-api.hh" namespace nix::daemon { -enum TrustedFlag : bool { NotTrusted = false, Trusted = true }; enum RecursiveFlag : bool { NotRecursive = false, Recursive = true }; void processConnection( diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 05dc9a3cc..abdfb1978 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -221,7 +221,7 @@ static DerivationOutput parseDerivationOutput(const Store & store, } const auto hashType = parseHashType(hashAlgo); if (hash == "impure") { - settings.requireExperimentalFeature(Xp::ImpureDerivations); + experimentalFeatureSettings.require(Xp::ImpureDerivations); assert(pathS == ""); return DerivationOutput::Impure { .method = std::move(method), @@ -236,7 +236,7 @@ static DerivationOutput parseDerivationOutput(const Store & store, }, }; } else { - settings.requireExperimentalFeature(Xp::CaDerivations); + experimentalFeatureSettings.require(Xp::CaDerivations); assert(pathS == ""); return DerivationOutput::CAFloating { .method = std::move(method), @@ -889,6 +889,67 @@ std::optional Derivation::tryResolve( return resolved; } + +void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const +{ + assert(drvPath.isDerivation()); + std::string drvName(drvPath.name()); + drvName = drvName.substr(0, drvName.size() - drvExtension.size()); + + if (drvName != name) { + throw Error("Derivation '%s' has name '%s' which does not match its path", store.printStorePath(drvPath), name); + } + + auto envHasRightPath = [&](const StorePath & actual, const std::string & varName) + { + auto j = env.find(varName); + if (j == env.end() || store.parseStorePath(j->second) != actual) + throw Error("derivation '%s' has incorrect environment variable '%s', should be '%s'", + store.printStorePath(drvPath), varName, store.printStorePath(actual)); + }; + + + // Don't need the answer, but do this anyways to assert is proper + // combination. The code below is more general and naturally allows + // combinations that are currently prohibited. + type(); + + std::optional hashesModulo; + for (auto & i : outputs) { + std::visit(overloaded { + [&](const DerivationOutput::InputAddressed & doia) { + if (!hashesModulo) { + // somewhat expensive so we do lazily + hashesModulo = hashDerivationModulo(store, *this, true); + } + auto currentOutputHash = get(hashesModulo->hashes, i.first); + if (!currentOutputHash) + throw Error("derivation '%s' has unexpected output '%s' (local-store / hashesModulo) named '%s'", + store.printStorePath(drvPath), store.printStorePath(doia.path), i.first); + StorePath recomputed = store.makeOutputPath(i.first, *currentOutputHash, drvName); + if (doia.path != recomputed) + throw Error("derivation '%s' has incorrect output '%s', should be '%s'", + store.printStorePath(drvPath), store.printStorePath(doia.path), store.printStorePath(recomputed)); + envHasRightPath(doia.path, i.first); + }, + [&](const DerivationOutput::CAFixed & dof) { + StorePath path = store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName); + envHasRightPath(path, i.first); + }, + [&](const DerivationOutput::CAFloating &) { + /* Nothing to check */ + }, + [&](const DerivationOutput::Deferred &) { + /* Nothing to check */ + }, + [&](const DerivationOutput::Impure &) { + /* Nothing to check */ + }, + }, i.second.raw()); + } +} + + const Hash impureOutputHash = hashString(htSHA256, "impure"); nlohmann::json DerivationOutput::toJSON( @@ -916,10 +977,79 @@ nlohmann::json DerivationOutput::toJSON( return res; } + +DerivationOutput DerivationOutput::fromJSON( + const Store & store, std::string_view drvName, std::string_view outputName, + const nlohmann::json & _json) +{ + std::set keys; + auto json = (std::map) _json; + + for (const auto & [key, _] : json) + keys.insert(key); + + auto methodAlgo = [&]() -> std::pair { + std::string hashAlgo = json["hashAlgo"]; + auto method = FileIngestionMethod::Flat; + if (hashAlgo.substr(0, 2) == "r:") { + method = FileIngestionMethod::Recursive; + hashAlgo = hashAlgo.substr(2); + } + auto hashType = parseHashType(hashAlgo); + return { method, hashType }; + }; + + if (keys == (std::set { "path" })) { + return DerivationOutput::InputAddressed { + .path = store.parseStorePath((std::string) json["path"]), + }; + } + + else if (keys == (std::set { "path", "hashAlgo", "hash" })) { + auto [method, hashType] = methodAlgo(); + auto dof = DerivationOutput::CAFixed { + .hash = { + .method = method, + .hash = Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType), + }, + }; + if (dof.path(store, drvName, outputName) != store.parseStorePath((std::string) json["path"])) + throw Error("Path doesn't match derivation output"); + return dof; + } + + else if (keys == (std::set { "hashAlgo" })) { + auto [method, hashType] = methodAlgo(); + return DerivationOutput::CAFloating { + .method = method, + .hashType = hashType, + }; + } + + else if (keys == (std::set { })) { + return DerivationOutput::Deferred {}; + } + + else if (keys == (std::set { "hashAlgo", "impure" })) { + auto [method, hashType] = methodAlgo(); + return DerivationOutput::Impure { + .method = method, + .hashType = hashType, + }; + } + + else { + throw Error("invalid JSON for derivation output"); + } +} + + nlohmann::json Derivation::toJSON(const Store & store) const { nlohmann::json res = nlohmann::json::object(); + res["name"] = name; + { nlohmann::json & outputsObj = res["outputs"]; outputsObj = nlohmann::json::object(); @@ -950,4 +1080,43 @@ nlohmann::json Derivation::toJSON(const Store & store) const return res; } + +Derivation Derivation::fromJSON( + const Store & store, + const nlohmann::json & json) +{ + Derivation res; + + res.name = json["name"]; + + { + auto & outputsObj = json["outputs"]; + for (auto & [outputName, output] : outputsObj.items()) { + res.outputs.insert_or_assign( + outputName, + DerivationOutput::fromJSON(store, res.name, outputName, output)); + } + } + + { + auto & inputsList = json["inputSrcs"]; + for (auto & input : inputsList) + res.inputSrcs.insert(store.parseStorePath(static_cast(input))); + } + + { + auto & inputDrvsObj = json["inputDrvs"]; + for (auto & [inputDrvPath, inputOutputs] : inputDrvsObj.items()) + res.inputDrvs[store.parseStorePath(inputDrvPath)] = + static_cast(inputOutputs); + } + + res.platform = json["system"]; + res.builder = json["builder"]; + res.args = json["args"]; + res.env = json["env"]; + + return res; +} + } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 8456b29e7..ccdde36ca 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "path.hh" #include "types.hh" @@ -6,6 +7,7 @@ #include "content-address.hh" #include "repair-flag.hh" #include "sync.hh" +#include "comparator.hh" #include #include @@ -17,43 +19,83 @@ class Store; /* Abstract syntax of derivations. */ -/* The traditional non-fixed-output derivation type. */ +/** + * The traditional non-fixed-output derivation type. + */ struct DerivationOutputInputAddressed { StorePath path; + + GENERATE_CMP(DerivationOutputInputAddressed, me->path); }; -/* Fixed-output derivations, whose output paths are content addressed - according to that fixed output. */ +/** + * Fixed-output derivations, whose output paths are content + * addressed according to that fixed output. + */ struct DerivationOutputCAFixed { - FixedOutputHash hash; /* hash used for expected hash computation */ + /** + * hash used for expected hash computation + */ + FixedOutputHash hash; + + /** + * Return the \ref StorePath "store path" corresponding to this output + * + * @param drvName The name of the derivation this is an output of, without the `.drv`. + * @param outputName The name of this output. + */ StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const; + + GENERATE_CMP(DerivationOutputCAFixed, me->hash); }; -/* Floating-output derivations, whose output paths are content addressed, but - not fixed, and so are dynamically calculated from whatever the output ends - up being. */ +/** + * Floating-output derivations, whose output paths are content + * addressed, but not fixed, and so are dynamically calculated from + * whatever the output ends up being. + * */ struct DerivationOutputCAFloating { - /* information used for expected hash computation */ + /** + * How the file system objects will be serialized for hashing + */ FileIngestionMethod method; + + /** + * How the serialization will be hashed + */ HashType hashType; + + GENERATE_CMP(DerivationOutputCAFloating, me->method, me->hashType); }; -/* Input-addressed output which depends on a (CA) derivation whose hash isn't - * known yet. +/** + * Input-addressed output which depends on a (CA) derivation whose hash + * isn't known yet. */ -struct DerivationOutputDeferred {}; +struct DerivationOutputDeferred { + GENERATE_CMP(DerivationOutputDeferred); +}; -/* Impure output which is moved to a content-addressed location (like - CAFloating) but isn't registered as a realization. +/** + * Impure output which is moved to a content-addressed location (like + * CAFloating) but isn't registered as a realization. */ struct DerivationOutputImpure { - /* information used for expected hash computation */ + /** + * How the file system objects will be serialized for hashing + */ FileIngestionMethod method; + + /** + * How the serialization will be hashed + */ HashType hashType; + + GENERATE_CMP(DerivationOutputImpure, me->method, me->hashType); }; typedef std::variant< @@ -64,6 +106,9 @@ typedef std::variant< DerivationOutputImpure > _DerivationOutputRaw; +/** + * A single output of a BasicDerivation (and Derivation). + */ struct DerivationOutput : _DerivationOutputRaw { using Raw = _DerivationOutputRaw; @@ -75,9 +120,12 @@ struct DerivationOutput : _DerivationOutputRaw using Deferred = DerivationOutputDeferred; using Impure = DerivationOutputImpure; - /* Note, when you use this function you should make sure that you're passing - the right derivation name. When in doubt, you should use the safer - interface provided by BasicDerivation::outputsAndOptPaths */ + /** + * \note when you use this function you should make sure that you're + * passing the right derivation name. When in doubt, you should use + * the safer interface provided by + * BasicDerivation::outputsAndOptPaths + */ std::optional path(const Store & store, std::string_view drvName, std::string_view outputName) const; inline const Raw & raw() const { @@ -88,30 +136,70 @@ struct DerivationOutput : _DerivationOutputRaw const Store & store, std::string_view drvName, std::string_view outputName) const; + static DerivationOutput fromJSON( + const Store & store, + std::string_view drvName, + std::string_view outputName, + const nlohmann::json & json); }; typedef std::map DerivationOutputs; -/* These are analogues to the previous DerivationOutputs data type, but they - also contains, for each output, the (optional) store path in which it would - be written. To calculate values of these types, see the corresponding - functions in BasicDerivation */ +/** + * These are analogues to the previous DerivationOutputs data type, + * but they also contains, for each output, the (optional) store + * path in which it would be written. To calculate values of these + * types, see the corresponding functions in BasicDerivation. + */ typedef std::map>> DerivationOutputsAndOptPaths; -/* For inputs that are sub-derivations, we specify exactly which - output IDs we are interested in. */ +/** + * For inputs that are sub-derivations, we specify exactly which + * output IDs we are interested in. + */ typedef std::map DerivationInputs; +/** + * Input-addressed derivation types + */ struct DerivationType_InputAddressed { + /** + * True iff the derivation type can't be determined statically, + * for instance because it (transitively) depends on a content-addressed + * derivation. + */ bool deferred; }; +/** + * Content-addressed derivation types + */ struct DerivationType_ContentAddressed { + /** + * Whether the derivation should be built safely inside a sandbox. + */ bool sandboxed; + /** + * Whether the derivation's outputs' content-addresses are "fixed" + * or "floating. + * + * - Fixed: content-addresses are written down as part of the + * derivation itself. If the outputs don't end up matching the + * build fails. + * + * - Floating: content-addresses are not written down, we do not + * know them until we perform the build. + */ bool fixed; }; +/** + * Impure derivation type + * + * This is similar at buil-time to the content addressed, not standboxed, not fixed + * type, but has some restrictions on its usage. + */ struct DerivationType_Impure { }; @@ -128,30 +216,38 @@ struct DerivationType : _DerivationTypeRaw { using ContentAddressed = DerivationType_ContentAddressed; using Impure = DerivationType_Impure; - /* Do the outputs of the derivation have paths calculated from their content, - or from the derivation itself? */ + /** + * Do the outputs of the derivation have paths calculated from their + * content, or from the derivation itself? + */ bool isCA() const; - /* Is the content of the outputs fixed a-priori via a hash? Never true for - non-CA derivations. */ + /** + * Is the content of the outputs fixed a priori via a hash? + * Never true for non-CA derivations. + */ bool isFixed() const; - /* Whether the derivation is fully sandboxed. If false, the - sandbox is opened up, e.g. the derivation has access to the - network. Note that whether or not we actually sandbox the - derivation is controlled separately. Always true for non-CA - derivations. */ + /** + * Whether the derivation is fully sandboxed. If false, the sandbox + * is opened up, e.g. the derivation has access to the network. Note + * that whether or not we actually sandbox the derivation is + * controlled separately. Always true for non-CA derivations. + */ bool isSandboxed() const; - /* Whether the derivation is expected to produce the same result - every time, and therefore it only needs to be built once. This - is only false for derivations that have the attribute '__impure - = true'. */ + /** + * Whether the derivation is expected to produce the same result + * every time, and therefore it only needs to be built once. This is + * only false for derivations that have the attribute '__impure = + * true'. + */ bool isPure() const; - /* Does the derivation knows its own output paths? - Only true when there's no floating-ca derivation involved in the - closure, or if fixed output. + /** + * Does the derivation knows its own output paths? + * Only true when there's no floating-ca derivation involved in the + * closure, or if fixed output. */ bool hasKnownOutputPaths() const; @@ -162,8 +258,14 @@ struct DerivationType : _DerivationTypeRaw { struct BasicDerivation { - DerivationOutputs outputs; /* keyed on symbolic IDs */ - StorePathSet inputSrcs; /* inputs that are sources */ + /** + * keyed on symbolic IDs + */ + DerivationOutputs outputs; + /** + * inputs that are sources + */ + StorePathSet inputSrcs; std::string platform; Path builder; Strings args; @@ -175,128 +277,195 @@ struct BasicDerivation bool isBuiltin() const; - /* Return true iff this is a fixed-output derivation. */ + /** + * Return true iff this is a fixed-output derivation. + */ DerivationType type() const; - /* Return the output names of a derivation. */ + /** + * Return the output names of a derivation. + */ StringSet outputNames() const; - /* Calculates the maps that contains all the DerivationOutputs, but - augmented with knowledge of the Store paths they would be written - into. */ + /** + * Calculates the maps that contains all the DerivationOutputs, but + * augmented with knowledge of the Store paths they would be written + * into. + */ DerivationOutputsAndOptPaths outputsAndOptPaths(const Store & store) const; static std::string_view nameFromPath(const StorePath & storePath); + + GENERATE_CMP(BasicDerivation, + me->outputs, + me->inputSrcs, + me->platform, + me->builder, + me->args, + me->env, + me->name); }; struct Derivation : BasicDerivation { - DerivationInputs inputDrvs; /* inputs that are sub-derivations */ + /** + * inputs that are sub-derivations + */ + DerivationInputs inputDrvs; - /* Print a derivation. */ + /** + * Print a derivation. + */ std::string unparse(const Store & store, bool maskOutputs, std::map * actualInputs = nullptr) const; - /* Return the underlying basic derivation but with these changes: - - 1. Input drvs are emptied, but the outputs of them that were used are - added directly to input sources. - - 2. Input placeholders are replaced with realized input store paths. */ + /** + * Return the underlying basic derivation but with these changes: + * + * 1. Input drvs are emptied, but the outputs of them that were used + * are added directly to input sources. + * + * 2. Input placeholders are replaced with realized input store + * paths. + */ std::optional tryResolve(Store & store) const; - /* Like the above, but instead of querying the Nix database for - realisations, uses a given mapping from input derivation paths - + output names to actual output store paths. */ + /** + * Like the above, but instead of querying the Nix database for + * realisations, uses a given mapping from input derivation paths + + * output names to actual output store paths. + */ std::optional tryResolve( Store & store, const std::map, StorePath> & inputDrvOutputs) const; + /* Check that the derivation is valid and does not present any + illegal states. + + This is mainly a matter of checking the outputs, where our C++ + representation supports all sorts of combinations we do not yet + allow. */ + void checkInvariants(Store & store, const StorePath & drvPath) const; + Derivation() = default; Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { } Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { } nlohmann::json toJSON(const Store & store) const; + static Derivation fromJSON( + const Store & store, + const nlohmann::json & json); + + GENERATE_CMP(Derivation, + static_cast(*me), + me->inputDrvs); }; class Store; -/* Write a derivation to the Nix store, and return its path. */ +/** + * Write a derivation to the Nix store, and return its path. + */ StorePath writeDerivation(Store & store, const Derivation & drv, RepairFlag repair = NoRepair, bool readOnly = false); -/* Read a derivation from a file. */ +/** + * Read a derivation from a file. + */ Derivation parseDerivation(const Store & store, std::string && s, std::string_view name); -// FIXME: remove +/** + * \todo Remove. + * + * Use Path::isDerivation instead. + */ bool isDerivation(std::string_view fileName); -/* Calculate the name that will be used for the store path for this - output. - - This is usually -, but is just when - the output name is "out". */ +/** + * Calculate the name that will be used for the store path for this + * output. + * + * This is usually -, but is just when + * the output name is "out". + */ std::string outputPathName(std::string_view drvName, std::string_view outputName); -// The hashes modulo of a derivation. -// -// Each output is given a hash, although in practice only the content-addressed -// derivations (fixed-output or not) will have a different hash for each -// output. +/** + * The hashes modulo of a derivation. + * + * Each output is given a hash, although in practice only the content-addressed + * derivations (fixed-output or not) will have a different hash for each + * output. + */ struct DrvHash { + /** + * Map from output names to hashes + */ std::map hashes; enum struct Kind : bool { - // Statically determined derivations. - // This hash will be directly used to compute the output paths + /** + * Statically determined derivations. + * This hash will be directly used to compute the output paths + */ Regular, - // Floating-output derivations (and their reverse dependencies). + + /** + * Floating-output derivations (and their reverse dependencies). + */ Deferred, }; + /** + * The kind of derivation this is, simplified for just "derivation hash + * modulo" purposes. + */ Kind kind; }; void operator |= (DrvHash::Kind & self, const DrvHash::Kind & other) noexcept; -/* Returns hashes with the details of fixed-output subderivations - expunged. - - A fixed-output derivation is a derivation whose outputs have a - specified content hash and hash algorithm. (Currently they must have - exactly one output (`out'), which is specified using the `outputHash' - and `outputHashAlgo' attributes, but the algorithm doesn't assume - this.) We don't want changes to such derivations to propagate upwards - through the dependency graph, changing output paths everywhere. - - For instance, if we change the url in a call to the `fetchurl' - function, we do not want to rebuild everything depending on it---after - all, (the hash of) the file being downloaded is unchanged. So the - *output paths* should not change. On the other hand, the *derivation - paths* should change to reflect the new dependency graph. - - For fixed-output derivations, this returns a map from the name of - each output to its hash, unique up to the output's contents. - - For regular derivations, it returns a single hash of the derivation - ATerm, after subderivations have been likewise expunged from that - derivation. +/** + * Returns hashes with the details of fixed-output subderivations + * expunged. + * + * A fixed-output derivation is a derivation whose outputs have a + * specified content hash and hash algorithm. (Currently they must have + * exactly one output (`out`), which is specified using the `outputHash` + * and `outputHashAlgo` attributes, but the algorithm doesn't assume + * this.) We don't want changes to such derivations to propagate upwards + * through the dependency graph, changing output paths everywhere. + * + * For instance, if we change the url in a call to the `fetchurl` + * function, we do not want to rebuild everything depending on it---after + * all, (the hash of) the file being downloaded is unchanged. So the + * *output paths* should not change. On the other hand, the *derivation + * paths* should change to reflect the new dependency graph. + * + * For fixed-output derivations, this returns a map from the name of + * each output to its hash, unique up to the output's contents. + * + * For regular derivations, it returns a single hash of the derivation + * ATerm, after subderivations have been likewise expunged from that + * derivation. */ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); -/* - Return a map associating each output to a hash that uniquely identifies its - derivation (modulo the self-references). - - FIXME: what is the Hash in this map? +/** + * Return a map associating each output to a hash that uniquely identifies its + * derivation (modulo the self-references). + * + * \todo What is the Hash in this map? */ std::map staticOutputHashes(Store & store, const Derivation & drv); -/* Memoisation of hashDerivationModulo(). */ +/** + * Memoisation of hashDerivationModulo(). + */ typedef std::map DrvHashes; // FIXME: global, though at least thread-safe. @@ -308,21 +477,25 @@ struct Sink; Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name); void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv); -/* This creates an opaque and almost certainly unique string - deterministically from the output name. - - It is used as a placeholder to allow derivations to refer to their - own outputs without needing to use the hash of a derivation in - itself, making the hash near-impossible to calculate. */ +/** + * This creates an opaque and almost certainly unique string + * deterministically from the output name. + * + * It is used as a placeholder to allow derivations to refer to their + * own outputs without needing to use the hash of a derivation in + * itself, making the hash near-impossible to calculate. + */ std::string hashPlaceholder(const std::string_view outputName); -/* This creates an opaque and almost certainly unique string - deterministically from a derivation path and output name. - - It is used as a placeholder to allow derivations to refer to - content-addressed paths whose content --- and thus the path - themselves --- isn't yet known. This occurs when a derivation has a - dependency which is a CA derivation. */ +/** + * This creates an opaque and almost certainly unique string + * deterministically from a derivation path and output name. + * + * It is used as a placeholder to allow derivations to refer to + * content-addressed paths whose content --- and thus the path + * themselves --- isn't yet known. This occurs when a derivation has a + * dependency which is a CA derivation. + */ std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName); extern const Hash impureOutputHash; diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index e0d86a42f..e5f0f1b33 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -105,7 +105,7 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const auto drvHashes = staticOutputHashes(store, store.readDerivation(p.drvPath)); for (auto& [outputName, outputPath] : p.outputs) { - if (settings.isExperimentalFeatureEnabled( + if (experimentalFeatureSettings.isEnabled( Xp::CaDerivations)) { auto drvOutput = get(drvHashes, outputName); if (!drvOutput) diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 9e0cce377..2155776b1 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "util.hh" #include "path.hh" @@ -105,7 +106,7 @@ using _BuiltPathRaw = std::variant< >; /** - * A built path. Similar to a `DerivedPath`, but enriched with the corresponding + * A built path. Similar to a DerivedPath, but enriched with the corresponding * output path(s). */ struct BuiltPath : _BuiltPathRaw { diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index b4fbe0b70..ae2777d0c 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -7,6 +7,13 @@ struct DummyStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; const std::string name() override { return "Dummy Store"; } + + std::string doc() override + { + return + #include "dummy-store.md" + ; + } }; struct DummyStore : public virtual DummyStoreConfig, public virtual Store @@ -32,6 +39,14 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store callback(nullptr); } + /** + * The dummy store is incapable of *not* trusting! :) + */ + virtual std::optional isTrustedClient() override + { + return Trusted; + } + static std::set uriSchemes() { return {"dummy"}; } diff --git a/src/libstore/dummy-store.md b/src/libstore/dummy-store.md new file mode 100644 index 000000000..eb7b4ba0d --- /dev/null +++ b/src/libstore/dummy-store.md @@ -0,0 +1,13 @@ +R"( + +**Store URL format**: `dummy://` + +This store type represents a store that contains no store paths and +cannot be written to. It's useful when you want to use the Nix +evaluator when no actual Nix store exists, e.g. + +```console +# nix eval --store dummy:// --expr '1 + 2' +``` + +)" diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index b5fe7c03b..2346accbe 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -318,7 +318,7 @@ struct curlFileTransfer : public FileTransfer if (request.verifyTLS) { if (settings.caFile != "") - curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.c_str()); + curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.get().c_str()); } else { curl_easy_setopt(req, CURLOPT_SSL_VERIFYPEER, 0); curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0); @@ -407,6 +407,10 @@ struct curlFileTransfer : public FileTransfer err = Misc; } else { // Don't bother retrying on certain cURL errors either + + // Allow selecting a subset of enum values + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wswitch-enum" switch (code) { case CURLE_FAILED_INIT: case CURLE_URL_MALFORMAT: @@ -427,6 +431,7 @@ struct curlFileTransfer : public FileTransfer default: // Shut up warnings break; } + #pragma GCC diagnostic pop } attempt++; diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index 07d58f53a..378c6ff78 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "hash.hh" @@ -87,39 +88,56 @@ struct FileTransfer { virtual ~FileTransfer() { } - /* Enqueue a data transfer request, returning a future to the result of - the download. The future may throw a FileTransferError - exception. */ + /** + * Enqueue a data transfer request, returning a future to the result of + * the download. The future may throw a FileTransferError + * exception. + */ virtual void enqueueFileTransfer(const FileTransferRequest & request, Callback callback) = 0; std::future enqueueFileTransfer(const FileTransferRequest & request); - /* Synchronously download a file. */ + /** + * Synchronously download a file. + */ FileTransferResult download(const FileTransferRequest & request); - /* Synchronously upload a file. */ + /** + * Synchronously upload a file. + */ FileTransferResult upload(const FileTransferRequest & request); - /* Download a file, writing its data to a sink. The sink will be - invoked on the thread of the caller. */ + /** + * Download a file, writing its data to a sink. The sink will be + * invoked on the thread of the caller. + */ void download(FileTransferRequest && request, Sink & sink); enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; }; -/* Return a shared FileTransfer object. Using this object is preferred - because it enables connection reuse and HTTP/2 multiplexing. */ +/** + * @return a shared FileTransfer object. + * + * Using this object is preferred because it enables connection reuse + * and HTTP/2 multiplexing. + */ ref getFileTransfer(); -/* Return a new FileTransfer object. */ +/** + * @return a new FileTransfer object + * + * Prefer getFileTransfer() to this; see its docs for why. + */ ref makeFileTransfer(); class FileTransferError : public Error { public: FileTransfer::Error error; - std::optional response; // intentionally optional + /// intentionally optional + std::optional response; template FileTransferError(FileTransfer::Error error, std::optional response, const Args & ... args); diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh index c825e84f2..1df19e647 100644 --- a/src/libstore/fs-accessor.hh +++ b/src/libstore/fs-accessor.hh @@ -1,11 +1,14 @@ #pragma once +///@file #include "types.hh" namespace nix { -/* An abstract class for accessing a filesystem-like structure, such - as a (possibly remote) Nix store or the contents of a NAR file. */ +/** + * An abstract class for accessing a filesystem-like structure, such + * as a (possibly remote) Nix store or the contents of a NAR file. + */ class FSAccessor { public: @@ -14,8 +17,17 @@ public: struct Stat { Type type = tMissing; - uint64_t fileSize = 0; // regular files only + /** + * regular files only + */ + uint64_t fileSize = 0; + /** + * regular files only + */ bool isExecutable = false; // regular files only + /** + * regular files only + */ uint64_t narOffset = 0; // regular files only }; diff --git a/src/libstore/gc-store.hh b/src/libstore/gc-store.hh index b3cbbad74..2c26c65c4 100644 --- a/src/libstore/gc-store.hh +++ b/src/libstore/gc-store.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" @@ -11,19 +12,20 @@ typedef std::unordered_map> Roots; struct GCOptions { - /* Garbage collector operation: - - - `gcReturnLive': return the set of paths reachable from - (i.e. in the closure of) the roots. - - - `gcReturnDead': return the set of paths not reachable from - the roots. - - - `gcDeleteDead': actually delete the latter set. - - - `gcDeleteSpecific': delete the paths listed in - `pathsToDelete', insofar as they are not reachable. - */ + /** + * Garbage collector operation: + * + * - `gcReturnLive`: return the set of paths reachable from + * (i.e. in the closure of) the roots. + * + * - `gcReturnDead`: return the set of paths not reachable from + * the roots. + * + * - `gcDeleteDead`: actually delete the latter set. + * + * - `gcDeleteSpecific`: delete the paths listed in + * `pathsToDelete`, insofar as they are not reachable. + */ typedef enum { gcReturnLive, gcReturnDead, @@ -33,28 +35,38 @@ struct GCOptions GCAction action{gcDeleteDead}; - /* If `ignoreLiveness' is set, then reachability from the roots is - ignored (dangerous!). However, the paths must still be - unreferenced *within* the store (i.e., there can be no other - store paths that depend on them). */ + /** + * If `ignoreLiveness` is set, then reachability from the roots is + * ignored (dangerous!). However, the paths must still be + * unreferenced *within* the store (i.e., there can be no other + * store paths that depend on them). + */ bool ignoreLiveness{false}; - /* For `gcDeleteSpecific', the paths to delete. */ + /** + * For `gcDeleteSpecific`, the paths to delete. + */ StorePathSet pathsToDelete; - /* Stop after at least `maxFreed' bytes have been freed. */ + /** + * Stop after at least `maxFreed` bytes have been freed. + */ uint64_t maxFreed{std::numeric_limits::max()}; }; struct GCResults { - /* Depending on the action, the GC roots, or the paths that would - be or have been deleted. */ + /** + * Depending on the action, the GC roots, or the paths that would + * be or have been deleted. + */ PathSet paths; - /* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the - number of bytes that would be or was freed. */ + /** + * For `gcReturnDead`, `gcDeleteDead` and `gcDeleteSpecific`, the + * number of bytes that would be or was freed. + */ uint64_t bytesFreed = 0; }; @@ -63,21 +75,27 @@ struct GcStore : public virtual Store { inline static std::string operationName = "Garbage collection"; - /* Add an indirect root, which is merely a symlink to `path' from - /nix/var/nix/gcroots/auto/. `path' is supposed - to be a symlink to a store path. The garbage collector will - automatically remove the indirect root when it finds that - `path' has disappeared. */ + /** + * Add an indirect root, which is merely a symlink to `path` from + * `/nix/var/nix/gcroots/auto/`. `path` is supposed + * to be a symlink to a store path. The garbage collector will + * automatically remove the indirect root when it finds that + * `path` has disappeared. + */ virtual void addIndirectRoot(const Path & path) = 0; - /* Find the roots of the garbage collector. Each root is a pair - (link, storepath) where `link' is the path of the symlink - outside of the Nix store that point to `storePath'. If - 'censor' is true, privacy-sensitive information about roots - found in /proc is censored. */ + /** + * Find the roots of the garbage collector. Each root is a pair + * `(link, storepath)` where `link` is the path of the symlink + * outside of the Nix store that point to `storePath`. If + * `censor` is true, privacy-sensitive information about roots + * found in `/proc` is censored. + */ virtual Roots findRoots(bool censor) = 0; - /* Perform a garbage collection. */ + /** + * Perform a garbage collection. + */ virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0; }; diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index fae79c1a0..823b4af74 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -44,14 +44,9 @@ Settings::Settings() lockCPU = getEnv("NIX_AFFINITY_HACK") == "1"; allowSymlinkedStore = getEnv("NIX_IGNORE_SYMLINK_STORE") == "1"; - caFile = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or("")); - if (caFile == "") { - for (auto & fn : {"/etc/ssl/certs/ca-certificates.crt", "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"}) - if (pathExists(fn)) { - caFile = fn; - break; - } - } + auto sslOverride = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or("")); + if (sslOverride != "") + caFile = sslOverride; /* Backwards compatibility. */ auto s = getEnv("NIX_REMOTE_SYSTEMS"); @@ -166,18 +161,6 @@ StringSet Settings::getDefaultExtraPlatforms() return extraPlatforms; } -bool Settings::isExperimentalFeatureEnabled(const ExperimentalFeature & feature) -{ - auto & f = experimentalFeatures.get(); - return std::find(f.begin(), f.end(), feature) != f.end(); -} - -void Settings::requireExperimentalFeature(const ExperimentalFeature & feature) -{ - if (!isExperimentalFeatureEnabled(feature)) - throw MissingExperimentalFeature(feature); -} - bool Settings::isWSL1() { struct utsname utsbuf; @@ -187,6 +170,13 @@ bool Settings::isWSL1() return hasSuffix(utsbuf.release, "-Microsoft"); } +Path Settings::getDefaultSSLCertFile() +{ + for (auto & fn : {"/etc/ssl/certs/ca-certificates.crt", "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"}) + if (pathExists(fn)) return fn; + return ""; +} + const std::string nixVersion = PACKAGE_VERSION; NLOHMANN_JSON_SERIALIZE_ENUM(SandboxMode, { diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 93086eaf8..63c7389da 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -1,9 +1,9 @@ #pragma once +///@file #include "types.hh" #include "config.hh" #include "util.hh" -#include "experimental-features.hh" #include #include @@ -64,40 +64,63 @@ class Settings : public Config { bool isWSL1(); + Path getDefaultSSLCertFile(); + public: Settings(); Path nixPrefix; - /* The directory where we store sources and derived files. */ + /** + * The directory where we store sources and derived files. + */ Path nixStore; Path nixDataDir; /* !!! fix */ - /* The directory where we log various operations. */ + /** + * The directory where we log various operations. + */ Path nixLogDir; - /* The directory where state is stored. */ + /** + * The directory where state is stored. + */ Path nixStateDir; - /* The directory where system configuration files are stored. */ + /** + * The directory where system configuration files are stored. + */ Path nixConfDir; - /* A list of user configuration files to load. */ + /** + * A list of user configuration files to load. + */ std::vector nixUserConfFiles; - /* The directory where the main programs are stored. */ + /** + * The directory where the main programs are stored. + */ Path nixBinDir; - /* The directory where the man pages are stored. */ + /** + * The directory where the man pages are stored. + */ Path nixManDir; - /* File name of the socket the daemon listens to. */ + /** + * File name of the socket the daemon listens to. + */ Path nixDaemonSocketFile; Setting storeUri{this, getEnv("NIX_REMOTE").value_or("auto"), "store", - "The default Nix store to use."}; + R"( + The [URL of the Nix store](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + to use for most operations. + See [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md) + for supported store types and settings. + )"}; Setting keepFailed{this, false, "keep-failed", "Whether to keep temporary directories of failed builds."}; @@ -114,7 +137,9 @@ public: )", {"build-fallback"}}; - /* Whether to show build log output in real time. */ + /** + * Whether to show build log output in real time. + */ bool verboseBuild = true; Setting logLines{this, 10, "log-lines", @@ -150,8 +175,10 @@ public: )", {"build-cores"}, false}; - /* Read-only mode. Don't copy stuff to the store, don't change - the database. */ + /** + * Read-only mode. Don't copy stuff to the store, don't change + * the database. + */ bool readOnlyMode = false; Setting thisSystem{ @@ -451,7 +478,9 @@ public: )", {"env-keep-derivations"}}; - /* Whether to lock the Nix client and worker to the same CPU. */ + /** + * Whether to lock the Nix client and worker to the same CPU. + */ bool lockCPU; Setting sandboxMode{ @@ -678,8 +707,9 @@ public: Strings{"https://cache.nixos.org/"}, "substituters", R"( - A list of URLs of substituters, separated by whitespace. Substituters - are tried based on their Priority value, which each substituter can set + A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + to be used as substituters, separated by whitespace. + Substituters are tried based on their Priority value, which each substituter can set independently. Lower value means higher priority. The default is `https://cache.nixos.org`, with a Priority of 40. @@ -697,7 +727,8 @@ public: Setting trustedSubstituters{ this, {}, "trusted-substituters", R"( - A list of URLs of substituters, separated by whitespace. These are + A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format), + separated by whitespace. These are not used by default, but can be enabled by users of the Nix daemon by specifying `--option substituters urls` on the command line. Unprivileged users are only allowed to pass a subset of the @@ -826,8 +857,22 @@ public: > `.netrc`. )"}; - /* Path to the SSL CA file used */ - Path caFile; + Setting caFile{ + this, getDefaultSSLCertFile(), "ssl-cert-file", + R"( + The path of a file containing CA certificates used to + authenticate `https://` downloads. Nix by default will use + the first of the following files that exists: + + 1. `/etc/ssl/certs/ca-certificates.crt` + 2. `/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt` + + The path can be overridden by the following environment + variables, in order of precedence: + + 1. `NIX_SSL_CERT_FILE` + 2. `SSL_CERT_FILE` + )"}; #if __linux__ Setting filterSyscalls{ @@ -932,13 +977,6 @@ public: are loaded as plugins (non-recursively). )"}; - Setting> experimentalFeatures{this, {}, "experimental-features", - "Experimental Nix features to enable."}; - - bool isExperimentalFeatureEnabled(const ExperimentalFeature &); - - void requireExperimentalFeature(const ExperimentalFeature &); - Setting narBufferSize{this, 32 * 1024 * 1024, "nar-buffer-size", "Maximum size of NARs before spilling them to disk."}; @@ -981,8 +1019,10 @@ public: // FIXME: don't use a global variable. extern Settings settings; -/* This should be called after settings are initialized, but before - anything else */ +/** + * This should be called after settings are initialized, but before + * anything else + */ void initPlugins(); void loadConfFile(); @@ -992,12 +1032,16 @@ std::vector getUserConfigFiles(); extern const std::string nixVersion; -/* NB: This is not sufficient. You need to call initNix() */ +/** + * NB: This is not sufficient. You need to call initNix() + */ void initLibStore(); -/* It's important to initialize before doing _anything_, which is why we - call upon the programmer to handle this correctly. However, we only add - this in a key locations, so as not to litter the code. */ +/** + * It's important to initialize before doing _anything_, which is why we + * call upon the programmer to handle this correctly. However, we only add + * this in a key locations, so as not to litter the code. + */ void assertLibStoreInitialized(); } diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 1479822a9..85c5eed4c 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -12,7 +12,14 @@ struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig { using BinaryCacheStoreConfig::BinaryCacheStoreConfig; - const std::string name() override { return "Http Binary Cache Store"; } + const std::string name() override { return "HTTP Binary Cache Store"; } + + std::string doc() override + { + return + #include "http-binary-cache-store.md" + ; + } }; class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public virtual BinaryCacheStore @@ -187,6 +194,18 @@ protected: }}); } + /** + * This isn't actually necessary read only. We support "upsert" now, so we + * have a notion of authentication via HTTP POST/PUT. + * + * For now, we conservatively say we don't know. + * + * \todo try to expose our HTTP authentication status. + */ + std::optional isTrustedClient() override + { + return std::nullopt; + } }; static RegisterStoreImplementation regHttpBinaryCacheStore; diff --git a/src/libstore/http-binary-cache-store.md b/src/libstore/http-binary-cache-store.md new file mode 100644 index 000000000..20c26d0c2 --- /dev/null +++ b/src/libstore/http-binary-cache-store.md @@ -0,0 +1,8 @@ +R"( + +**Store URL format**: `http://...`, `https://...` + +This store allows a binary cache to be accessed via the HTTP +protocol. + +)" diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 2c9dd2680..eb471d8fc 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -1,3 +1,4 @@ +#include "ssh-store-config.hh" #include "archive.hh" #include "pool.hh" #include "remote-store.hh" @@ -12,17 +13,24 @@ namespace nix { -struct LegacySSHStoreConfig : virtual StoreConfig +struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig { - using StoreConfig::StoreConfig; - const Setting maxConnections{(StoreConfig*) this, 1, "max-connections", "maximum number of concurrent SSH connections"}; - const Setting sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"}; - const Setting sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key", "The public half of the host's SSH key"}; - const Setting compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"}; - const Setting remoteProgram{(StoreConfig*) this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"}; - const Setting remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"}; + using CommonSSHStoreConfig::CommonSSHStoreConfig; - const std::string name() override { return "Legacy SSH Store"; } + const Setting remoteProgram{(StoreConfig*) this, "nix-store", "remote-program", + "Path to the `nix-store` executable on the remote machine."}; + + const Setting maxConnections{(StoreConfig*) this, 1, "max-connections", + "Maximum number of concurrent SSH connections."}; + + const std::string name() override { return "SSH Store"; } + + std::string doc() override + { + return + #include "legacy-ssh-store.md" + ; + } }; struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store @@ -51,6 +59,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor LegacySSHStore(const std::string & scheme, const std::string & host, const Params & params) : StoreConfig(params) + , CommonSSHStoreConfig(params) , LegacySSHStoreConfig(params) , Store(params) , host(host) @@ -380,6 +389,15 @@ public: return conn->remoteVersion; } + /** + * The legacy ssh protocol doesn't support checking for trusted-user. + * Try using ssh-ng:// instead if you want to know. + */ + std::optional isTrustedClient() override + { + return std::nullopt; + } + void queryRealisationUncached(const DrvOutput &, Callback> callback) noexcept override // TODO: Implement diff --git a/src/libstore/legacy-ssh-store.md b/src/libstore/legacy-ssh-store.md new file mode 100644 index 000000000..043acebd6 --- /dev/null +++ b/src/libstore/legacy-ssh-store.md @@ -0,0 +1,8 @@ +R"( + +**Store URL format**: `ssh://[username@]hostname` + +This store type allows limited access to a remote store on another +machine via SSH. + +)" diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index f20b1fa02..5481dd762 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -11,6 +11,13 @@ struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig using BinaryCacheStoreConfig::BinaryCacheStoreConfig; const std::string name() override { return "Local Binary Cache Store"; } + + std::string doc() override + { + return + #include "local-binary-cache-store.md" + ; + } }; class LocalBinaryCacheStore : public virtual LocalBinaryCacheStoreConfig, public virtual BinaryCacheStore @@ -88,6 +95,10 @@ protected: return paths; } + std::optional isTrustedClient() override + { + return Trusted; + } }; void LocalBinaryCacheStore::init() diff --git a/src/libstore/local-binary-cache-store.md b/src/libstore/local-binary-cache-store.md new file mode 100644 index 000000000..93fddc840 --- /dev/null +++ b/src/libstore/local-binary-cache-store.md @@ -0,0 +1,16 @@ +R"( + +**Store URL format**: `file://`*path* + +This store allows reading and writing a binary cache stored in *path* +in the local filesystem. If *path* does not exist, it will be created. + +For example, the following builds or downloads `nixpkgs#hello` into +the local store and then copies it to the binary cache in +`/tmp/binary-cache`: + +``` +# nix copy --to file:///tmp/binary-cache nixpkgs#hello +``` + +)" diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 947707341..a03bb88f5 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" #include "gc-store.hh" @@ -9,20 +10,28 @@ namespace nix { struct LocalFSStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; + // FIXME: the (StoreConfig*) cast works around a bug in gcc that causes // it to omit the call to the Setting constructor. Clang works fine // either way. + const PathSetting rootDir{(StoreConfig*) this, true, "", - "root", "directory prefixed to all other paths"}; + "root", + "Directory prefixed to all other paths."}; + const PathSetting stateDir{(StoreConfig*) this, false, rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir, - "state", "directory where Nix will store state"}; + "state", + "Directory where Nix will store state."}; + const PathSetting logDir{(StoreConfig*) this, false, rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir, - "log", "directory where Nix will store state"}; + "log", + "directory where Nix will store log files."}; + const PathSetting realStoreDir{(StoreConfig*) this, false, rootDir != "" ? rootDir + "/nix/store" : storeDir, "real", - "physical path to the Nix store"}; + "Physical path of the Nix store."}; }; class LocalFSStore : public virtual LocalFSStoreConfig, @@ -39,7 +48,9 @@ public: void narFromPath(const StorePath & path, Sink & sink) override; ref getFSAccessor() override; - /* Register a permanent GC root. */ + /** + * Register a permanent GC root. + */ Path addPermRoot(const StorePath & storePath, const Path & gcRoot); virtual Path getRealStoreDir() { return realStoreDir; } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c9a466ee8..c3be1b461 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -44,6 +44,13 @@ namespace nix { +std::string LocalStoreConfig::doc() +{ + return + #include "local-store.md" + ; +} + struct LocalStore::State::Stmts { /* Some precompiled SQLite statements. */ SQLiteStmt RegisterValidPath; @@ -336,7 +343,7 @@ LocalStore::LocalStore(const Params & params) else openDB(*state, false); - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { migrateCASchema(state->db, dbDir + "/ca-schema", globalLock); } @@ -366,7 +373,7 @@ LocalStore::LocalStore(const Params & params) state->stmts->QueryPathFromHashPart.create(state->db, "select path from ValidPaths where path >= ? limit 1;"); state->stmts->QueryValidPaths.create(state->db, "select path from ValidPaths"); - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { state->stmts->RegisterRealisedOutput.create(state->db, R"( insert into Realisations (drvPath, outputName, outputPath, signatures) @@ -413,6 +420,13 @@ LocalStore::LocalStore(const Params & params) } +LocalStore::LocalStore(std::string scheme, std::string path, const Params & params) + : LocalStore(params) +{ + throw UnimplementedError("LocalStore"); +} + + AutoCloseFD LocalStore::openGCLock() { Path fnGCLock = stateDir + "/gc.lock"; @@ -696,65 +710,9 @@ void canonicalisePathMetaData(const Path & path, canonicalisePathMetaData(path, uidRange, inodesSeen); } - -void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv) -{ - assert(drvPath.isDerivation()); - std::string drvName(drvPath.name()); - drvName = drvName.substr(0, drvName.size() - drvExtension.size()); - - auto envHasRightPath = [&](const StorePath & actual, const std::string & varName) - { - auto j = drv.env.find(varName); - if (j == drv.env.end() || parseStorePath(j->second) != actual) - throw Error("derivation '%s' has incorrect environment variable '%s', should be '%s'", - printStorePath(drvPath), varName, printStorePath(actual)); - }; - - - // Don't need the answer, but do this anyways to assert is proper - // combination. The code below is more general and naturally allows - // combinations that are currently prohibited. - drv.type(); - - std::optional hashesModulo; - for (auto & i : drv.outputs) { - std::visit(overloaded { - [&](const DerivationOutput::InputAddressed & doia) { - if (!hashesModulo) { - // somewhat expensive so we do lazily - hashesModulo = hashDerivationModulo(*this, drv, true); - } - auto currentOutputHash = get(hashesModulo->hashes, i.first); - if (!currentOutputHash) - throw Error("derivation '%s' has unexpected output '%s' (local-store / hashesModulo) named '%s'", - printStorePath(drvPath), printStorePath(doia.path), i.first); - StorePath recomputed = makeOutputPath(i.first, *currentOutputHash, drvName); - if (doia.path != recomputed) - throw Error("derivation '%s' has incorrect output '%s', should be '%s'", - printStorePath(drvPath), printStorePath(doia.path), printStorePath(recomputed)); - envHasRightPath(doia.path, i.first); - }, - [&](const DerivationOutput::CAFixed & dof) { - StorePath path = makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName); - envHasRightPath(path, i.first); - }, - [&](const DerivationOutput::CAFloating &) { - /* Nothing to check */ - }, - [&](const DerivationOutput::Deferred &) { - /* Nothing to check */ - }, - [&](const DerivationOutput::Impure &) { - /* Nothing to check */ - }, - }, i.second.raw()); - } -} - void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) { - settings.requireExperimentalFeature(Xp::CaDerivations); + experimentalFeatureSettings.require(Xp::CaDerivations); if (checkSigs == NoCheckSigs || !realisationIsUntrusted(info)) registerDrvOutput(info); else @@ -763,7 +721,7 @@ void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag check void LocalStore::registerDrvOutput(const Realisation & info) { - settings.requireExperimentalFeature(Xp::CaDerivations); + experimentalFeatureSettings.require(Xp::CaDerivations); retrySQLite([&]() { auto state(_state.lock()); if (auto oldR = queryRealisation_(*state, info.id)) { @@ -862,7 +820,7 @@ uint64_t LocalStore::addValidPath(State & state, derivations). Note that if this throws an error, then the DB transaction is rolled back, so the path validity registration above is undone. */ - if (checkOutputs) checkDerivationOutputs(info.path, drv); + if (checkOutputs) drv.checkInvariants(*this, info.path); for (auto & i : drv.outputsAndOptPaths(*this)) { /* Floating CA derivations have indeterminate output paths until @@ -1052,7 +1010,7 @@ LocalStore::queryPartialDerivationOutputMap(const StorePath & path_) return outputs; }); - if (!settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) + if (!experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) return outputs; auto drv = readInvalidDerivation(path); @@ -1120,54 +1078,6 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) } -// FIXME: move this, it's not specific to LocalStore. -void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) -{ - if (!settings.useSubstitutes) return; - for (auto & sub : getDefaultSubstituters()) { - for (auto & path : paths) { - if (infos.count(path.first)) - // Choose first succeeding substituter. - continue; - - auto subPath(path.first); - - // Recompute store path so that we can use a different store root. - if (path.second) { - subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second); - if (sub->storeDir == storeDir) - assert(subPath == path.first); - if (subPath != path.first) - debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path.first), sub->printStorePath(subPath), sub->getUri()); - } else if (sub->storeDir != storeDir) continue; - - debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath)); - try { - auto info = sub->queryPathInfo(subPath); - - if (sub->storeDir != storeDir && !(info->isContentAddressed(*sub) && info->references.empty())) - continue; - - auto narInfo = std::dynamic_pointer_cast( - std::shared_ptr(info)); - infos.insert_or_assign(path.first, SubstitutablePathInfo{ - info->deriver, - info->references, - narInfo ? narInfo->fileSize : 0, - info->narSize}); - } catch (InvalidPath &) { - } catch (SubstituterDisabled &) { - } catch (Error & e) { - if (settings.tryFallback) - logError(e.info()); - else - throw; - } - } - } -} - - void LocalStore::registerValidPath(const ValidPathInfo & info) { registerValidPaths({{info.path, info}}); @@ -1209,8 +1119,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) for (auto & [_, i] : infos) if (i.path.isDerivation()) { // FIXME: inefficient; we already loaded the derivation in addValidPath(). - checkDerivationOutputs(i.path, - readInvalidDerivation(i.path)); + readInvalidDerivation(i.path).checkInvariants(*this, i.path); } /* Do a topological sort of the paths. This will throw an @@ -1719,6 +1628,11 @@ unsigned int LocalStore::getProtocol() return PROTOCOL_VERSION; } +std::optional LocalStore::isTrustedClient() +{ + return Trusted; +} + #if defined(FS_IOC_SETFLAGS) && defined(FS_IOC_GETFLAGS) && defined(FS_IMMUTABLE_FL) @@ -1950,5 +1864,6 @@ std::optional LocalStore::getVersion() return nixVersion; } +static RegisterStoreImplementation regLocalStore; } // namespace nix diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index a84eb7c26..55add18dd 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "sqlite.hh" @@ -18,10 +19,14 @@ namespace nix { -/* Nix store and database schema version. Version 1 (or 0) was Nix <= - 0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10. - Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is - Nix 1.0. Version 7 is Nix 1.3. Version 10 is 2.0. */ +/** + * Nix store and database schema version. + * + * Version 1 (or 0) was Nix <= + * 0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10. + * Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is + * Nix 1.0. Version 7 is Nix 1.3. Version 10 is 2.0. + */ const int nixSchemaVersion = 10; @@ -38,40 +43,52 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig Setting requireSigs{(StoreConfig*) this, settings.requireSigs, - "require-sigs", "whether store paths should have a trusted signature on import"}; + "require-sigs", + "Whether store paths copied into this store should have a trusted signature."}; const std::string name() override { return "Local Store"; } -}; + std::string doc() override; +}; class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore, public virtual GcStore { private: - /* Lock file used for upgrading. */ + /** + * Lock file used for upgrading. + */ AutoCloseFD globalLock; struct State { - /* The SQLite database object. */ + /** + * The SQLite database object. + */ SQLite db; struct Stmts; std::unique_ptr stmts; - /* The last time we checked whether to do an auto-GC, or an - auto-GC finished. */ + /** + * The last time we checked whether to do an auto-GC, or an + * auto-GC finished. + */ std::chrono::time_point lastGCCheck; - /* Whether auto-GC is running. If so, get gcFuture to wait for - the GC to finish. */ + /** + * Whether auto-GC is running. If so, get gcFuture to wait for + * the GC to finish. + */ bool gcRunning = false; std::shared_future gcFuture; - /* How much disk space was available after the previous - auto-GC. If the current available disk space is below - minFree but not much below availAfterGC, then there is no - point in starting a new GC. */ + /** + * How much disk space was available after the previous + * auto-GC. If the current available disk space is below + * minFree but not much below availAfterGC, then there is no + * point in starting a new GC. + */ uint64_t availAfterGC = std::numeric_limits::max(); std::unique_ptr publicKeys; @@ -94,16 +111,26 @@ private: public: - // Hack for build-remote.cc. + /** + * Hack for build-remote.cc. + */ PathSet locksHeld; - /* Initialise the local store, upgrading the schema if - necessary. */ + /** + * Initialise the local store, upgrading the schema if + * necessary. + */ LocalStore(const Params & params); + LocalStore(std::string scheme, std::string path, const Params & params); ~LocalStore(); - /* Implementations of abstract store API methods. */ + static std::set uriSchemes() + { return {}; } + + /** + * Implementations of abstract store API methods. + */ std::string getUri() override; @@ -127,9 +154,6 @@ public: StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; - void querySubstitutablePathInfos(const StorePathCAMap & paths, - SubstitutablePathInfos & infos) override; - bool pathInfoIsUntrusted(const ValidPathInfo &) override; bool realisationIsUntrusted(const Realisation & ) override; @@ -151,13 +175,19 @@ private: void createTempRootsFile(); - /* The file to which we write our temporary roots. */ + /** + * The file to which we write our temporary roots. + */ Sync _fdTempRoots; - /* The global GC lock. */ + /** + * The global GC lock. + */ Sync _fdGCLock; - /* Connection to the garbage collector. */ + /** + * Connection to the garbage collector. + */ Sync _fdRootsSocket; public: @@ -176,42 +206,54 @@ public: void collectGarbage(const GCOptions & options, GCResults & results) override; - /* Optimise the disk space usage of the Nix store by hard-linking - files with the same contents. */ + /** + * Optimise the disk space usage of the Nix store by hard-linking + * files with the same contents. + */ void optimiseStore(OptimiseStats & stats); void optimiseStore() override; - /* Optimise a single store path. Optionally, test the encountered - symlinks for corruption. */ + /** + * Optimise a single store path. Optionally, test the encountered + * symlinks for corruption. + */ void optimisePath(const Path & path, RepairFlag repair); bool verifyStore(bool checkContents, RepairFlag repair) override; - /* Register the validity of a path, i.e., that `path' exists, that - the paths referenced by it exists, and in the case of an output - path of a derivation, that it has been produced by a successful - execution of the derivation (or something equivalent). Also - register the hash of the file system contents of the path. The - hash must be a SHA-256 hash. */ + /** + * Register the validity of a path, i.e., that `path` exists, that + * the paths referenced by it exists, and in the case of an output + * path of a derivation, that it has been produced by a successful + * execution of the derivation (or something equivalent). Also + * register the hash of the file system contents of the path. The + * hash must be a SHA-256 hash. + */ void registerValidPath(const ValidPathInfo & info); void registerValidPaths(const ValidPathInfos & infos); unsigned int getProtocol() override; + std::optional isTrustedClient() override; + void vacuumDB(); void repairPath(const StorePath & path) override; void addSignatures(const StorePath & storePath, const StringSet & sigs) override; - /* If free disk space in /nix/store if below minFree, delete - garbage until it exceeds maxFree. */ + /** + * If free disk space in /nix/store if below minFree, delete + * garbage until it exceeds maxFree. + */ void autoGC(bool sync = true); - /* Register the store path 'output' as the output named 'outputName' of - derivation 'deriver'. */ + /** + * Register the store path 'output' as the output named 'outputName' of + * derivation 'deriver'. + */ void registerDrvOutput(const Realisation & info) override; void registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) override; void cacheDrvOutputMapping( @@ -241,7 +283,9 @@ private: void invalidatePath(State & state, const StorePath & path); - /* Delete a path from the Nix store. */ + /** + * Delete a path from the Nix store. + */ void invalidatePathChecked(const StorePath & path); void verifyPath(const Path & path, const StringSet & store, @@ -264,8 +308,6 @@ private: std::pair createTempDirInStore(); - void checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv); - typedef std::unordered_set InodeHash; InodeHash loadInodeHash(); @@ -276,8 +318,10 @@ private: bool isValidPath_(State & state, const StorePath & path); void queryReferrers(State & state, const StorePath & path, StorePathSet & referrers); - /* Add signatures to a ValidPathInfo or Realisation using the secret keys - specified by the ‘secret-key-files’ option. */ + /** + * Add signatures to a ValidPathInfo or Realisation using the secret keys + * specified by the ‘secret-key-files’ option. + */ void signPathInfo(ValidPathInfo & info); void signRealisation(Realisation &); @@ -307,18 +351,23 @@ typedef std::pair Inode; typedef std::set InodesSeen; -/* "Fix", or canonicalise, the meta-data of the files in a store path - after it has been built. In particular: - - the last modification date on each file is set to 1 (i.e., - 00:00:01 1/1/1970 UTC) - - the permissions are set of 444 or 555 (i.e., read-only with or - without execute permission; setuid bits etc. are cleared) - - the owner and group are set to the Nix user and group, if we're - running as root. - If uidRange is not empty, this function will throw an error if it - encounters files owned by a user outside of the closed interval - [uidRange->first, uidRange->second]. -*/ +/** + * "Fix", or canonicalise, the meta-data of the files in a store path + * after it has been built. In particular: + * + * - the last modification date on each file is set to 1 (i.e., + * 00:00:01 1/1/1970 UTC) + * + * - the permissions are set of 444 or 555 (i.e., read-only with or + * without execute permission; setuid bits etc. are cleared) + * + * - the owner and group are set to the Nix user and group, if we're + * running as root. + * + * If uidRange is not empty, this function will throw an error if it + * encounters files owned by a user outside of the closed interval + * [uidRange->first, uidRange->second]. + */ void canonicalisePathMetaData( const Path & path, std::optional> uidRange, diff --git a/src/libstore/local-store.md b/src/libstore/local-store.md new file mode 100644 index 000000000..8174df839 --- /dev/null +++ b/src/libstore/local-store.md @@ -0,0 +1,39 @@ +R"( + +**Store URL format**: `local`, *root* + +This store type accesses a Nix store in the local filesystem directly +(i.e. not via the Nix daemon). *root* is an absolute path that is +prefixed to other directories such as the Nix store directory. The +store pseudo-URL `local` denotes a store that uses `/` as its root +directory. + +A store that uses a *root* other than `/` is called a *chroot +store*. With such stores, the store directory is "logically" still +`/nix/store`, so programs stored in them can only be built and +executed by `chroot`-ing into *root*. Chroot stores only support +building and running on Linux when [`mount namespaces`](https://man7.org/linux/man-pages/man7/mount_namespaces.7.html) and [`user namespaces`](https://man7.org/linux/man-pages/man7/user_namespaces.7.html) are +enabled. + +For example, the following uses `/tmp/root` as the chroot environment +to build or download `nixpkgs#hello` and then execute it: + +```console +# nix run --store /tmp/root nixpkgs#hello +Hello, world! +``` + +Here, the "physical" store location is `/tmp/root/nix/store`, and +Nix's store metadata is in `/tmp/root/nix/var/nix/db`. + +It is also possible, but not recommended, to change the "logical" +location of the Nix store from its default of `/nix/store`. This makes +it impossible to use default substituters such as +`https://cache.nixos.org/`, and thus you may have to build everything +locally. Here is an example: + +```console +# nix build --store 'local?store=/tmp/my-nix/store&state=/tmp/my-nix/state&log=/tmp/my-nix/log' nixpkgs#hello +``` + +)" diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc index 4fe1fcf56..7202a64b3 100644 --- a/src/libstore/lock.cc +++ b/src/libstore/lock.cc @@ -129,7 +129,7 @@ struct AutoUserLock : UserLock useUserNamespace = false; #endif - settings.requireExperimentalFeature(Xp::AutoAllocateUids); + experimentalFeatureSettings.require(Xp::AutoAllocateUids); assert(settings.startId > 0); assert(settings.uidCount % maxIdsPerBuild == 0); assert((uint64_t) settings.startId + (uint64_t) settings.uidCount <= std::numeric_limits::max()); diff --git a/src/libstore/lock.hh b/src/libstore/lock.hh index 7f1934510..1c268e1fb 100644 --- a/src/libstore/lock.hh +++ b/src/libstore/lock.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" @@ -12,14 +13,18 @@ struct UserLock { virtual ~UserLock() { } - /* Get the first and last UID. */ + /** + * Get the first and last UID. + */ std::pair getUIDRange() { auto first = getUID(); return {first, first + getUIDCount() - 1}; } - /* Get the first UID. */ + /** + * Get the first UID. + */ virtual uid_t getUID() = 0; virtual uid_t getUIDCount() = 0; @@ -29,8 +34,10 @@ struct UserLock virtual std::vector getSupplementaryGIDs() = 0; }; -/* Acquire a user lock for a UID range of size `nrIds`. Note that this - may return nullptr if no user is available. */ +/** + * Acquire a user lock for a UID range of size `nrIds`. Note that this + * may return nullptr if no user is available. + */ std::unique_ptr acquireUserLock(uid_t nrIds, bool useUserNamespace); bool useBuildUsers(); diff --git a/src/libstore/log-store.hh b/src/libstore/log-store.hh index e4d95bab6..a84f7dbeb 100644 --- a/src/libstore/log-store.hh +++ b/src/libstore/log-store.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" @@ -9,8 +10,10 @@ struct LogStore : public virtual Store { inline static std::string operationName = "Build log storage and retrieval"; - /* Return the build log of the specified store path, if available, - or null otherwise. */ + /** + * Return the build log of the specified store path, if available, + * or null otherwise. + */ std::optional getBuildLog(const StorePath & path); virtual std::optional getBuildLogExact(const StorePath & path) = 0; diff --git a/src/libstore/machines.hh b/src/libstore/machines.hh index 834626de9..1adeaf1f0 100644 --- a/src/libstore/machines.hh +++ b/src/libstore/machines.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" diff --git a/src/libstore/make-content-addressed.hh b/src/libstore/make-content-addressed.hh index c4a66ed41..2ce6ec7bc 100644 --- a/src/libstore/make-content-addressed.hh +++ b/src/libstore/make-content-addressed.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index b28768459..89148d415 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -326,7 +326,7 @@ OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, throw Error( "the derivation '%s' doesn't have an output named '%s'", store.printStorePath(bfd.drvPath), output); - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { DrvOutput outputId { *outputHash, output }; auto realisation = store.queryRealisation(outputId); if (!realisation) diff --git a/src/libstore/names.hh b/src/libstore/names.hh index 3977fc6cc..d82b99bb4 100644 --- a/src/libstore/names.hh +++ b/src/libstore/names.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 9a0003588..f0dfcb19b 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -275,6 +275,7 @@ json listNar(ref accessor, const Path & path, bool recurse) obj["type"] = "symlink"; obj["target"] = accessor->readLink(path); break; + case FSAccessor::Type::tMissing: default: throw Error("path '%s' does not exist in NAR", path); } diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh index 7d998ae0b..5e19bd3c7 100644 --- a/src/libstore/nar-accessor.hh +++ b/src/libstore/nar-accessor.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include @@ -9,24 +10,30 @@ namespace nix { struct Source; -/* Return an object that provides access to the contents of a NAR - file. */ +/** + * Return an object that provides access to the contents of a NAR + * file. + */ ref makeNarAccessor(std::string && nar); ref makeNarAccessor(Source & source); -/* Create a NAR accessor from a NAR listing (in the format produced by - listNar()). The callback getNarBytes(offset, length) is used by the - readFile() method of the accessor to get the contents of files - inside the NAR. */ +/** + * Create a NAR accessor from a NAR listing (in the format produced by + * listNar()). The callback getNarBytes(offset, length) is used by the + * readFile() method of the accessor to get the contents of files + * inside the NAR. + */ typedef std::function GetNarBytes; ref makeLazyNarAccessor( const std::string & listing, GetNarBytes getNarBytes); -/* Write a JSON representation of the contents of a NAR (except file - contents). */ +/** + * Write a JSON representation of the contents of a NAR (except file + * contents). + */ nlohmann::json listNar(ref accessor, const Path & path, bool recurse); } diff --git a/src/libstore/nar-info-disk-cache.hh b/src/libstore/nar-info-disk-cache.hh index 4877f56d8..bbd1d05d5 100644 --- a/src/libstore/nar-info-disk-cache.hh +++ b/src/libstore/nar-info-disk-cache.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "ref.hh" #include "nar-info.hh" @@ -42,8 +43,10 @@ public: const std::string & uri, const DrvOutput & id) = 0; }; -/* Return a singleton cache object that can be used concurrently by - multiple threads. */ +/** + * Return a singleton cache object that can be used concurrently by + * multiple threads. + */ ref getNarInfoDiskCache(); ref getTestNarInfoDiskCache(Path dbPath); diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 01683ec73..3cae8e659 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "hash.hh" diff --git a/src/libstore/outputs-spec.hh b/src/libstore/outputs-spec.hh index 46bc35ebc..5a726fe90 100644 --- a/src/libstore/outputs-spec.hh +++ b/src/libstore/outputs-spec.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -9,6 +10,9 @@ namespace nix { +/** + * A non-empty set of outputs, specified by name + */ struct OutputNames : std::set { using std::set::set; @@ -18,6 +22,9 @@ struct OutputNames : std::set { : std::set(s) { assert(!empty()); } + /** + * Needs to be "inherited manually" + */ OutputNames(std::set && s) : std::set(s) { assert(!empty()); } @@ -28,6 +35,9 @@ struct OutputNames : std::set { OutputNames() = delete; }; +/** + * The set of all outputs, without needing to name them explicitly + */ struct AllOutputs : std::monostate { }; typedef std::variant _OutputsSpecRaw; @@ -36,7 +46,9 @@ struct OutputsSpec : _OutputsSpecRaw { using Raw = _OutputsSpecRaw; using Raw::Raw; - /* Force choosing a variant */ + /** + * Force choosing a variant + */ OutputsSpec() = delete; using Names = OutputNames; @@ -52,14 +64,20 @@ struct OutputsSpec : _OutputsSpecRaw { bool contains(const std::string & output) const; - /* Create a new OutputsSpec which is the union of this and that. */ + /** + * Create a new OutputsSpec which is the union of this and that. + */ OutputsSpec union_(const OutputsSpec & that) const; - /* Whether this OutputsSpec is a subset of that. */ + /** + * Whether this OutputsSpec is a subset of that. + */ bool isSubsetOf(const OutputsSpec & outputs) const; - /* Parse a string of the form 'output1,...outputN' or - '*', returning the outputs spec. */ + /** + * Parse a string of the form 'output1,...outputN' or '*', returning + * the outputs spec. + */ static OutputsSpec parse(std::string_view s); static std::optional parseOpt(std::string_view s); @@ -81,8 +99,10 @@ struct ExtendedOutputsSpec : _ExtendedOutputsSpecRaw { return static_cast(*this); } - /* Parse a string of the form 'prefix^output1,...outputN' or - 'prefix^*', returning the prefix and the extended outputs spec. */ + /** + * Parse a string of the form 'prefix^output1,...outputN' or + * 'prefix^*', returning the prefix and the extended outputs spec. + */ static std::pair parse(std::string_view s); static std::optional> parseOpt(std::string_view s); diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index bfb3857c0..71085a604 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "derivations.hh" #include "store-api.hh" diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index a7fcbd232..9af1309a0 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "crypto.hh" #include "path.hh" @@ -18,8 +19,14 @@ struct SubstitutablePathInfo { std::optional deriver; StorePathSet references; - uint64_t downloadSize; /* 0 = unknown or inapplicable */ - uint64_t narSize; /* 0 = unknown */ + /** + * 0 = unknown or inapplicable + */ + uint64_t downloadSize; + /** + * 0 = unknown + */ + uint64_t narSize; }; typedef std::map SubstitutablePathInfos; @@ -29,35 +36,40 @@ struct ValidPathInfo { StorePath path; std::optional deriver; - // TODO document this + /** + * \todo document this + */ Hash narHash; StorePathSet references; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown uint64_t id; // internal use only - /* Whether the path is ultimately trusted, that is, it's a - derivation output that was built locally. */ + /** + * Whether the path is ultimately trusted, that is, it's a + * derivation output that was built locally. + */ bool ultimate = false; StringSet sigs; // note: not necessarily verified - /* If non-empty, an assertion that the path is content-addressed, - i.e., that the store path is computed from a cryptographic hash - of the contents of the path, plus some other bits of data like - the "name" part of the path. Such a path doesn't need - signatures, since we don't have to trust anybody's claim that - the path is the output of a particular derivation. (In the - extensional store model, we have to trust that the *contents* - of an output path of a derivation were actually produced by - that derivation. In the intensional model, we have to trust - that a particular output path was produced by a derivation; the - path then implies the contents.) - - Ideally, the content-addressability assertion would just be a Boolean, - and the store path would be computed from the name component, ‘narHash’ - and ‘references’. However, we support many types of content addresses. - */ + /** + * If non-empty, an assertion that the path is content-addressed, + * i.e., that the store path is computed from a cryptographic hash + * of the contents of the path, plus some other bits of data like + * the "name" part of the path. Such a path doesn't need + * signatures, since we don't have to trust anybody's claim that + * the path is the output of a particular derivation. (In the + * extensional store model, we have to trust that the *contents* + * of an output path of a derivation were actually produced by + * that derivation. In the intensional model, we have to trust + * that a particular output path was produced by a derivation; the + * path then implies the contents.) + * + * Ideally, the content-addressability assertion would just be a Boolean, + * and the store path would be computed from the name component, ‘narHash’ + * and ‘references’. However, we support many types of content addresses. + */ std::optional ca; bool operator == (const ValidPathInfo & i) const @@ -68,27 +80,35 @@ struct ValidPathInfo && references == i.references; } - /* Return a fingerprint of the store path to be used in binary - cache signatures. It contains the store path, the base-32 - SHA-256 hash of the NAR serialisation of the path, the size of - the NAR, and the sorted references. The size field is strictly - speaking superfluous, but might prevent endless/excessive data - attacks. */ + /** + * Return a fingerprint of the store path to be used in binary + * cache signatures. It contains the store path, the base-32 + * SHA-256 hash of the NAR serialisation of the path, the size of + * the NAR, and the sorted references. The size field is strictly + * speaking superfluous, but might prevent endless/excessive data + * attacks. + */ std::string fingerprint(const Store & store) const; void sign(const Store & store, const SecretKey & secretKey); - /* Return true iff the path is verifiably content-addressed. */ + /** + * @return true iff the path is verifiably content-addressed. + */ bool isContentAddressed(const Store & store) const; static const size_t maxSigs = std::numeric_limits::max(); - /* Return the number of signatures on this .narinfo that were - produced by one of the specified keys, or maxSigs if the path - is content-addressed. */ + /** + * Return the number of signatures on this .narinfo that were + * produced by one of the specified keys, or maxSigs if the path + * is content-addressed. + */ size_t checkSignatures(const Store & store, const PublicKeys & publicKeys) const; - /* Verify a single signature. */ + /** + * Verify a single signature. + */ bool checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const; Strings shortRefs() const; diff --git a/src/libstore/path-regex.hh b/src/libstore/path-regex.hh index 6893c3876..4f8dc4c1f 100644 --- a/src/libstore/path-regex.hh +++ b/src/libstore/path-regex.hh @@ -1,4 +1,5 @@ #pragma once +///@file namespace nix { diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh index 5d25656a5..d75850868 100644 --- a/src/libstore/path-with-outputs.hh +++ b/src/libstore/path-with-outputs.hh @@ -1,17 +1,19 @@ #pragma once +///@file #include "path.hh" #include "derived-path.hh" namespace nix { -/* This is a deprecated old type just for use by the old CLI, and older - versions of the RPC protocols. In new code don't use it; you want - `DerivedPath` instead. - - `DerivedPath` is better because it handles more cases, and does so more - explicitly without devious punning tricks. -*/ +/** + * This is a deprecated old type just for use by the old CLI, and older + * versions of the RPC protocols. In new code don't use it; you want + * `DerivedPath` instead. + * + * `DerivedPath` is better because it handles more cases, and does so more + * explicitly without devious punning tricks. + */ struct StorePathWithOutputs { StorePath path; @@ -30,9 +32,11 @@ std::pair parsePathWithOutputs(std::string_view s); class Store; -/* Split a string specifying a derivation and a set of outputs - (/nix/store/hash-foo!out1,out2,...) into the derivation path - and the outputs. */ +/** + * Split a string specifying a derivation and a set of outputs + * (/nix/store/hash-foo!out1,out2,...) into the derivation path + * and the outputs. + */ StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs); StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs); diff --git a/src/libstore/path.hh b/src/libstore/path.hh index 1e5579b90..4ca6747b3 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include @@ -8,13 +9,22 @@ namespace nix { struct Hash; +/** + * \ref StorePath "Store path" is the fundamental reference type of Nix. + * A store paths refers to a Store object. + * + * See glossary.html#gloss-store-path for more information on a + * conceptual level. + */ class StorePath { std::string baseName; public: - /* Size of the hash part of store paths, in base-32 characters. */ + /** + * Size of the hash part of store paths, in base-32 characters. + */ constexpr static size_t HashLen = 32; // i.e. 160 bits constexpr static size_t MaxPathLen = 211; @@ -45,8 +55,9 @@ public: return baseName != other.baseName; } - /* Check whether a file name ends with the extension for - derivations. */ + /** + * Check whether a file name ends with the extension for derivations. + */ bool isDerivation() const; std::string_view name() const @@ -67,7 +78,10 @@ public: typedef std::set StorePathSet; typedef std::vector StorePaths; -/* Extension of derivations in the Nix store. */ +/** + * The file extension of \ref Derivation derivations when serialized + * into store objects. + */ const std::string drvExtension = ".drv"; } diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh index 5e3a734b4..4921df352 100644 --- a/src/libstore/pathlocks.hh +++ b/src/libstore/pathlocks.hh @@ -1,15 +1,20 @@ #pragma once +///@file #include "util.hh" namespace nix { -/* Open (possibly create) a lock file and return the file descriptor. - -1 is returned if create is false and the lock could not be opened - because it doesn't exist. Any other error throws an exception. */ +/** + * Open (possibly create) a lock file and return the file descriptor. + * -1 is returned if create is false and the lock could not be opened + * because it doesn't exist. Any other error throws an exception. + */ AutoCloseFD openLockFile(const Path & path, bool create); -/* Delete an open lock file. */ +/** + * Delete an open lock file. + */ void deleteLockFile(const Path & path, int fd); enum LockType { ltRead, ltWrite, ltNone }; diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 179161ff7..ba5c8583f 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -282,28 +282,48 @@ std::string optimisticLockProfile(const Path & profile) Path profilesDir() { - auto profileRoot = createNixStateDir() + "/profiles"; + auto profileRoot = + (getuid() == 0) + ? rootProfilesDir() + : createNixStateDir() + "/profiles"; createDirs(profileRoot); return profileRoot; } +Path rootProfilesDir() +{ + return settings.nixStateDir + "/profiles/per-user/root"; +} + Path getDefaultProfile() { Path profileLink = settings.useXDGBaseDirectories ? createNixStateDir() + "/profile" : getHome() + "/.nix-profile"; try { - auto profile = - getuid() == 0 - ? settings.nixStateDir + "/profiles/default" - : profilesDir() + "/profile"; + auto profile = profilesDir() + "/profile"; if (!pathExists(profileLink)) { replaceSymlink(profile, profileLink); } + // Backwards compatibiliy measure: Make root's profile available as + // `.../default` as it's what NixOS and most of the init scripts expect + Path globalProfileLink = settings.nixStateDir + "/profiles/default"; + if (getuid() == 0 && !pathExists(globalProfileLink)) { + replaceSymlink(profile, globalProfileLink); + } return absPath(readLink(profileLink), dirOf(profileLink)); } catch (Error &) { return profileLink; } } +Path defaultChannelsDir() +{ + return profilesDir() + "/channels"; +} + +Path rootChannelsDir() +{ + return rootProfilesDir() + "/channels"; +} } diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh index fbf95b850..4e1f42e83 100644 --- a/src/libstore/profiles.hh +++ b/src/libstore/profiles.hh @@ -1,6 +1,7 @@ #pragma once +///@file -#include "types.hh" + #include "types.hh" #include "pathlocks.hh" #include @@ -23,9 +24,11 @@ struct Generation typedef std::list Generations; -/* Returns the list of currently present generations for the specified - profile, sorted by generation number. Also returns the number of - the current generation. */ +/** + * Returns the list of currently present generations for the specified + * profile, sorted by generation number. Also returns the number of + * the current generation. + */ std::pair> findGenerations(Path profile); class LocalFSStore; @@ -46,35 +49,60 @@ void deleteGenerationsOlderThan(const Path & profile, std::string_view timeSpec, void switchLink(Path link, Path target); -/* Roll back a profile to the specified generation, or to the most - recent one older than the current. */ +/** + * Roll back a profile to the specified generation, or to the most + * recent one older than the current. + */ void switchGeneration( const Path & profile, std::optional dstGen, bool dryRun); -/* Ensure exclusive access to a profile. Any command that modifies - the profile first acquires this lock. */ +/** + * Ensure exclusive access to a profile. Any command that modifies + * the profile first acquires this lock. + */ void lockProfile(PathLocks & lock, const Path & profile); -/* Optimistic locking is used by long-running operations like `nix-env - -i'. Instead of acquiring the exclusive lock for the entire - duration of the operation, we just perform the operation - optimistically (without an exclusive lock), and check at the end - whether the profile changed while we were busy (i.e., the symlink - target changed). If so, the operation is restarted. Restarting is - generally cheap, since the build results are still in the Nix - store. Most of the time, only the user environment has to be - rebuilt. */ +/** + * Optimistic locking is used by long-running operations like `nix-env + * -i'. Instead of acquiring the exclusive lock for the entire + * duration of the operation, we just perform the operation + * optimistically (without an exclusive lock), and check at the end + * whether the profile changed while we were busy (i.e., the symlink + * target changed). If so, the operation is restarted. Restarting is + * generally cheap, since the build results are still in the Nix + * store. Most of the time, only the user environment has to be + * rebuilt. + */ std::string optimisticLockProfile(const Path & profile); -/* Creates and returns the path to a directory suitable for storing the user’s - profiles. */ +/** + * Create and return the path to a directory suitable for storing the user’s + * profiles. + */ Path profilesDir(); -/* Resolve the default profile (~/.nix-profile by default, $XDG_STATE_HOME/ - nix/profile if XDG Base Directory Support is enabled), and create if doesn't - exist */ +/** + * Return the path to the profile directory for root (but don't try creating it) + */ +Path rootProfilesDir(); + +/** + * Create and return the path to the file used for storing the users's channels + */ +Path defaultChannelsDir(); + +/** + * Return the path to the channel directory for root (but don't try creating it) + */ +Path rootChannelsDir(); + +/** + * Resolve the default profile (~/.nix-profile by default, + * $XDG_STATE_HOME/nix/profile if XDG Base Directory Support is enabled), + * and create if doesn't exist + */ Path getDefaultProfile(); } diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 48d0283de..a18cf2aa8 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include diff --git a/src/libstore/references.hh b/src/libstore/references.hh index 6f381f96c..52d71b333 100644 --- a/src/libstore/references.hh +++ b/src/libstore/references.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "hash.hh" #include "path.hh" diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh index 99f5544ef..e2673b6f6 100644 --- a/src/libstore/remote-fs-accessor.hh +++ b/src/libstore/remote-fs-accessor.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "fs-accessor.hh" #include "ref.hh" diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index d1296627a..e128c3a29 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -42,6 +42,40 @@ void write(const Store & store, Sink & out, const StorePath & storePath) } +std::optional read(const Store & store, Source & from, Phantom> _) +{ + auto temp = readNum(from); + switch (temp) { + case 0: + return std::nullopt; + case 1: + return { Trusted }; + case 2: + return { NotTrusted }; + default: + throw Error("Invalid trusted status from remote"); + } +} + +void write(const Store & store, Sink & out, const std::optional & optTrusted) +{ + if (!optTrusted) + out << (uint8_t)0; + else { + switch (*optTrusted) { + case Trusted: + out << (uint8_t)1; + break; + case NotTrusted: + out << (uint8_t)2; + break; + default: + assert(false); + }; + } +} + + ContentAddress read(const Store & store, Source & from, Phantom _) { return parseContentAddress(readString(from)); @@ -226,6 +260,13 @@ void RemoteStore::initConnection(Connection & conn) conn.daemonNixVersion = readString(conn.from); } + if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 35) { + conn.remoteTrustsUs = worker_proto::read(*this, conn.from, Phantom> {}); + } else { + // We don't know the answer; protocol to old. + conn.remoteTrustsUs = std::nullopt; + } + auto ex = conn.processStderr(); if (ex) std::rethrow_exception(ex); } @@ -265,7 +306,7 @@ void RemoteStore::setOptions(Connection & conn) overrides.erase(settings.buildCores.name); overrides.erase(settings.useSubstitutes.name); overrides.erase(loggerSettings.showTrace.name); - overrides.erase(settings.experimentalFeatures.name); + overrides.erase(experimentalFeatureSettings.experimentalFeatures.name); overrides.erase(settings.pluginFiles.name); conn.to << overrides.size(); for (auto & i : overrides) @@ -876,7 +917,7 @@ std::vector RemoteStore::buildPathsWithResults( "the derivation '%s' doesn't have an output named '%s'", printStorePath(bfd.drvPath), output); auto outputId = DrvOutput{ *outputHash, output }; - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { auto realisation = queryRealisation(outputId); if (!realisation) @@ -1082,6 +1123,11 @@ unsigned int RemoteStore::getProtocol() return conn->daemonVersion; } +std::optional RemoteStore::isTrustedClient() +{ + auto conn(getConnection()); + return conn->remoteTrustsUs; +} void RemoteStore::flushBadConnections() { diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 8cd7cc822..1c45f543e 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -22,15 +23,19 @@ struct RemoteStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; - const Setting maxConnections{(StoreConfig*) this, 1, - "max-connections", "maximum number of concurrent connections to the Nix daemon"}; + const Setting maxConnections{(StoreConfig*) this, 1, "max-connections", + "Maximum number of concurrent connections to the Nix daemon."}; - const Setting maxConnectionAge{(StoreConfig*) this, std::numeric_limits::max(), - "max-connection-age", "number of seconds to reuse a connection"}; + const Setting maxConnectionAge{(StoreConfig*) this, + std::numeric_limits::max(), + "max-connection-age", + "Maximum age of a connection before it is closed."}; }; -/* FIXME: RemoteStore is a misnomer - should be something like - DaemonStore. */ +/** + * \todo RemoteStore is a misnomer - should be something like + * DaemonStore. + */ class RemoteStore : public virtual RemoteStoreConfig, public virtual Store, public virtual GcStore, @@ -66,7 +71,9 @@ public: void querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) override; - /* Add a content-addressable store path. `dump` will be drained. */ + /** + * Add a content-addressable store path. `dump` will be drained. + */ ref addCAToStore( Source & dump, std::string_view name, @@ -74,7 +81,9 @@ public: const StorePathSet & references, RepairFlag repair); - /* Add a content-addressable store path. Does not support references. `dump` will be drained. */ + /** + * Add a content-addressable store path. Does not support references. `dump` will be drained. + */ StorePath addToStoreFromDump(Source & dump, std::string_view name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) override; @@ -141,6 +150,8 @@ public: unsigned int getProtocol() override; + std::optional isTrustedClient() override; + void flushBadConnections(); struct Connection @@ -148,6 +159,7 @@ public: FdSink to; FdSource from; unsigned int daemonVersion; + std::optional remoteTrustsUs; std::optional daemonNixVersion; std::chrono::time_point startTime; diff --git a/src/libstore/repair-flag.hh b/src/libstore/repair-flag.hh index a13cda312..f412d6a20 100644 --- a/src/libstore/repair-flag.hh +++ b/src/libstore/repair-flag.hh @@ -1,4 +1,5 @@ #pragma once +///@file namespace nix { diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 8006bd733..d2fc6abaf 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -192,19 +192,72 @@ S3BinaryCacheStore::S3BinaryCacheStore(const Params & params) struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig { using BinaryCacheStoreConfig::BinaryCacheStoreConfig; - const Setting profile{(StoreConfig*) this, "", "profile", "The name of the AWS configuration profile to use."}; - const Setting region{(StoreConfig*) this, Aws::Region::US_EAST_1, "region", {"aws-region"}}; - const Setting scheme{(StoreConfig*) this, "", "scheme", "The scheme to use for S3 requests, https by default."}; - const Setting endpoint{(StoreConfig*) this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."}; - const Setting narinfoCompression{(StoreConfig*) this, "", "narinfo-compression", "compression method for .narinfo files"}; - const Setting lsCompression{(StoreConfig*) this, "", "ls-compression", "compression method for .ls files"}; - const Setting logCompression{(StoreConfig*) this, "", "log-compression", "compression method for log/* files"}; + + const Setting profile{(StoreConfig*) this, "", "profile", + R"( + The name of the AWS configuration profile to use. By default + Nix will use the `default` profile. + )"}; + + const Setting region{(StoreConfig*) this, Aws::Region::US_EAST_1, "region", + R"( + The region of the S3 bucket. If your bucket is not in + `us–east-1`, you should always explicitly specify the region + parameter. + )"}; + + const Setting scheme{(StoreConfig*) this, "", "scheme", + R"( + The scheme used for S3 requests, `https` (default) or `http`. This + option allows you to disable HTTPS for binary caches which don't + support it. + + > **Note** + > + > HTTPS should be used if the cache might contain sensitive + > information. + )"}; + + const Setting endpoint{(StoreConfig*) this, "", "endpoint", + R"( + The URL of the endpoint of an S3-compatible service such as MinIO. + Do not specify this setting if you're using Amazon S3. + + > **Note** + > + > This endpoint must support HTTPS and will use path-based + > addressing instead of virtual host based addressing. + )"}; + + const Setting narinfoCompression{(StoreConfig*) this, "", "narinfo-compression", + "Compression method for `.narinfo` files."}; + + const Setting lsCompression{(StoreConfig*) this, "", "ls-compression", + "Compression method for `.ls` files."}; + + const Setting logCompression{(StoreConfig*) this, "", "log-compression", + R"( + Compression method for `log/*` files. It is recommended to + use a compression method supported by most web browsers + (e.g. `brotli`). + )"}; + const Setting multipartUpload{ - (StoreConfig*) this, false, "multipart-upload", "whether to use multi-part uploads"}; + (StoreConfig*) this, false, "multipart-upload", + "Whether to use multi-part uploads."}; + const Setting bufferSize{ - (StoreConfig*) this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"}; + (StoreConfig*) this, 5 * 1024 * 1024, "buffer-size", + "Size (in bytes) of each part in multi-part uploads."}; const std::string name() override { return "S3 Binary Cache Store"; } + + std::string doc() override + { + return + #include "s3-binary-cache-store.md" + ; + } }; struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual S3BinaryCacheStore @@ -456,6 +509,16 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual return paths; } + /** + * For now, we conservatively say we don't know. + * + * \todo try to expose our S3 authentication status. + */ + std::optional isTrustedClient() override + { + return std::nullopt; + } + static std::set uriSchemes() { return {"s3"}; } }; diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh index bce828b11..c62ea5147 100644 --- a/src/libstore/s3-binary-cache-store.hh +++ b/src/libstore/s3-binary-cache-store.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "binary-cache-store.hh" diff --git a/src/libstore/s3-binary-cache-store.md b/src/libstore/s3-binary-cache-store.md new file mode 100644 index 000000000..70fe0eb09 --- /dev/null +++ b/src/libstore/s3-binary-cache-store.md @@ -0,0 +1,8 @@ +R"( + +**Store URL format**: `s3://`*bucket-name* + +This store allows reading and writing a binary cache stored in an AWS +S3 bucket. + +)" diff --git a/src/libstore/s3.hh b/src/libstore/s3.hh index cdb3e5908..f0aeb3bed 100644 --- a/src/libstore/s3.hh +++ b/src/libstore/s3.hh @@ -1,4 +1,5 @@ #pragma once +///@file #if ENABLE_S3 diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh index 3f76baa82..553fd3a09 100644 --- a/src/libstore/serve-protocol.hh +++ b/src/libstore/serve-protocol.hh @@ -1,4 +1,5 @@ #pragma once +///@file namespace nix { diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index 1853731a2..b735838ec 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -10,7 +11,9 @@ struct sqlite3_stmt; namespace nix { -/* RAII wrapper to close a SQLite database automatically. */ +/** + * RAII wrapper to close a SQLite database automatically. + */ struct SQLite { sqlite3 * db = 0; @@ -22,7 +25,9 @@ struct SQLite ~SQLite(); operator sqlite3 * () { return db; } - /* Disable synchronous mode, set truncate journal mode. */ + /** + * Disable synchronous mode, set truncate journal mode. + */ void isCache(); void exec(const std::string & stmt); @@ -30,7 +35,9 @@ struct SQLite uint64_t getLastInsertedRowId(); }; -/* RAII wrapper to create and destroy SQLite prepared statements. */ +/** + * RAII wrapper to create and destroy SQLite prepared statements. + */ struct SQLiteStmt { sqlite3 * db = 0; @@ -42,7 +49,9 @@ struct SQLiteStmt ~SQLiteStmt(); operator sqlite3_stmt * () { return stmt; } - /* Helper for binding / executing statements. */ + /** + * Helper for binding / executing statements. + */ class Use { friend struct SQLiteStmt; @@ -55,7 +64,9 @@ struct SQLiteStmt ~Use(); - /* Bind the next parameter. */ + /** + * Bind the next parameter. + */ Use & operator () (std::string_view value, bool notNull = true); Use & operator () (const unsigned char * data, size_t len, bool notNull = true); Use & operator () (int64_t value, bool notNull = true); @@ -63,11 +74,15 @@ struct SQLiteStmt int step(); - /* Execute a statement that does not return rows. */ + /** + * Execute a statement that does not return rows. + */ void exec(); - /* For statements that return 0 or more rows. Returns true iff - a row is available. */ + /** + * For statements that return 0 or more rows. Returns true iff + * a row is available. + */ bool next(); std::string getStr(int col); @@ -81,8 +96,10 @@ struct SQLiteStmt } }; -/* RAII helper that ensures transactions are aborted unless explicitly - committed. */ +/** + * RAII helper that ensures transactions are aborted unless explicitly + * committed. + */ struct SQLiteTxn { bool active = false; @@ -124,8 +141,10 @@ MakeError(SQLiteBusy, SQLiteError); void handleSQLiteBusy(const SQLiteBusy & e); -/* Convenience function for retrying a SQLite transaction when the - database is busy. */ +/** + * Convenience function for retrying a SQLite transaction when the + * database is busy. + */ template T retrySQLite(F && fun) { diff --git a/src/libstore/ssh-store-config.hh b/src/libstore/ssh-store-config.hh new file mode 100644 index 000000000..c27a5d00f --- /dev/null +++ b/src/libstore/ssh-store-config.hh @@ -0,0 +1,29 @@ +#pragma once +///@file + +#include "store-api.hh" + +namespace nix { + +struct CommonSSHStoreConfig : virtual StoreConfig +{ + using StoreConfig::StoreConfig; + + const Setting sshKey{(StoreConfig*) this, "", "ssh-key", + "Path to the SSH private key used to authenticate to the remote machine."}; + + const Setting sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key", + "The public host key of the remote machine."}; + + const Setting compress{(StoreConfig*) this, false, "compress", + "Whether to enable SSH compression."}; + + const Setting remoteStore{(StoreConfig*) this, "", "remote-store", + R"( + [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + to be used on the remote machine. The default is `auto` + (i.e. use the Nix daemon or `/nix/store` directly). + )"}; +}; + +} diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index cfa063803..962221ad2 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -1,3 +1,4 @@ +#include "ssh-store-config.hh" #include "store-api.hh" #include "remote-store.hh" #include "remote-fs-accessor.hh" @@ -8,17 +9,22 @@ namespace nix { -struct SSHStoreConfig : virtual RemoteStoreConfig +struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig { using RemoteStoreConfig::RemoteStoreConfig; + using CommonSSHStoreConfig::CommonSSHStoreConfig; - const Setting sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"}; - const Setting sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key", "The public half of the host's SSH key"}; - const Setting compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"}; - const Setting remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"}; - const Setting remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"}; + const Setting remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program", + "Path to the `nix-daemon` executable on the remote machine."}; - const std::string name() override { return "SSH Store"; } + const std::string name() override { return "Experimental SSH Store"; } + + std::string doc() override + { + return + #include "ssh-store.md" + ; + } }; class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore @@ -28,6 +34,7 @@ public: SSHStore(const std::string & scheme, const std::string & host, const Params & params) : StoreConfig(params) , RemoteStoreConfig(params) + , CommonSSHStoreConfig(params) , SSHStoreConfig(params) , Store(params) , RemoteStore(params) diff --git a/src/libstore/ssh-store.md b/src/libstore/ssh-store.md new file mode 100644 index 000000000..881537e71 --- /dev/null +++ b/src/libstore/ssh-store.md @@ -0,0 +1,8 @@ +R"( + +**Store URL format**: `ssh-ng://[username@]hostname` + +Experimental store type that allows full access to a Nix store on a +remote machine. + +)" diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 69bfe3418..6f6deda51 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -1,4 +1,5 @@ #include "ssh.hh" +#include "finally.hh" namespace nix { @@ -35,6 +36,9 @@ void SSHMaster::addCommonSSHOpts(Strings & args) } if (compress) args.push_back("-C"); + + args.push_back("-oPermitLocalCommand=yes"); + args.push_back("-oLocalCommand=echo started"); } std::unique_ptr SSHMaster::startCommand(const std::string & command) @@ -49,6 +53,11 @@ std::unique_ptr SSHMaster::startCommand(const std::string ProcessOptions options; options.dieWithParent = false; + if (!fakeSSH && !useMaster) { + logger->pause(); + } + Finally cleanup = [&]() { logger->resume(); }; + conn->sshPid = startProcess([&]() { restoreProcessContext(); @@ -86,6 +95,18 @@ std::unique_ptr SSHMaster::startCommand(const std::string in.readSide = -1; out.writeSide = -1; + // Wait for the SSH connection to be established, + // So that we don't overwrite the password prompt with our progress bar. + if (!fakeSSH && !useMaster) { + std::string reply; + try { + reply = readLine(out.readSide.get()); + } catch (EndOfFile & e) { } + + if (reply != "started") + throw Error("failed to start SSH connection to '%s'", host); + } + conn->out = std::move(out.readSide); conn->in = std::move(in.writeSide); @@ -109,6 +130,9 @@ Path SSHMaster::startMaster() ProcessOptions options; options.dieWithParent = false; + logger->pause(); + Finally cleanup = [&]() { logger->resume(); }; + state->sshMaster = startProcess([&]() { restoreProcessContext(); @@ -117,11 +141,7 @@ Path SSHMaster::startMaster() if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1) throw SysError("duping over stdout"); - Strings args = - { "ssh", host.c_str(), "-M", "-N", "-S", state->socketPath - , "-o", "LocalCommand=echo started" - , "-o", "PermitLocalCommand=yes" - }; + Strings args = { "ssh", host.c_str(), "-M", "-N", "-S", state->socketPath }; if (verbosity >= lvlChatty) args.push_back("-v"); addCommonSSHOpts(args); diff --git a/src/libstore/ssh.hh b/src/libstore/ssh.hh index dabbcedda..c86a8a986 100644 --- a/src/libstore/ssh.hh +++ b/src/libstore/ssh.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "util.hh" #include "sync.hh" diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 19b0a7f5f..8fc3ed46a 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -445,10 +445,10 @@ StringSet StoreConfig::getDefaultSystemFeatures() { auto res = settings.systemFeatures.get(); - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) res.insert("ca-derivations"); - if (settings.isExperimentalFeatureEnabled(Xp::RecursiveNix)) + if (experimentalFeatureSettings.isEnabled(Xp::RecursiveNix)) res.insert("recursive-nix"); return res; @@ -507,6 +507,54 @@ StorePathSet Store::queryDerivationOutputs(const StorePath & path) return outputPaths; } + +void Store::querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) +{ + if (!settings.useSubstitutes) return; + for (auto & sub : getDefaultSubstituters()) { + for (auto & path : paths) { + if (infos.count(path.first)) + // Choose first succeeding substituter. + continue; + + auto subPath(path.first); + + // Recompute store path so that we can use a different store root. + if (path.second) { + subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second); + if (sub->storeDir == storeDir) + assert(subPath == path.first); + if (subPath != path.first) + debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path.first), sub->printStorePath(subPath), sub->getUri()); + } else if (sub->storeDir != storeDir) continue; + + debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath)); + try { + auto info = sub->queryPathInfo(subPath); + + if (sub->storeDir != storeDir && !(info->isContentAddressed(*sub) && info->references.empty())) + continue; + + auto narInfo = std::dynamic_pointer_cast( + std::shared_ptr(info)); + infos.insert_or_assign(path.first, SubstitutablePathInfo{ + info->deriver, + info->references, + narInfo ? narInfo->fileSize : 0, + info->narSize}); + } catch (InvalidPath &) { + } catch (SubstituterDisabled &) { + } catch (Error & e) { + if (settings.tryFallback) + logError(e.info()); + else + throw; + } + } + } +} + + bool Store::isValidPath(const StorePath & storePath) { { @@ -1017,7 +1065,7 @@ std::map copyPaths( for (auto & path : paths) { storePaths.insert(path.path()); if (auto realisation = std::get_if(&path.raw)) { - settings.requireExperimentalFeature(Xp::CaDerivations); + experimentalFeatureSettings.require(Xp::CaDerivations); toplevelRealisations.insert(*realisation); } } @@ -1101,7 +1149,8 @@ std::map copyPaths( return storePathForDst; }; - uint64_t total = 0; + // total is accessed by each copy, which are each handled in separate threads + std::atomic total = 0; for (auto & missingPath : sortedMissing) { auto info = srcStore.queryPathInfo(missingPath); @@ -1250,7 +1299,7 @@ std::optional Store::getBuildDerivationPath(const StorePath & path) } } - if (!settings.isExperimentalFeatureEnabled(Xp::CaDerivations) || !isValidPath(path)) + if (!experimentalFeatureSettings.isEnabled(Xp::CaDerivations) || !isValidPath(path)) return path; auto drv = readDerivation(path); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 4d8db3596..3cb48bff5 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "nar-info.hh" #include "realisation.hh" @@ -55,7 +56,10 @@ namespace nix { */ MakeError(SubstError, Error); -MakeError(BuildError, Error); // denotes a permanent build failure +/** + * denotes a permanent build failure + */ +MakeError(BuildError, Error); MakeError(InvalidPath, Error); MakeError(Unsupported, Error); MakeError(SubstituteGone, Error); @@ -78,11 +82,14 @@ enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true }; enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true }; enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true }; -/* Magic header of exportPath() output (obsolete). */ +/** + * Magic header of exportPath() output (obsolete). + */ const uint32_t exportMagic = 0x4558494e; enum BuildMode { bmNormal, bmRepair, bmCheck }; +enum TrustedFlag : bool { NotTrusted = false, Trusted = true }; struct BuildResult; @@ -101,17 +108,41 @@ struct StoreConfig : public Config virtual const std::string name() = 0; + virtual std::string doc() + { + return ""; + } + const PathSetting storeDir_{this, false, settings.nixStore, - "store", "path to the Nix store"}; + "store", + R"( + Logical location of the Nix store, usually + `/nix/store`. Note that you can only copy store paths + between stores if they have the same `store` setting. + )"}; const Path storeDir = storeDir_; - const Setting pathInfoCacheSize{this, 65536, "path-info-cache-size", "size of the in-memory store path information cache"}; + const Setting pathInfoCacheSize{this, 65536, "path-info-cache-size", + "Size of the in-memory store path metadata cache."}; - const Setting isTrusted{this, false, "trusted", "whether paths from this store can be used as substitutes even when they lack trusted signatures"}; + const Setting isTrusted{this, false, "trusted", + R"( + Whether paths from this store can be used as substitutes + even if they are not signed by a key listed in the + [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) + setting. + )"}; - Setting priority{this, 0, "priority", "priority of this substituter (lower value means higher priority)"}; + Setting priority{this, 0, "priority", + R"( + Priority of this store when used as a substituter. A lower value means a higher priority. + )"}; - Setting wantMassQuery{this, false, "want-mass-query", "whether this substituter can be queried efficiently for path validity"}; + Setting wantMassQuery{this, false, "want-mass-query", + R"( + Whether this store (when used as a substituter) can be + queried efficiently for path validity. + )"}; Setting systemFeatures{this, getDefaultSystemFeatures(), "system-features", @@ -125,23 +156,30 @@ public: typedef std::map Params; - - protected: struct PathInfoCacheValue { - // Time of cache entry creation or update + /** + * Time of cache entry creation or update + */ std::chrono::time_point time_point = std::chrono::steady_clock::now(); - // Null if missing + /** + * Null if missing + */ std::shared_ptr value; - // Whether the value is valid as a cache entry. The path may not exist. + /** + * Whether the value is valid as a cache entry. The path may not + * exist. + */ bool isKnownNow(); - // Past tense, because a path can only be assumed to exists when - // isKnownNow() && didExist() + /** + * Past tense, because a path can only be assumed to exists when + * isKnownNow() && didExist() + */ inline bool didExist() { return value != nullptr; } @@ -175,35 +213,53 @@ public: std::string printStorePath(const StorePath & path) const; - // FIXME: remove + /** + * Deprecated + * + * \todo remove + */ StorePathSet parseStorePathSet(const PathSet & paths) const; PathSet printStorePathSet(const StorePathSet & path) const; - /* Display a set of paths in human-readable form (i.e., between quotes - and separated by commas). */ + /** + * Display a set of paths in human-readable form (i.e., between quotes + * and separated by commas). + */ std::string showPaths(const StorePathSet & paths); - /* Return true if ‘path’ is in the Nix store (but not the Nix - store itself). */ + /** + * @return true if ‘path’ is in the Nix store (but not the Nix + * store itself). + */ bool isInStore(PathView path) const; - /* Return true if ‘path’ is a store path, i.e. a direct child of - the Nix store. */ + /** + * @return true if ‘path’ is a store path, i.e. a direct child of the + * Nix store. + */ bool isStorePath(std::string_view path) const; - /* Split a path like /nix/store/-/ into - /nix/store/- and /. */ + /** + * Split a path like /nix/store/-/ into + * /nix/store/- and /. + */ std::pair toStorePath(PathView path) const; - /* Follow symlinks until we end up with a path in the Nix store. */ + /** + * Follow symlinks until we end up with a path in the Nix store. + */ Path followLinksToStore(std::string_view path) const; - /* Same as followLinksToStore(), but apply toStorePath() to the - result. */ + /** + * Same as followLinksToStore(), but apply toStorePath() to the + * result. + */ StorePath followLinksToStorePath(std::string_view path) const; - /* Constructs a unique store path name. */ + /** + * Constructs a unique store path name. + */ StorePath makeStorePath(std::string_view type, std::string_view hash, std::string_view name) const; StorePath makeStorePath(std::string_view type, @@ -224,33 +280,40 @@ public: const StorePathSet & references = {}, bool hasSelfReference = false) const; - /* This is the preparatory part of addToStore(); it computes the - store path to which srcPath is to be copied. Returns the store - path and the cryptographic hash of the contents of srcPath. */ + /** + * Preparatory part of addToStore(). + * + * @return the store path to which srcPath is to be copied + * and the cryptographic hash of the contents of srcPath. + */ std::pair computeStorePathForPath(std::string_view name, const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const; - /* Preparatory part of addTextToStore(). - - !!! Computation of the path should take the references given to - addTextToStore() into account, otherwise we have a (relatively - minor) security hole: a caller can register a source file with - bogus references. If there are too many references, the path may - not be garbage collected when it has to be (not really a problem, - the caller could create a root anyway), or it may be garbage - collected when it shouldn't be (more serious). - - Hashing the references would solve this (bogus references would - simply yield a different store path, so other users wouldn't be - affected), but it has some backwards compatibility issues (the - hashing scheme changes), so I'm not doing that for now. */ + /** + * Preparatory part of addTextToStore(). + * + * !!! Computation of the path should take the references given to + * addTextToStore() into account, otherwise we have a (relatively + * minor) security hole: a caller can register a source file with + * bogus references. If there are too many references, the path may + * not be garbage collected when it has to be (not really a problem, + * the caller could create a root anyway), or it may be garbage + * collected when it shouldn't be (more serious). + * + * Hashing the references would solve this (bogus references would + * simply yield a different store path, so other users wouldn't be + * affected), but it has some backwards compatibility issues (the + * hashing scheme changes), so I'm not doing that for now. + */ StorePath computeStorePathForText( std::string_view name, std::string_view s, const StorePathSet & references) const; - /* Check whether a path is valid. */ + /** + * Check whether a path is valid. + */ bool isValidPath(const StorePath & path); protected: @@ -259,53 +322,68 @@ protected: public: - /* If requested, substitute missing paths. This - implements nix-copy-closure's --use-substitutes - flag. */ + /** + * If requested, substitute missing paths. This + * implements nix-copy-closure's --use-substitutes + * flag. + */ void substitutePaths(const StorePathSet & paths); - /* Query which of the given paths is valid. Optionally, try to - substitute missing paths. */ + /** + * Query which of the given paths is valid. Optionally, try to + * substitute missing paths. + */ virtual StorePathSet queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute = NoSubstitute); - /* Query the set of all valid paths. Note that for some store - backends, the name part of store paths may be replaced by 'x' - (i.e. you'll get /nix/store/-x rather than - /nix/store/-). Use queryPathInfo() to obtain the - full store path. FIXME: should return a set of - std::variant to get rid of this hack. */ + /** + * Query the set of all valid paths. Note that for some store + * backends, the name part of store paths may be replaced by 'x' + * (i.e. you'll get /nix/store/-x rather than + * /nix/store/-). Use queryPathInfo() to obtain the + * full store path. FIXME: should return a set of + * std::variant to get rid of this hack. + */ virtual StorePathSet queryAllValidPaths() { unsupported("queryAllValidPaths"); } constexpr static const char * MissingName = "x"; - /* Query information about a valid path. It is permitted to omit - the name part of the store path. */ + /** + * Query information about a valid path. It is permitted to omit + * the name part of the store path. + */ ref queryPathInfo(const StorePath & path); - /* Asynchronous version of queryPathInfo(). */ + /** + * Asynchronous version of queryPathInfo(). + */ void queryPathInfo(const StorePath & path, Callback> callback) noexcept; - /* Query the information about a realisation. */ + /** + * Query the information about a realisation. + */ std::shared_ptr queryRealisation(const DrvOutput &); - /* Asynchronous version of queryRealisation(). */ + /** + * Asynchronous version of queryRealisation(). + */ void queryRealisation(const DrvOutput &, Callback> callback) noexcept; - /* Check whether the given valid path info is sufficiently attested, by - either being signed by a trusted public key or content-addressed, in - order to be included in the given store. - - These same checks would be performed in addToStore, but this allows an - earlier failure in the case where dependencies need to be added too, but - the addToStore wouldn't fail until those dependencies are added. Also, - we don't really want to add the dependencies listed in a nar info we - don't trust anyyways. - */ + /** + * Check whether the given valid path info is sufficiently attested, by + * either being signed by a trusted public key or content-addressed, in + * order to be included in the given store. + * + * These same checks would be performed in addToStore, but this allows an + * earlier failure in the case where dependencies need to be added too, but + * the addToStore wouldn't fail until those dependencies are added. Also, + * we don't really want to add the dependencies listed in a nar info we + * don't trust anyyways. + */ virtual bool pathInfoIsUntrusted(const ValidPathInfo &) { return true; @@ -325,53 +403,77 @@ protected: public: - /* Queries the set of incoming FS references for a store path. - The result is not cleared. */ + /** + * Queries the set of incoming FS references for a store path. + * The result is not cleared. + */ virtual void queryReferrers(const StorePath & path, StorePathSet & referrers) { unsupported("queryReferrers"); } - /* Return all currently valid derivations that have `path' as an - output. (Note that the result of `queryDeriver()' is the - derivation that was actually used to produce `path', which may - not exist anymore.) */ + /** + * @return all currently valid derivations that have `path` as an + * output. + * + * (Note that the result of `queryDeriver()` is the derivation that + * was actually used to produce `path`, which may not exist + * anymore.) + */ virtual StorePathSet queryValidDerivers(const StorePath & path) { return {}; }; - /* Query the outputs of the derivation denoted by `path'. */ + /** + * Query the outputs of the derivation denoted by `path`. + */ virtual StorePathSet queryDerivationOutputs(const StorePath & path); - /* Query the mapping outputName => outputPath for the given derivation. All - outputs are mentioned so ones mising the mapping are mapped to - `std::nullopt`. */ + /** + * Query the mapping outputName => outputPath for the given + * derivation. All outputs are mentioned so ones mising the mapping + * are mapped to `std::nullopt`. + */ virtual std::map> queryPartialDerivationOutputMap(const StorePath & path); - /* Query the mapping outputName=>outputPath for the given derivation. - Assume every output has a mapping and throw an exception otherwise. */ + /** + * Query the mapping outputName=>outputPath for the given derivation. + * Assume every output has a mapping and throw an exception otherwise. + */ OutputPathMap queryDerivationOutputMap(const StorePath & path); - /* Query the full store path given the hash part of a valid store - path, or empty if the path doesn't exist. */ + /** + * Query the full store path given the hash part of a valid store + * path, or empty if the path doesn't exist. + */ virtual std::optional queryPathFromHashPart(const std::string & hashPart) = 0; - /* Query which of the given paths have substitutes. */ + /** + * Query which of the given paths have substitutes. + */ virtual StorePathSet querySubstitutablePaths(const StorePathSet & paths) { return {}; }; - /* Query substitute info (i.e. references, derivers and download - sizes) of a map of paths to their optional ca values. The info - of the first succeeding substituter for each path will be - returned. If a path does not have substitute info, it's omitted - from the resulting ‘infos’ map. */ + /** + * Query substitute info (i.e. references, derivers and download + * sizes) of a map of paths to their optional ca values. The info of + * the first succeeding substituter for each path will be returned. + * If a path does not have substitute info, it's omitted from the + * resulting ‘infos’ map. + */ virtual void querySubstitutablePathInfos(const StorePathCAMap & paths, - SubstitutablePathInfos & infos) { return; }; + SubstitutablePathInfos & infos); - /* Import a path into the store. */ + /** + * Import a path into the store. + */ virtual void addToStore(const ValidPathInfo & info, Source & narSource, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) = 0; - // A list of paths infos along with a source providing the content of the - // associated store path + /** + * A list of paths infos along with a source providing the content + * of the associated store path + */ using PathsSource = std::vector>>; - /* Import multiple paths into the store. */ + /** + * Import multiple paths into the store. + */ virtual void addMultipleToStore( Source & source, RepairFlag repair = NoRepair, @@ -383,10 +485,14 @@ public: RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); - /* Copy the contents of a path to the store and register the - validity the resulting path. The resulting path is returned. - The function object `filter' can be used to exclude files (see - libutil/archive.hh). */ + /** + * Copy the contents of a path to the store and register the + * validity the resulting path. + * + * @return The resulting path is returned. + * @param filter This function can be used to exclude files (see + * libutil/archive.hh). + */ virtual StorePath addToStore( std::string_view name, const Path & srcPath, @@ -396,26 +502,33 @@ public: RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()); - /* Copy the contents of a path to the store and register the - validity the resulting path, using a constant amount of - memory. */ + /** + * Copy the contents of a path to the store and register the + * validity the resulting path, using a constant amount of + * memory. + */ ValidPathInfo addToStoreSlow(std::string_view name, const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, std::optional expectedCAHash = {}); - /* Like addToStore(), but the contents of the path are contained - in `dump', which is either a NAR serialisation (if recursive == - true) or simply the contents of a regular file (if recursive == - false). - `dump` may be drained */ - // FIXME: remove? + /** + * Like addToStore(), but the contents of the path are contained + * in `dump`, which is either a NAR serialisation (if recursive == + * true) or simply the contents of a regular file (if recursive == + * false). + * `dump` may be drained + * + * \todo remove? + */ virtual StorePath addToStoreFromDump(Source & dump, std::string_view name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) { unsupported("addToStoreFromDump"); } - /* Like addToStore, but the contents written to the output path is - a regular file containing the given string. */ + /** + * Like addToStore, but the contents written to the output path is a + * regular file containing the given string. + */ virtual StorePath addTextToStore( std::string_view name, std::string_view s, @@ -436,140 +549,180 @@ public: virtual void registerDrvOutput(const Realisation & output, CheckSigsFlag checkSigs) { return registerDrvOutput(output); } - /* Write a NAR dump of a store path. */ + /** + * Write a NAR dump of a store path. + */ virtual void narFromPath(const StorePath & path, Sink & sink) = 0; - /* For each path, if it's a derivation, build it. Building a - derivation means ensuring that the output paths are valid. If - they are already valid, this is a no-op. Otherwise, validity - can be reached in two ways. First, if the output paths is - substitutable, then build the path that way. Second, the - output paths can be created by running the builder, after - recursively building any sub-derivations. For inputs that are - not derivations, substitute them. */ + /** + * For each path, if it's a derivation, build it. Building a + * derivation means ensuring that the output paths are valid. If + * they are already valid, this is a no-op. Otherwise, validity + * can be reached in two ways. First, if the output paths is + * substitutable, then build the path that way. Second, the + * output paths can be created by running the builder, after + * recursively building any sub-derivations. For inputs that are + * not derivations, substitute them. + */ virtual void buildPaths( const std::vector & paths, BuildMode buildMode = bmNormal, std::shared_ptr evalStore = nullptr); - /* Like `buildPaths()`, but return a vector of `BuildResult`s - corresponding to each element in `paths`. Note that in case of - a build/substitution error, this function won't throw an - exception, but return a `BuildResult` containing an error - message. */ + /** + * Like buildPaths(), but return a vector of \ref BuildResult + * BuildResults corresponding to each element in paths. Note that in + * case of a build/substitution error, this function won't throw an + * exception, but return a BuildResult containing an error message. + */ virtual std::vector buildPathsWithResults( const std::vector & paths, BuildMode buildMode = bmNormal, std::shared_ptr evalStore = nullptr); - /* Build a single non-materialized derivation (i.e. not from an - on-disk .drv file). - - ‘drvPath’ is used to deduplicate worker goals so it is imperative that - is correct. That said, it doesn't literally need to be store path that - would be calculated from writing this derivation to the store: it is OK - if it instead is that of a Derivation which would resolve to this (by - taking the outputs of it's input derivations and adding them as input - sources) such that the build time referenceable-paths are the same. - - In the input-addressed case, we usually *do* use an "original" - unresolved derivations's path, as that is what will be used in the - `buildPaths` case. Also, the input-addressed output paths are verified - only by that contents of that specific unresolved derivation, so it is - nice to keep that information around so if the original derivation is - ever obtained later, it can be verified whether the trusted user in fact - used the proper output path. - - In the content-addressed case, we want to always use the - resolved drv path calculated from the provided derivation. This serves - two purposes: - - - It keeps the operation trustless, by ruling out a maliciously - invalid drv path corresponding to a non-resolution-equivalent - derivation. - - - For the floating case in particular, it ensures that the derivation - to output mapping respects the resolution equivalence relation, so - one cannot choose different resolution-equivalent derivations to - subvert dependency coherence (i.e. the property that one doesn't end - up with multiple different versions of dependencies without - explicitly choosing to allow it). - */ + /** + * Build a single non-materialized derivation (i.e. not from an + * on-disk .drv file). + * + * @param drvPath This is used to deduplicate worker goals so it is + * imperative that is correct. That said, it doesn't literally need + * to be store path that would be calculated from writing this + * derivation to the store: it is OK if it instead is that of a + * Derivation which would resolve to this (by taking the outputs of + * it's input derivations and adding them as input sources) such + * that the build time referenceable-paths are the same. + * + * In the input-addressed case, we usually *do* use an "original" + * unresolved derivations's path, as that is what will be used in the + * buildPaths case. Also, the input-addressed output paths are verified + * only by that contents of that specific unresolved derivation, so it is + * nice to keep that information around so if the original derivation is + * ever obtained later, it can be verified whether the trusted user in fact + * used the proper output path. + * + * In the content-addressed case, we want to always use the resolved + * drv path calculated from the provided derivation. This serves two + * purposes: + * + * - It keeps the operation trustless, by ruling out a maliciously + * invalid drv path corresponding to a non-resolution-equivalent + * derivation. + * + * - For the floating case in particular, it ensures that the derivation + * to output mapping respects the resolution equivalence relation, so + * one cannot choose different resolution-equivalent derivations to + * subvert dependency coherence (i.e. the property that one doesn't end + * up with multiple different versions of dependencies without + * explicitly choosing to allow it). + */ virtual BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal); - /* Ensure that a path is valid. If it is not currently valid, it - may be made valid by running a substitute (if defined for the - path). */ + /** + * Ensure that a path is valid. If it is not currently valid, it + * may be made valid by running a substitute (if defined for the + * path). + */ virtual void ensurePath(const StorePath & path); - /* Add a store path as a temporary root of the garbage collector. - The root disappears as soon as we exit. */ + /** + * Add a store path as a temporary root of the garbage collector. + * The root disappears as soon as we exit. + */ virtual void addTempRoot(const StorePath & path) { debug("not creating temporary root, store doesn't support GC"); } - /* Return a string representing information about the path that - can be loaded into the database using `nix-store --load-db' or - `nix-store --register-validity'. */ + /** + * @return a string representing information about the path that + * can be loaded into the database using `nix-store --load-db` or + * `nix-store --register-validity`. + */ std::string makeValidityRegistration(const StorePathSet & paths, bool showDerivers, bool showHash); - /* Write a JSON representation of store path metadata, such as the - hash and the references. If ‘includeImpureInfo’ is true, - variable elements such as the registration time are - included. If ‘showClosureSize’ is true, the closure size of - each path is included. */ + /** + * Write a JSON representation of store path metadata, such as the + * hash and the references. + * + * @param includeImpureInfo If true, variable elements such as the + * registration time are included. + * + * @param showClosureSize If true, the closure size of each path is + * included. + */ nlohmann::json pathInfoToJSON(const StorePathSet & storePaths, bool includeImpureInfo, bool showClosureSize, Base hashBase = Base32, AllowInvalidFlag allowInvalid = DisallowInvalid); - /* Return the size of the closure of the specified path, that is, - the sum of the size of the NAR serialisation of each path in - the closure. */ + /** + * @return the size of the closure of the specified path, that is, + * the sum of the size of the NAR serialisation of each path in the + * closure. + */ std::pair getClosureSize(const StorePath & storePath); - /* Optimise the disk space usage of the Nix store by hard-linking files - with the same contents. */ + /** + * Optimise the disk space usage of the Nix store by hard-linking files + * with the same contents. + */ virtual void optimiseStore() { }; - /* Check the integrity of the Nix store. Returns true if errors - remain. */ + /** + * Check the integrity of the Nix store. + * + * @return true if errors remain. + */ virtual bool verifyStore(bool checkContents, RepairFlag repair = NoRepair) { return false; }; - /* Return an object to access files in the Nix store. */ + /** + * @return An object to access files in the Nix store. + */ virtual ref getFSAccessor() { unsupported("getFSAccessor"); } - /* Repair the contents of the given path by redownloading it using - a substituter (if available). */ + /** + * Repair the contents of the given path by redownloading it using + * a substituter (if available). + */ virtual void repairPath(const StorePath & path) { unsupported("repairPath"); } - /* Add signatures to the specified store path. The signatures are - not verified. */ + /** + * Add signatures to the specified store path. The signatures are + * not verified. + */ virtual void addSignatures(const StorePath & storePath, const StringSet & sigs) { unsupported("addSignatures"); } /* Utility functions. */ - /* Read a derivation, after ensuring its existence through - ensurePath(). */ + /** + * Read a derivation, after ensuring its existence through + * ensurePath(). + */ Derivation derivationFromPath(const StorePath & drvPath); - /* Read a derivation (which must already be valid). */ + /** + * Read a derivation (which must already be valid). + */ Derivation readDerivation(const StorePath & drvPath); - /* Read a derivation from a potentially invalid path. */ + /** + * Read a derivation from a potentially invalid path. + */ Derivation readInvalidDerivation(const StorePath & drvPath); - /* Place in `out' the set of all store paths in the file system - closure of `storePath'; that is, all paths than can be directly - or indirectly reached from it. `out' is not cleared. If - `flipDirection' is true, the set of paths that can reach - `storePath' is returned; that is, the closures under the - `referrers' relation instead of the `references' relation is - returned. */ + /** + * @param [out] out Place in here the set of all store paths in the + * file system closure of `storePath`; that is, all paths than can + * be directly or indirectly reached from it. `out` is not cleared. + * + * @param flipDirection If true, the set of paths that can reach + * `storePath` is returned; that is, the closures under the + * `referrers` relation instead of the `references` relation is + * returned. + */ virtual void computeFSClosure(const StorePathSet & paths, StorePathSet & out, bool flipDirection = false, bool includeOutputs = false, bool includeDerivers = false); @@ -578,27 +731,34 @@ public: StorePathSet & out, bool flipDirection = false, bool includeOutputs = false, bool includeDerivers = false); - /* Given a set of paths that are to be built, return the set of - derivations that will be built, and the set of output paths - that will be substituted. */ + /** + * Given a set of paths that are to be built, return the set of + * derivations that will be built, and the set of output paths that + * will be substituted. + */ virtual void queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, uint64_t & downloadSize, uint64_t & narSize); - /* Sort a set of paths topologically under the references - relation. If p refers to q, then p precedes q in this list. */ + /** + * Sort a set of paths topologically under the references + * relation. If p refers to q, then p precedes q in this list. + */ StorePaths topoSortPaths(const StorePathSet & paths); - /* Export multiple paths in the format expected by ‘nix-store - --import’. */ + /** + * Export multiple paths in the format expected by ‘nix-store + * --import’. + */ void exportPaths(const StorePathSet & paths, Sink & sink); void exportPath(const StorePath & path, Sink & sink); - /* Import a sequence of NAR dumps created by exportPaths() into - the Nix store. Optionally, the contents of the NARs are - preloaded into the specified FS accessor to speed up subsequent - access. */ + /** + * Import a sequence of NAR dumps created by exportPaths() into the + * Nix store. Optionally, the contents of the NARs are preloaded + * into the specified FS accessor to speed up subsequent access. + */ StorePaths importPaths(Source & source, CheckSigsFlag checkSigs = CheckSigs); struct Stats @@ -620,8 +780,9 @@ public: const Stats & getStats(); - /* Computes the full closure of of a set of store-paths for e.g. - derivations that need this information for `exportReferencesGraph`. + /** + * Computes the full closure of of a set of store-paths for e.g. + * derivations that need this information for `exportReferencesGraph`. */ StorePathSet exportReferences(const StorePathSet & storePaths, const StorePathSet & inputPaths); @@ -632,23 +793,40 @@ public: */ std::optional getBuildDerivationPath(const StorePath &); - /* Hack to allow long-running processes like hydra-queue-runner to - occasionally flush their path info cache. */ + /** + * Hack to allow long-running processes like hydra-queue-runner to + * occasionally flush their path info cache. + */ void clearPathInfoCache() { state.lock()->pathInfoCache.clear(); } - /* Establish a connection to the store, for store types that have - a notion of connection. Otherwise this is a no-op. */ + /** + * Establish a connection to the store, for store types that have + * a notion of connection. Otherwise this is a no-op. + */ virtual void connect() { }; - /* Get the protocol version of this store or it's connection. */ + /** + * Get the protocol version of this store or it's connection. + */ virtual unsigned int getProtocol() { return 0; }; + /** + * @return/ whether store trusts *us*. + * + * `std::nullopt` means we do not know. + * + * @note This is the opposite of the StoreConfig::isTrusted + * store setting. That is about whether *we* trust the store. + */ + virtual std::optional isTrustedClient() = 0; + + virtual Path toRealPath(const Path & storePath) { return storePath; @@ -659,7 +837,7 @@ public: return toRealPath(printStorePath(storePath)); } - /* + /** * Synchronises the options of the client with those of the daemon * (a no-op when there’s no daemon) */ @@ -671,7 +849,13 @@ protected: Stats stats; - /* Unsupported methods. */ + /** + * Helper for methods that are not unsupported: this is used for + * default definitions for virtual methods that are meant to be overriden. + * + * \todo Using this should be a last resort. It is better to make + * the method "virtual pure" and/or move it to a subclass. + */ [[noreturn]] void unsupported(const std::string & op) { throw Unsupported("operation '%s' is not supported by store '%s'", op, getUri()); @@ -680,7 +864,9 @@ protected: }; -/* Copy a path from one store to another. */ +/** + * Copy a path from one store to another. + */ void copyStorePath( Store & srcStore, Store & dstStore, @@ -689,12 +875,14 @@ void copyStorePath( CheckSigsFlag checkSigs = CheckSigs); -/* Copy store paths from one store to another. The paths may be copied - in parallel. They are copied in a topologically sorted order (i.e. - if A is a reference of B, then A is copied before B), but the set - of store paths is not automatically closed; use copyClosure() for - that. Returns a map of what each path was copied to the dstStore - as. */ +/** + * Copy store paths from one store to another. The paths may be copied + * in parallel. They are copied in a topologically sorted order (i.e. if + * A is a reference of B, then A is copied before B), but the set of + * store paths is not automatically closed; use copyClosure() for that. + * + * @return a map of what each path was copied to the dstStore as. + */ std::map copyPaths( Store & srcStore, Store & dstStore, const RealisedPath::Set &, @@ -709,7 +897,9 @@ std::map copyPaths( CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); -/* Copy the closure of `paths` from `srcStore` to `dstStore`. */ +/** + * Copy the closure of `paths` from `srcStore` to `dstStore`. + */ void copyClosure( Store & srcStore, Store & dstStore, const RealisedPath::Set & paths, @@ -724,52 +914,61 @@ void copyClosure( CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); -/* Remove the temporary roots file for this process. Any temporary - root becomes garbage after this point unless it has been registered - as a (permanent) root. */ +/** + * Remove the temporary roots file for this process. Any temporary + * root becomes garbage after this point unless it has been registered + * as a (permanent) root. + */ void removeTempRoots(); -/* Resolve the derived path completely, failing if any derivation output - is unknown. */ +/** + * Resolve the derived path completely, failing if any derivation output + * is unknown. + */ OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * evalStore = nullptr); -/* Return a Store object to access the Nix store denoted by - ‘uri’ (slight misnomer...). Supported values are: - - * ‘local’: The Nix store in /nix/store and database in - /nix/var/nix/db, accessed directly. - - * ‘daemon’: The Nix store accessed via a Unix domain socket - connection to nix-daemon. - - * ‘unix://’: The Nix store accessed via a Unix domain socket - connection to nix-daemon, with the socket located at . - - * ‘auto’ or ‘’: Equivalent to ‘local’ or ‘daemon’ depending on - whether the user has write access to the local Nix - store/database. - - * ‘file://’: A binary cache stored in . - - * ‘https://’: A binary cache accessed via HTTP. - - * ‘s3://’: A writable binary cache stored on Amazon's Simple - Storage Service. - - * ‘ssh://[user@]’: A remote Nix store accessed by running - ‘nix-store --serve’ via SSH. - - You can pass parameters to the store implementation by appending - ‘?key=value&key=value&...’ to the URI. -*/ +/** + * @return a Store object to access the Nix store denoted by + * ‘uri’ (slight misnomer...). + * + * @param uri Supported values are: + * + * - ‘local’: The Nix store in /nix/store and database in + * /nix/var/nix/db, accessed directly. + * + * - ‘daemon’: The Nix store accessed via a Unix domain socket + * connection to nix-daemon. + * + * - ‘unix://’: The Nix store accessed via a Unix domain socket + * connection to nix-daemon, with the socket located at . + * + * - ‘auto’ or ‘’: Equivalent to ‘local’ or ‘daemon’ depending on + * whether the user has write access to the local Nix + * store/database. + * + * - ‘file://’: A binary cache stored in . + * + * - ‘https://’: A binary cache accessed via HTTP. + * + * - ‘s3://’: A writable binary cache stored on Amazon's Simple + * Storage Service. + * + * - ‘ssh://[user@]’: A remote Nix store accessed by running + * ‘nix-store --serve’ via SSH. + * + * You can pass parameters to the store implementation by appending + * ‘?key=value&key=value&...’ to the URI. + */ ref openStore(const std::string & uri = settings.storeUri.get(), const Store::Params & extraParams = Store::Params()); -/* Return the default substituter stores, defined by the - ‘substituters’ option and various legacy options. */ +/** + * @return the default substituter stores, defined by the + * ‘substituters’ option and various legacy options. + */ std::list> getDefaultSubstituters(); struct StoreFactory @@ -812,8 +1011,10 @@ struct RegisterStoreImplementation }; -/* Display a set of paths in human-readable form (i.e., between quotes - and separated by commas). */ +/** + * Display a set of paths in human-readable form (i.e., between quotes + * and separated by commas). + */ std::string showPaths(const PathSet & paths); @@ -822,7 +1023,9 @@ std::optional decodeValidPathInfo( std::istream & str, std::optional hashGiven = std::nullopt); -/* Split URI into protocol+hierarchy part and its parameter set. */ +/** + * Split URI into protocol+hierarchy part and its parameter set. + */ std::pair splitUriAndParams(const std::string & uri); std::optional getDerivationCA(const BasicDerivation & drv); diff --git a/src/libstore/store-cast.hh b/src/libstore/store-cast.hh index ff62fc359..2473e72c5 100644 --- a/src/libstore/store-cast.hh +++ b/src/libstore/store-cast.hh @@ -1,9 +1,17 @@ #pragma once +///@file #include "store-api.hh" namespace nix { +/** + * Helper to try downcasting a Store with a nice method if it fails. + * + * This is basically an alternative to the user-facing part of + * Store::unsupported that allows us to still have a nice message but + * better interface design. + */ template T & require(Store & store) { diff --git a/src/libstore/tests/derivation.cc b/src/libstore/tests/derivation.cc index 12be8504d..80ee52fd0 100644 --- a/src/libstore/tests/derivation.cc +++ b/src/libstore/tests/derivation.cc @@ -11,15 +11,29 @@ class DerivationTest : public LibStoreTest { }; -#define TEST_JSON(TYPE, NAME, STR, VAL, ...) \ - TEST_F(DerivationTest, TYPE ## _ ## NAME ## _to_json) { \ - using nlohmann::literals::operator "" _json; \ - ASSERT_EQ( \ - STR ## _json, \ - (TYPE { VAL }).toJSON(*store __VA_OPT__(,) __VA_ARGS__)); \ +#define TEST_JSON(NAME, STR, VAL, DRV_NAME, OUTPUT_NAME) \ + TEST_F(DerivationTest, DerivationOutput_ ## NAME ## _to_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + STR ## _json, \ + (DerivationOutput { VAL }).toJSON( \ + *store, \ + DRV_NAME, \ + OUTPUT_NAME)); \ + } \ + \ + TEST_F(DerivationTest, DerivationOutput_ ## NAME ## _from_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + DerivationOutput { VAL }, \ + DerivationOutput::fromJSON( \ + *store, \ + DRV_NAME, \ + OUTPUT_NAME, \ + STR ## _json)); \ } -TEST_JSON(DerivationOutput, inputAddressed, +TEST_JSON(inputAddressed, R"({ "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" })", @@ -28,7 +42,7 @@ TEST_JSON(DerivationOutput, inputAddressed, }), "drv-name", "output-name") -TEST_JSON(DerivationOutput, caFixed, +TEST_JSON(caFixed, R"({ "hashAlgo": "r:sha256", "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", @@ -42,7 +56,7 @@ TEST_JSON(DerivationOutput, caFixed, }), "drv-name", "output-name") -TEST_JSON(DerivationOutput, caFloating, +TEST_JSON(caFloating, R"({ "hashAlgo": "r:sha256" })", @@ -52,12 +66,12 @@ TEST_JSON(DerivationOutput, caFloating, }), "drv-name", "output-name") -TEST_JSON(DerivationOutput, deferred, +TEST_JSON(deferred, R"({ })", DerivationOutput::Deferred { }, "drv-name", "output-name") -TEST_JSON(DerivationOutput, impure, +TEST_JSON(impure, R"({ "hashAlgo": "r:sha256", "impure": true @@ -68,8 +82,28 @@ TEST_JSON(DerivationOutput, impure, }), "drv-name", "output-name") -TEST_JSON(Derivation, impure, +#undef TEST_JSON + +#define TEST_JSON(NAME, STR, VAL, DRV_NAME) \ + TEST_F(DerivationTest, Derivation_ ## NAME ## _to_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + STR ## _json, \ + (Derivation { VAL }).toJSON(*store)); \ + } \ + \ + TEST_F(DerivationTest, Derivation_ ## NAME ## _from_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + Derivation { VAL }, \ + Derivation::fromJSON( \ + *store, \ + STR ## _json)); \ + } + +TEST_JSON(simple, R"({ + "name": "my-derivation", "inputSrcs": [ "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" ], @@ -92,6 +126,7 @@ TEST_JSON(Derivation, impure, })", ({ Derivation drv; + drv.name = "my-derivation"; drv.inputSrcs = { store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), }; @@ -117,7 +152,8 @@ TEST_JSON(Derivation, impure, }, }; drv; - })) + }), + "drv-name") #undef TEST_JSON diff --git a/src/libstore/tests/derived-path.hh b/src/libstore/tests/derived-path.hh index 3bc812440..506f3ccb1 100644 --- a/src/libstore/tests/derived-path.hh +++ b/src/libstore/tests/derived-path.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include diff --git a/src/libstore/tests/libstore.hh b/src/libstore/tests/libstore.hh index 05397659b..ef93457b5 100644 --- a/src/libstore/tests/libstore.hh +++ b/src/libstore/tests/libstore.hh @@ -1,3 +1,6 @@ +#pragma once +///@file + #include #include diff --git a/src/libstore/tests/outputs-spec.hh b/src/libstore/tests/outputs-spec.hh index 2d455c817..ded331b33 100644 --- a/src/libstore/tests/outputs-spec.hh +++ b/src/libstore/tests/outputs-spec.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include diff --git a/src/libstore/tests/path.hh b/src/libstore/tests/path.hh index d7f1a8988..21cb62310 100644 --- a/src/libstore/tests/path.hh +++ b/src/libstore/tests/path.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 5c38323cd..0fb7c38e9 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -26,9 +26,9 @@ UDSRemoteStore::UDSRemoteStore(const Params & params) UDSRemoteStore::UDSRemoteStore( - const std::string scheme, - std::string socket_path, - const Params & params) + const std::string scheme, + std::string socket_path, + const Params & params) : UDSRemoteStore(params) { path.emplace(socket_path); diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index d31a4d592..bd1dcb67c 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "remote-store.hh" #include "local-fs-store.hh" @@ -15,6 +16,13 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon } const std::string name() override { return "Local Daemon Store"; } + + std::string doc() override + { + return + #include "uds-remote-store.md" + ; + } }; class UDSRemoteStore : public virtual UDSRemoteStoreConfig, public virtual LocalFSStore, public virtual RemoteStore diff --git a/src/libstore/uds-remote-store.md b/src/libstore/uds-remote-store.md new file mode 100644 index 000000000..8df0bd6ff --- /dev/null +++ b/src/libstore/uds-remote-store.md @@ -0,0 +1,9 @@ +R"( + +**Store URL format**: `daemon`, `unix://`*path* + +This store type accesses a Nix store by talking to a Nix daemon +listening on the Unix domain socket *path*. The store pseudo-URL +`daemon` is equivalent to `unix:///nix/var/nix/daemon-socket/socket`. + +)" diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 87088a3ac..c7a6f8688 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" #include "serialise.hh" @@ -9,11 +10,15 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION (1 << 8 | 34) +#define PROTOCOL_VERSION (1 << 8 | 35) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) +/** + * Enumeration of all the request types for the "worker protocol", used + * by unix:// and ssh-ng:// stores. + */ typedef enum { wopIsValidPath = 1, wopHasSubstitutes = 3, @@ -74,7 +79,12 @@ typedef enum { class Store; struct Source; -/* To guide overloading */ +/** + * Used to guide overloading + * + * See https://en.cppreference.com/w/cpp/language/adl for the broader + * concept of what is going on here. + */ template struct Phantom {}; @@ -93,6 +103,7 @@ MAKE_WORKER_PROTO(, DerivedPath); MAKE_WORKER_PROTO(, Realisation); MAKE_WORKER_PROTO(, DrvOutput); MAKE_WORKER_PROTO(, BuildResult); +MAKE_WORKER_PROTO(, std::optional); MAKE_WORKER_PROTO(template, std::vector); MAKE_WORKER_PROTO(template, std::set); @@ -103,18 +114,19 @@ MAKE_WORKER_PROTO(X_, Y_); #undef X_ #undef Y_ -/* These use the empty string for the null case, relying on the fact - that the underlying types never serialize to the empty string. - - We do this instead of a generic std::optional instance because - ordinal tags (0 or 1, here) are a bit of a compatability hazard. For - the same reason, we don't have a std::variant instances (ordinal - tags 0...n). - - We could the generic instances and then these as specializations for - compatability, but that's proven a bit finnicky, and also makes the - worker protocol harder to implement in other languages where such - specializations may not be allowed. +/** + * These use the empty string for the null case, relying on the fact + * that the underlying types never serialize to the empty string. + * + * We do this instead of a generic std::optional instance because + * ordinal tags (0 or 1, here) are a bit of a compatability hazard. For + * the same reason, we don't have a std::variant instances (ordinal + * tags 0...n). + * + * We could the generic instances and then these as specializations for + * compatability, but that's proven a bit finnicky, and also makes the + * worker protocol harder to implement in other languages where such + * specializations may not be allowed. */ MAKE_WORKER_PROTO(, std::optional); MAKE_WORKER_PROTO(, std::optional); diff --git a/src/libutil/abstract-setting-to-json.hh b/src/libutil/abstract-setting-to-json.hh index 2d82b54e7..7b6c3fcb5 100644 --- a/src/libutil/abstract-setting-to-json.hh +++ b/src/libutil/abstract-setting-to-json.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include "config.hh" diff --git a/src/libutil/ansicolor.hh b/src/libutil/ansicolor.hh index 38305e71c..86becafa6 100644 --- a/src/libutil/ansicolor.hh +++ b/src/libutil/ansicolor.hh @@ -1,8 +1,12 @@ #pragma once +/** + * @file + * + * @brief Some ANSI escape sequences. + */ namespace nix { -/* Some ANSI escape sequences. */ #define ANSI_NORMAL "\e[0m" #define ANSI_BOLD "\e[1m" #define ANSI_FAINT "\e[2m" diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index e42dea540..2cf164a41 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "serialise.hh" @@ -7,54 +8,73 @@ namespace nix { -/* dumpPath creates a Nix archive of the specified path. The format - is as follows: - - IF path points to a REGULAR FILE: - dump(path) = attrs( - [ ("type", "regular") - , ("contents", contents(path)) - ]) - - IF path points to a DIRECTORY: - dump(path) = attrs( - [ ("type", "directory") - , ("entries", concat(map(f, sort(entries(path))))) - ]) - where f(fn) = attrs( - [ ("name", fn) - , ("file", dump(path + "/" + fn)) - ]) - - where: - - attrs(as) = concat(map(attr, as)) + encN(0) - attrs((a, b)) = encS(a) + encS(b) - - encS(s) = encN(len(s)) + s + (padding until next 64-bit boundary) - - encN(n) = 64-bit little-endian encoding of n. - - contents(path) = the contents of a regular file. - - sort(strings) = lexicographic sort by 8-bit value (strcmp). - - entries(path) = the entries of a directory, without `.' and - `..'. - - `+' denotes string concatenation. */ - - +/** + * dumpPath creates a Nix archive of the specified path. + * + * @param path the file system data to dump. Dumping is recursive so if + * this is a directory we dump it and all its children. + * + * @param [out] sink The serialised archive is fed into this sink. + * + * @param filter Can be used to skip certain files. + * + * The format is as follows: + * + * ``` + * IF path points to a REGULAR FILE: + * dump(path) = attrs( + * [ ("type", "regular") + * , ("contents", contents(path)) + * ]) + * + * IF path points to a DIRECTORY: + * dump(path) = attrs( + * [ ("type", "directory") + * , ("entries", concat(map(f, sort(entries(path))))) + * ]) + * where f(fn) = attrs( + * [ ("name", fn) + * , ("file", dump(path + "/" + fn)) + * ]) + * + * where: + * + * attrs(as) = concat(map(attr, as)) + encN(0) + * attrs((a, b)) = encS(a) + encS(b) + * + * encS(s) = encN(len(s)) + s + (padding until next 64-bit boundary) + * + * encN(n) = 64-bit little-endian encoding of n. + * + * contents(path) = the contents of a regular file. + * + * sort(strings) = lexicographic sort by 8-bit value (strcmp). + * + * entries(path) = the entries of a directory, without `.` and + * `..`. + * + * `+` denotes string concatenation. + * ``` + */ void dumpPath(const Path & path, Sink & sink, PathFilter & filter = defaultPathFilter); -/* Same as `void dumpPath()`, but returns the last modified date of the path */ +/** + * Same as dumpPath(), but returns the last modified date of the path. + */ time_t dumpPathAndGetMtime(const Path & path, Sink & sink, PathFilter & filter = defaultPathFilter); +/** + * Dump an archive with a single file with these contents. + * + * @param s Contents of the file. + */ void dumpString(std::string_view s, Sink & sink); -/* FIXME: fix this API, it sucks. */ +/** + * \todo Fix this API, it sucks. + */ struct ParseSink { virtual void createDirectory(const Path & path) { }; @@ -68,8 +88,10 @@ struct ParseSink virtual void createSymlink(const Path & path, const std::string & target) { }; }; -/* If the NAR archive contains a single file at top-level, then save - the contents of the file to `s'. Otherwise barf. */ +/** + * If the NAR archive contains a single file at top-level, then save + * the contents of the file to `s`. Otherwise barf. + */ struct RetrieveRegularNARSink : ParseSink { bool regular = true; @@ -97,7 +119,9 @@ void parseDump(ParseSink & sink, Source & source); void restorePath(const Path & path, Source & source); -/* Read a NAR from 'source' and write it to 'sink'. */ +/** + * Read a NAR from 'source' and write it to 'sink'. + */ void copyNAR(Source & source, Sink & sink); void copyPath(const Path & from, const Path & to); diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 35686a8aa..081dbeb28 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -52,7 +52,7 @@ std::shared_ptr completions; std::string completionMarker = "___COMPLETE___"; -std::optional needsCompletion(std::string_view s) +static std::optional needsCompletion(std::string_view s) { if (!completions) return {}; auto i = s.find(completionMarker); @@ -120,6 +120,12 @@ void Args::parseCmdline(const Strings & _cmdline) if (!argsSeen) initialFlagsProcessed(); + + /* Now that we are done parsing, make sure that any experimental + * feature required by the flags is enabled */ + for (auto & f : flagExperimentalFeatures) + experimentalFeatureSettings.require(f); + } bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) @@ -128,12 +134,18 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) auto process = [&](const std::string & name, const Flag & flag) -> bool { ++pos; + + if (auto & f = flag.experimentalFeature) + flagExperimentalFeatures.insert(*f); + std::vector args; bool anyCompleted = false; for (size_t n = 0 ; n < flag.handler.arity; ++n) { if (pos == end) { if (flag.handler.arity == ArityAny || anyCompleted) break; - throw UsageError("flag '%s' requires %d argument(s)", name, flag.handler.arity); + throw UsageError( + "flag '%s' requires %d argument(s), but only %d were given", + name, flag.handler.arity, n); } if (auto prefix = needsCompletion(*pos)) { anyCompleted = true; @@ -152,7 +164,11 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) for (auto & [name, flag] : longFlags) { if (!hiddenCategories.count(flag->category) && hasPrefix(name, std::string(*prefix, 2))) + { + if (auto & f = flag->experimentalFeature) + flagExperimentalFeatures.insert(*f); completions->add("--" + name, flag->description); + } } return false; } @@ -172,7 +188,8 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) if (prefix == "-") { completions->add("--"); for (auto & [flagName, flag] : shortFlags) - completions->add(std::string("-") + flagName, flag->description); + if (experimentalFeatureSettings.isEnabled(flag->experimentalFeature)) + completions->add(std::string("-") + flagName, flag->description); } } @@ -230,6 +247,11 @@ nlohmann::json Args::toJSON() j["arity"] = flag->handler.arity; if (!flag->labels.empty()) j["labels"] = flag->labels; + // TODO With C++23 use `std::optional::tranform` + if (auto & xp = flag->experimentalFeature) + j["experimental-feature"] = showExperimentalFeature(*xp); + else + j["experimental-feature"] = nullptr; flags[name] = std::move(j); } @@ -326,6 +348,11 @@ Strings argvToStrings(int argc, char * * argv) return args; } +std::optional Command::experimentalFeature () +{ + return { Xp::NixCommand }; +} + MultiCommand::MultiCommand(const Commands & commands_) : commands(commands_) { @@ -389,6 +416,11 @@ nlohmann::json MultiCommand::toJSON() cat["id"] = command->category(); cat["description"] = trim(categories[command->category()]); j["category"] = std::move(cat); + // TODO With C++23 use `std::optional::tranform` + if (auto xp = command->experimentalFeature()) + cat["experimental-feature"] = showExperimentalFeature(*xp); + else + cat["experimental-feature"] = nullptr; cmds[name] = std::move(j); } diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 7211ee307..d90129796 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -18,16 +19,22 @@ class Args { public: - /* Parse the command line, throwing a UsageError if something goes - wrong. */ + /** + * Parse the command line, throwing a UsageError if something goes + * wrong. + */ void parseCmdline(const Strings & cmdline); - /* Return a short one-line description of the command. */ + /** + * Return a short one-line description of the command. + */ virtual std::string description() { return ""; } virtual bool forceImpureByDefault() { return false; } - /* Return documentation about this command, in Markdown format. */ + /** + * Return documentation about this command, in Markdown format. + */ virtual std::string doc() { return ""; } protected: @@ -117,6 +124,8 @@ protected: Handler handler; std::function completer; + std::optional experimentalFeature; + static Flag mkHashTypeFlag(std::string && longName, HashType * ht); static Flag mkHashTypeOptFlag(std::string && longName, std::optional * oht); }; @@ -144,13 +153,17 @@ protected: std::set hiddenCategories; - /* Called after all command line flags before the first non-flag - argument (if any) have been processed. */ + /** + * Called after all command line flags before the first non-flag + * argument (if any) have been processed. + */ virtual void initialFlagsProcessed() {} - /* Called after the command line has been processed if we need to generate - completions. Useful for commands that need to know the whole command line - in order to know what completions to generate. */ + /** + * Called after the command line has been processed if we need to generate + * completions. Useful for commands that need to know the whole command line + * in order to know what completions to generate. + */ virtual void completionHook() { } public: @@ -164,7 +177,9 @@ public: expectedArgs.emplace_back(std::move(arg)); } - /* Expect a string argument. */ + /** + * Expect a string argument. + */ void expectArg(const std::string & label, std::string * dest, bool optional = false) { expectArgs({ @@ -174,7 +189,9 @@ public: }); } - /* Expect 0 or more arguments. */ + /** + * Expect 0 or more arguments. + */ void expectArgs(const std::string & label, std::vector * dest) { expectArgs({ @@ -188,29 +205,48 @@ public: friend class MultiCommand; MultiCommand * parent = nullptr; + +private: + + /** + * Experimental features needed when parsing args. These are checked + * after flag parsing is completed in order to support enabling + * experimental features coming after the flag that needs the + * experimental feature. + */ + std::set flagExperimentalFeatures; }; -/* A command is an argument parser that can be executed by calling its - run() method. */ +/** + * A command is an argument parser that can be executed by calling its + * run() method. + */ struct Command : virtual public Args { friend class MultiCommand; virtual ~Command() { } + /** + * Entry point to the command + */ virtual void run() = 0; typedef int Category; static constexpr Category catDefault = 0; + virtual std::optional experimentalFeature (); + virtual Category category() { return catDefault; } }; typedef std::map()>> Commands; -/* An argument parser that supports multiple subcommands, - i.e. ‘ ’. */ +/** + * An argument parser that supports multiple subcommands, + * i.e. ‘ ’. + */ class MultiCommand : virtual public Args { public: @@ -218,7 +254,9 @@ public: std::map categories; - // Selected command, if any. + /** + * Selected command, if any. + */ std::optional>> command; MultiCommand(const Commands & commands); @@ -253,8 +291,6 @@ enum CompletionType { }; extern CompletionType completionType; -std::optional needsCompletion(std::string_view s); - void completePath(size_t, std::string_view prefix); void completeDir(size_t, std::string_view prefix); diff --git a/src/libutil/callback.hh b/src/libutil/callback.hh index ef31794be..3710d1239 100644 --- a/src/libutil/callback.hh +++ b/src/libutil/callback.hh @@ -1,13 +1,16 @@ #pragma once +///@file #include #include namespace nix { -/* A callback is a wrapper around a lambda that accepts a valid of - type T or an exception. (We abuse std::future to pass the value or - exception.) */ +/** + * A callback is a wrapper around a lambda that accepts a valid of + * type T or an exception. (We abuse std::future to pass the value or + * exception.) + */ template class Callback { diff --git a/src/libutil/canon-path.cc b/src/libutil/canon-path.cc index b132b4262..ddf6db6d1 100644 --- a/src/libutil/canon-path.cc +++ b/src/libutil/canon-path.cc @@ -100,4 +100,30 @@ std::ostream & operator << (std::ostream & stream, const CanonPath & path) return stream; } +std::string CanonPath::makeRelative(const CanonPath & path) const +{ + auto p1 = begin(); + auto p2 = path.begin(); + + for (; p1 != end() && p2 != path.end() && *p1 == *p2; ++p1, ++p2) ; + + if (p1 == end() && p2 == path.end()) + return "."; + else if (p1 == end()) + return std::string(p2.remaining); + else { + std::string res; + while (p1 != end()) { + ++p1; + if (!res.empty()) res += '/'; + res += ".."; + } + if (p2 != path.end()) { + if (!res.empty()) res += '/'; + res += p2.remaining; + } + return res; + } +} + } diff --git a/src/libutil/canon-path.hh b/src/libutil/canon-path.hh index 9d5984584..614883c06 100644 --- a/src/libutil/canon-path.hh +++ b/src/libutil/canon-path.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -8,28 +9,31 @@ namespace nix { -/* A canonical representation of a path. It ensures the following: - - - It always starts with a slash. - - - It never ends with a slash, except if the path is "/". - - - A slash is never followed by a slash (i.e. no empty components). - - - There are no components equal to '.' or '..'. - - Note that the path does not need to correspond to an actually - existing path, and there is no guarantee that symlinks are - resolved. -*/ +/** + * A canonical representation of a path. It ensures the following: + * + * - It always starts with a slash. + * + * - It never ends with a slash, except if the path is "/". + * + * - A slash is never followed by a slash (i.e. no empty components). + * + * - There are no components equal to '.' or '..'. + * + * Note that the path does not need to correspond to an actually + * existing path, and there is no guarantee that symlinks are + * resolved. + */ class CanonPath { std::string path; public: - /* Construct a canon path from a non-canonical path. Any '.', '..' - or empty components are removed. */ + /** + * Construct a canon path from a non-canonical path. Any '.', '..' + * or empty components are removed. + */ CanonPath(std::string_view raw); explicit CanonPath(const char * raw) @@ -44,9 +48,11 @@ public: static CanonPath root; - /* If `raw` starts with a slash, return - `CanonPath(raw)`. Otherwise return a `CanonPath` representing - `root + "/" + raw`. */ + /** + * If `raw` starts with a slash, return + * `CanonPath(raw)`. Otherwise return a `CanonPath` representing + * `root + "/" + raw`. + */ CanonPath(std::string_view raw, const CanonPath & root); bool isRoot() const @@ -58,8 +64,10 @@ public: const std::string & abs() const { return path; } - /* Like abs(), but return an empty string if this path is - '/'. Thus the returned string never ends in a slash. */ + /** + * Like abs(), but return an empty string if this path is + * '/'. Thus the returned string never ends in a slash. + */ const std::string & absOrEmpty() const { const static std::string epsilon; @@ -85,6 +93,9 @@ public: bool operator != (const Iterator & x) const { return remaining.data() != x.remaining.data(); } + bool operator == (const Iterator & x) const + { return !(*this != x); } + const std::string_view operator * () const { return remaining.substr(0, slash); } @@ -104,7 +115,9 @@ public: std::optional parent() const; - /* Remove the last component. Panics if this path is the root. */ + /** + * Remove the last component. Panics if this path is the root. + */ void pop(); std::optional dirOf() const @@ -125,10 +138,12 @@ public: bool operator != (const CanonPath & x) const { return path != x.path; } - /* Compare paths lexicographically except that path separators - are sorted before any other character. That is, in the sorted order - a directory is always followed directly by its children. For - instance, 'foo' < 'foo/bar' < 'foo!'. */ + /** + * Compare paths lexicographically except that path separators + * are sorted before any other character. That is, in the sorted order + * a directory is always followed directly by its children. For + * instance, 'foo' < 'foo/bar' < 'foo!'. + */ bool operator < (const CanonPath & x) const { auto i = path.begin(); @@ -144,28 +159,44 @@ public: return i == path.end() && j != x.path.end(); } - /* Return true if `this` is equal to `parent` or a child of - `parent`. */ + /** + * Return true if `this` is equal to `parent` or a child of + * `parent`. + */ bool isWithin(const CanonPath & parent) const; CanonPath removePrefix(const CanonPath & prefix) const; - /* Append another path to this one. */ + /** + * Append another path to this one. + */ void extend(const CanonPath & x); - /* Concatenate two paths. */ + /** + * Concatenate two paths. + */ CanonPath operator + (const CanonPath & x) const; - /* Add a path component to this one. It must not contain any slashes. */ + /** + * Add a path component to this one. It must not contain any slashes. + */ void push(std::string_view c); CanonPath operator + (std::string_view c) const; - /* Check whether access to this path is allowed, which is the case - if 1) `this` is within any of the `allowed` paths; or 2) any of - the `allowed` paths are within `this`. (The latter condition - ensures access to the parents of allowed paths.) */ + /** + * Check whether access to this path is allowed, which is the case + * if 1) `this` is within any of the `allowed` paths; or 2) any of + * the `allowed` paths are within `this`. (The latter condition + * ensures access to the parents of allowed paths.) + */ bool isAllowed(const std::set & allowed) const; + + /** + * Return a representation `x` of `path` relative to `this`, i.e. + * `CanonPath(this.makeRelative(x), this) == path`. + */ + std::string makeRelative(const CanonPath & path) const; }; std::ostream & operator << (std::ostream & stream, const CanonPath & path); diff --git a/src/libutil/cgroup.hh b/src/libutil/cgroup.hh index d08c8ad29..574ae8e5b 100644 --- a/src/libutil/cgroup.hh +++ b/src/libutil/cgroup.hh @@ -1,4 +1,5 @@ #pragma once +///@file #if __linux__ @@ -18,10 +19,12 @@ struct CgroupStats std::optional cpuUser, cpuSystem; }; -/* Destroy the cgroup denoted by 'path'. The postcondition is that - 'path' does not exist, and thus any processes in the cgroup have - been killed. Also return statistics from the cgroup just before - destruction. */ +/** + * Destroy the cgroup denoted by 'path'. The postcondition is that + * 'path' does not exist, and thus any processes in the cgroup have + * been killed. Also return statistics from the cgroup just before + * destruction. + */ CgroupStats destroyCgroup(const Path & cgroup); } diff --git a/src/libutil/chunked-vector.hh b/src/libutil/chunked-vector.hh index 0a4f0b400..d914e2542 100644 --- a/src/libutil/chunked-vector.hh +++ b/src/libutil/chunked-vector.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -7,20 +8,24 @@ namespace nix { -/* Provides an indexable container like vector<> with memory overhead - guarantees like list<> by allocating storage in chunks of ChunkSize - elements instead of using a contiguous memory allocation like vector<> - does. Not using a single vector that is resized reduces memory overhead - on large data sets by on average (growth factor)/2, mostly - eliminates copies within the vector during resizing, and provides stable - references to its elements. */ +/** + * Provides an indexable container like vector<> with memory overhead + * guarantees like list<> by allocating storage in chunks of ChunkSize + * elements instead of using a contiguous memory allocation like vector<> + * does. Not using a single vector that is resized reduces memory overhead + * on large data sets by on average (growth factor)/2, mostly + * eliminates copies within the vector during resizing, and provides stable + * references to its elements. + */ template class ChunkedVector { private: uint32_t size_ = 0; std::vector> chunks; - /* keep this out of the ::add hot path */ + /** + * Keep this out of the ::add hot path + */ [[gnu::noinline]] auto & addChunk() { diff --git a/src/libutil/closure.hh b/src/libutil/closure.hh index 779b9b2d5..16e3b93e4 100644 --- a/src/libutil/closure.hh +++ b/src/libutil/closure.hh @@ -1,3 +1,6 @@ +#pragma once +///@file + #include #include #include "sync.hh" diff --git a/src/libutil/comparator.hh b/src/libutil/comparator.hh index eecd5b819..9f661c5c3 100644 --- a/src/libutil/comparator.hh +++ b/src/libutil/comparator.hh @@ -1,6 +1,8 @@ #pragma once +///@file -/* Awfull hacky generation of the comparison operators by doing a lexicographic +/** + * Awful hacky generation of the comparison operators by doing a lexicographic * comparison between the choosen fields. * * ``` @@ -15,12 +17,12 @@ * } * ``` */ -#define GENERATE_ONE_CMP(COMPARATOR, MY_TYPE, FIELDS...) \ +#define GENERATE_ONE_CMP(COMPARATOR, MY_TYPE, ...) \ bool operator COMPARATOR(const MY_TYPE& other) const { \ - const MY_TYPE* me = this; \ - auto fields1 = std::make_tuple( FIELDS ); \ - me = &other; \ - auto fields2 = std::make_tuple( FIELDS ); \ + __VA_OPT__(const MY_TYPE* me = this;) \ + auto fields1 = std::make_tuple( __VA_ARGS__ ); \ + __VA_OPT__(me = &other;) \ + auto fields2 = std::make_tuple( __VA_ARGS__ ); \ return fields1 COMPARATOR fields2; \ } #define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args) diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh index c470b82a5..3892831c2 100644 --- a/src/libutil/compression.hh +++ b/src/libutil/compression.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "ref.hh" #include "types.hh" diff --git a/src/libutil/compute-levels.hh b/src/libutil/compute-levels.hh index 8ded295f9..093e7a915 100644 --- a/src/libutil/compute-levels.hh +++ b/src/libutil/compute-levels.hh @@ -1,3 +1,6 @@ +#pragma once +///@file + #include "types.hh" namespace nix { diff --git a/src/libutil/config.cc b/src/libutil/config.cc index b349f2d80..8d63536d6 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -70,10 +70,17 @@ void AbstractConfig::reapplyUnknownSettings() set(s.first, s.second); } +// Whether we should process the option. Excludes aliases, which are handled elsewhere, and disabled features. +static bool applicable(const Config::SettingData & sd) +{ + return !sd.isAlias + && experimentalFeatureSettings.isEnabled(sd.setting->experimentalFeature); +} + void Config::getSettings(std::map & res, bool overriddenOnly) { for (auto & opt : _settings) - if (!opt.second.isAlias && (!overriddenOnly || opt.second.setting->overridden)) + if (applicable(opt.second) && (!overriddenOnly || opt.second.setting->overridden)) res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description}); } @@ -147,9 +154,8 @@ nlohmann::json Config::toJSON() { auto res = nlohmann::json::object(); for (auto & s : _settings) - if (!s.second.isAlias) { + if (applicable(s.second)) res.emplace(s.first, s.second.setting->toJSON()); - } return res; } @@ -157,24 +163,31 @@ std::string Config::toKeyValue() { auto res = std::string(); for (auto & s : _settings) - if (!s.second.isAlias) { + if (applicable(s.second)) res += fmt("%s = %s\n", s.first, s.second.setting->to_string()); - } return res; } void Config::convertToArgs(Args & args, const std::string & category) { - for (auto & s : _settings) + for (auto & s : _settings) { + /* We do include args for settings gated on disabled + experimental-features. The args themselves however will also be + gated on any experimental feature the underlying setting is. */ if (!s.second.isAlias) s.second.setting->convertToArg(args, category); + } } AbstractSetting::AbstractSetting( const std::string & name, const std::string & description, - const std::set & aliases) - : name(name), description(stripIndentation(description)), aliases(aliases) + const std::set & aliases, + std::optional experimentalFeature) + : name(name) + , description(stripIndentation(description)) + , aliases(aliases) + , experimentalFeature(experimentalFeature) { } @@ -210,6 +223,7 @@ void BaseSetting::convertToArg(Args & args, const std::string & category) .category = category, .labels = {"value"}, .handler = {[this](std::string s) { overridden = true; set(s); }}, + .experimentalFeature = experimentalFeature, }); if (isAppendable()) @@ -219,6 +233,7 @@ void BaseSetting::convertToArg(Args & args, const std::string & category) .category = category, .labels = {"value"}, .handler = {[this](std::string s) { overridden = true; set(s, true); }}, + .experimentalFeature = experimentalFeature, }); } @@ -270,13 +285,15 @@ template<> void BaseSetting::convertToArg(Args & args, const std::string & .longName = name, .description = fmt("Enable the `%s` setting.", name), .category = category, - .handler = {[this]() { override(true); }} + .handler = {[this]() { override(true); }}, + .experimentalFeature = experimentalFeature, }); args.addFlag({ .longName = "no-" + name, .description = fmt("Disable the `%s` setting.", name), .category = category, - .handler = {[this]() { override(false); }} + .handler = {[this]() { override(false); }}, + .experimentalFeature = experimentalFeature, }); } @@ -444,4 +461,30 @@ GlobalConfig::Register::Register(Config * config) configRegistrations->emplace_back(config); } +ExperimentalFeatureSettings experimentalFeatureSettings; + +static GlobalConfig::Register rSettings(&experimentalFeatureSettings); + +bool ExperimentalFeatureSettings::isEnabled(const ExperimentalFeature & feature) const +{ + auto & f = experimentalFeatures.get(); + return std::find(f.begin(), f.end(), feature) != f.end(); +} + +void ExperimentalFeatureSettings::require(const ExperimentalFeature & feature) const +{ + if (!isEnabled(feature)) + throw MissingExperimentalFeature(feature); +} + +bool ExperimentalFeatureSettings::isEnabled(const std::optional & feature) const +{ + return !feature || isEnabled(*feature); +} + +void ExperimentalFeatureSettings::require(const std::optional & feature) const +{ + if (feature) require(*feature); +} + } diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 7ac43c854..3c1d70294 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -1,12 +1,14 @@ +#pragma once +///@file + #include #include #include -#include "types.hh" - #include -#pragma once +#include "types.hh" +#include "experimental-features.hh" namespace nix { @@ -123,21 +125,21 @@ public: void reapplyUnknownSettings(); }; -/* A class to simplify providing configuration settings. The typical - use is to inherit Config and add Setting members: - - class MyClass : private Config - { - Setting foo{this, 123, "foo", "the number of foos to use"}; - Setting bar{this, "blabla", "bar", "the name of the bar"}; - - MyClass() : Config(readConfigFile("/etc/my-app.conf")) - { - std::cout << foo << "\n"; // will print 123 unless overridden - } - }; -*/ - +/** + * A class to simplify providing configuration settings. The typical + * use is to inherit Config and add Setting members: + * + * class MyClass : private Config + * { + * Setting foo{this, 123, "foo", "the number of foos to use"}; + * Setting bar{this, "blabla", "bar", "the name of the bar"}; + * + * MyClass() : Config(readConfigFile("/etc/my-app.conf")) + * { + * std::cout << foo << "\n"; // will print 123 unless overridden + * } + * }; + */ class Config : public AbstractConfig { friend class AbstractSetting; @@ -194,12 +196,15 @@ public: bool overridden = false; + std::optional experimentalFeature; + protected: AbstractSetting( const std::string & name, const std::string & description, - const std::set & aliases); + const std::set & aliases, + std::optional experimentalFeature = std::nullopt); virtual ~AbstractSetting() { @@ -224,7 +229,9 @@ protected: bool isOverridden() const { return overridden; } }; -/* A setting of type T. */ +/** + * A setting of type T. + */ template class BaseSetting : public AbstractSetting { @@ -240,8 +247,9 @@ public: const bool documentDefault, const std::string & name, const std::string & description, - const std::set & aliases = {}) - : AbstractSetting(name, description, aliases) + const std::set & aliases = {}, + std::optional experimentalFeature = std::nullopt) + : AbstractSetting(name, description, aliases, experimentalFeature) , value(def) , defaultValue(def) , documentDefault(documentDefault) @@ -296,8 +304,9 @@ public: const std::string & name, const std::string & description, const std::set & aliases = {}, - const bool documentDefault = true) - : BaseSetting(def, documentDefault, name, description, aliases) + const bool documentDefault = true, + std::optional experimentalFeature = std::nullopt) + : BaseSetting(def, documentDefault, name, description, aliases, experimentalFeature) { options->addSetting(this); } @@ -305,8 +314,10 @@ public: void operator =(const T & v) { this->assign(v); } }; -/* A special setting for Paths. These are automatically canonicalised - (e.g. "/foo//bar/" becomes "/foo/bar"). */ +/** + * A special setting for Paths. These are automatically canonicalised + * (e.g. "/foo//bar/" becomes "/foo/bar"). + */ class PathSetting : public BaseSetting { bool allowEmpty; @@ -357,4 +368,37 @@ struct GlobalConfig : public AbstractConfig extern GlobalConfig globalConfig; + +struct ExperimentalFeatureSettings : Config { + + Setting> experimentalFeatures{this, {}, "experimental-features", + "Experimental Nix features to enable."}; + + /** + * Check whether the given experimental feature is enabled. + */ + bool isEnabled(const ExperimentalFeature &) const; + + /** + * Require an experimental feature be enabled, throwing an error if it is + * not. + */ + void require(const ExperimentalFeature &) const; + + /** + * `std::nullopt` pointer means no feature, which means there is nothing that could be + * disabled, and so the function returns true in that case. + */ + bool isEnabled(const std::optional &) const; + + /** + * `std::nullopt` pointer means no feature, which means there is nothing that could be + * disabled, and so the function does nothing in that case. + */ + void require(const std::optional &) const; +}; + +// FIXME: don't use a global variable. +extern ExperimentalFeatureSettings experimentalFeatureSettings; + } diff --git a/src/libutil/error.cc b/src/libutil/error.cc index e4f0d4677..c9d61942a 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -302,14 +302,14 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s if (!einfo.traces.empty()) { size_t count = 0; for (const auto & trace : einfo.traces) { + if (trace.hint.str().empty()) continue; + if (frameOnly && !trace.frame) continue; + if (!showTrace && count > 3) { oss << "\n" << ANSI_WARNING "(stack trace truncated; use '--show-trace' to show the full trace)" ANSI_NORMAL << "\n"; break; } - if (trace.hint.str().empty()) continue; - if (frameOnly && !trace.frame) continue; - count++; frameOnly = trace.frame; diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 0ebeaba61..6a0923081 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -1,4 +1,19 @@ #pragma once +/** + * @file + * + * @brief This file defines two main structs/classes used in nix error handling. + * + * ErrorInfo provides a standard payload of error information, with conversion to string + * happening in the logger rather than at the call site. + * + * BaseError is the ancestor of nix specific exceptions (and Interrupted), and contains + * an ErrorInfo. + * + * ErrorInfo structs are sent to the logger as part of an exception, or directly with the + * logError or logWarning macros. + * See libutil/tests/logging.cc for usage examples. + */ #include "suggestions.hh" #include "ref.hh" @@ -26,22 +41,6 @@ namespace nix { -/* - - This file defines two main structs/classes used in nix error handling. - - ErrorInfo provides a standard payload of error information, with conversion to string - happening in the logger rather than at the call site. - - BaseError is the ancestor of nix specific exceptions (and Interrupted), and contains - an ErrorInfo. - - ErrorInfo structs are sent to the logger as part of an exception, or directly with the - logError or logWarning macros. - - See libutil/tests/logging.cc for usage examples. - - */ typedef enum { lvlError = 0, @@ -54,20 +53,26 @@ typedef enum { lvlVomit } Verbosity; -// the lines of code surrounding an error. +/** + * The lines of code surrounding an error. + */ struct LinesOfCode { std::optional prevLineOfCode; std::optional errLineOfCode; std::optional nextLineOfCode; }; -/* An abstract type that represents a location in a source file. */ +/** + * An abstract type that represents a location in a source file. + */ struct AbstractPos { uint32_t line = 0; uint32_t column = 0; - /* Return the contents of the source file. */ + /** + * Return the contents of the source file. + */ virtual std::optional getSource() const { return std::nullopt; }; @@ -104,8 +109,10 @@ struct ErrorInfo { std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool showTrace); -/* BaseError should generally not be caught, as it has Interrupted as - a subclass. Catch Error instead. */ +/** + * BaseError should generally not be caught, as it has Interrupted as + * a subclass. Catch Error instead. + */ class BaseError : public std::exception { protected: diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index ac372e03e..5948ad7ad 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "comparator.hh" #include "error.hh" @@ -12,7 +13,7 @@ namespace nix { * * If you update this, don’t forget to also change the map defining their * string representation in the corresponding `.cc` file. - **/ + */ enum struct ExperimentalFeature { CaDerivations, diff --git a/src/libutil/finally.hh b/src/libutil/finally.hh index dee2e8d2f..db654301f 100644 --- a/src/libutil/finally.hh +++ b/src/libutil/finally.hh @@ -1,6 +1,9 @@ #pragma once +///@file -/* A trivial class to run a function at the end of a scope. */ +/** + * A trivial class to run a function at the end of a scope. + */ template class Finally { diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index e11426b88..727255b45 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -8,20 +9,25 @@ namespace nix { -/* Inherit some names from other namespaces for convenience. */ +/** + * Inherit some names from other namespaces for convenience. + */ using boost::format; -/* A variadic template that does nothing. Useful to call a function - for all variadic arguments but ignoring the result. */ +/** + * A variadic template that does nothing. Useful to call a function + * for all variadic arguments but ignoring the result. + */ struct nop { template nop(T...) {} }; -/* A helper for formatting strings. ‘fmt(format, a_0, ..., a_n)’ is - equivalent to ‘boost::format(format) % a_0 % ... % - ... a_n’. However, ‘fmt(s)’ is equivalent to ‘s’ (so no %-expansion - takes place). */ - +/** + * A helper for formatting strings. ‘fmt(format, a_0, ..., a_n)’ is + * equivalent to ‘boost::format(format) % a_0 % ... % + * ... a_n’. However, ‘fmt(s)’ is equivalent to ‘s’ (so no %-expansion + * takes place). + */ template inline void formatHelper(F & f) { diff --git a/src/libutil/git.hh b/src/libutil/git.hh index cb13ef0e5..bf2b9a286 100644 --- a/src/libutil/git.hh +++ b/src/libutil/git.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -8,21 +9,23 @@ namespace nix { namespace git { -// A line from the output of `git ls-remote --symref`. -// -// These can be of two kinds: -// -// - Symbolic references of the form -// -// ref: {target} {reference} -// -// where {target} is itself a reference and {reference} is optional -// -// - Object references of the form -// -// {target} {reference} -// -// where {target} is a commit id and {reference} is mandatory +/** + * A line from the output of `git ls-remote --symref`. + * + * These can be of two kinds: + * + * - Symbolic references of the form + * + * ref: {target} {reference} + * + * where {target} is itself a reference and {reference} is optional + * + * - Object references of the form + * + * {target} {reference} + * + * where {target} is a commit id and {reference} is mandatory + */ struct LsRemoteRefLine { enum struct Kind { Symbolic, diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index d2fd0c15a..5735e4715 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -71,12 +71,13 @@ const std::string base16Chars = "0123456789abcdef"; static std::string printHash16(const Hash & hash) { - char buf[hash.hashSize * 2]; + std::string buf; + buf.reserve(hash.hashSize * 2); for (unsigned int i = 0; i < hash.hashSize; i++) { - buf[i * 2] = base16Chars[hash.hash[i] >> 4]; - buf[i * 2 + 1] = base16Chars[hash.hash[i] & 0x0f]; + buf.push_back(base16Chars[hash.hash[i] >> 4]); + buf.push_back(base16Chars[hash.hash[i] & 0x0f]); } - return std::string(buf, hash.hashSize * 2); + return buf; } @@ -130,7 +131,7 @@ std::string Hash::to_string(Base base, bool includeType) const break; case Base64: case SRI: - s += base64Encode(std::string((const char *) hash, hashSize)); + s += base64Encode(std::string_view((const char *) hash, hashSize)); break; } return s; @@ -403,7 +404,7 @@ HashType parseHashType(std::string_view s) throw UsageError("unknown hash algorithm '%1%'", s); } -std::string printHashType(HashType ht) +std::string_view printHashType(HashType ht) { switch (ht) { case htMD5: return "md5"; diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 00f70a572..be1fdba2a 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "serialise.hh" @@ -33,62 +34,86 @@ struct Hash HashType type; - /* Create a zero-filled hash object. */ + /** + * Create a zero-filled hash object. + */ Hash(HashType type); - /* Parse the hash from a string representation in the format - "[:]" or "-" (a - Subresource Integrity hash expression). If the 'type' argument - is not present, then the hash type must be specified in the - string. */ + /** + * Parse the hash from a string representation in the format + * "[:]" or "-" (a + * Subresource Integrity hash expression). If the 'type' argument + * is not present, then the hash type must be specified in the + * string. + */ static Hash parseAny(std::string_view s, std::optional type); - /* Parse a hash from a string representation like the above, except the - type prefix is mandatory is there is no separate arguement. */ + /** + * Parse a hash from a string representation like the above, except the + * type prefix is mandatory is there is no separate arguement. + */ static Hash parseAnyPrefixed(std::string_view s); - /* Parse a plain hash that musst not have any prefix indicating the type. - The type is passed in to disambiguate. */ + /** + * Parse a plain hash that musst not have any prefix indicating the type. + * The type is passed in to disambiguate. + */ static Hash parseNonSRIUnprefixed(std::string_view s, HashType type); static Hash parseSRI(std::string_view original); private: - /* The type must be provided, the string view must not include - prefix. `isSRI` helps disambigate the various base-* encodings. */ + /** + * The type must be provided, the string view must not include + * prefix. `isSRI` helps disambigate the various base-* encodings. + */ Hash(std::string_view s, HashType type, bool isSRI); public: - /* Check whether two hash are equal. */ + /** + * Check whether two hash are equal. + */ bool operator == (const Hash & h2) const; - /* Check whether two hash are not equal. */ + /** + * Check whether two hash are not equal. + */ bool operator != (const Hash & h2) const; - /* For sorting. */ + /** + * For sorting. + */ bool operator < (const Hash & h) const; - /* Returns the length of a base-16 representation of this hash. */ + /** + * Returns the length of a base-16 representation of this hash. + */ size_t base16Len() const { return hashSize * 2; } - /* Returns the length of a base-32 representation of this hash. */ + /** + * Returns the length of a base-32 representation of this hash. + */ size_t base32Len() const { return (hashSize * 8 - 1) / 5 + 1; } - /* Returns the length of a base-64 representation of this hash. */ + /** + * Returns the length of a base-64 representation of this hash. + */ size_t base64Len() const { return ((4 * hashSize / 3) + 3) & ~3; } - /* Return a string representation of the hash, in base-16, base-32 - or base-64. By default, this is prefixed by the hash type - (e.g. "sha256:"). */ + /** + * Return a string representation of the hash, in base-16, base-32 + * or base-64. By default, this is prefixed by the hash type + * (e.g. "sha256:"). + */ std::string to_string(Base base, bool includeType) const; std::string gitRev() const @@ -104,36 +129,54 @@ public: static Hash dummy; }; -/* Helper that defaults empty hashes to the 0 hash. */ +/** + * Helper that defaults empty hashes to the 0 hash. + */ Hash newHashAllowEmpty(std::string_view hashStr, std::optional ht); -/* Print a hash in base-16 if it's MD5, or base-32 otherwise. */ +/** + * Print a hash in base-16 if it's MD5, or base-32 otherwise. + */ std::string printHash16or32(const Hash & hash); -/* Compute the hash of the given string. */ +/** + * Compute the hash of the given string. + */ Hash hashString(HashType ht, std::string_view s); -/* Compute the hash of the given file. */ +/** + * Compute the hash of the given file. + */ Hash hashFile(HashType ht, const Path & path); -/* Compute the hash of the given path. The hash is defined as - (essentially) hashString(ht, dumpPath(path)). */ +/** + * Compute the hash of the given path. The hash is defined as + * (essentially) hashString(ht, dumpPath(path)). + */ typedef std::pair HashResult; HashResult hashPath(HashType ht, const Path & path, PathFilter & filter = defaultPathFilter); -/* Compress a hash to the specified number of bytes by cyclically - XORing bytes together. */ +/** + * Compress a hash to the specified number of bytes by cyclically + * XORing bytes together. + */ Hash compressHash(const Hash & hash, unsigned int newSize); -/* Parse a string representing a hash type. */ +/** + * Parse a string representing a hash type. + */ HashType parseHashType(std::string_view s); -/* Will return nothing on parse error */ +/** + * Will return nothing on parse error + */ std::optional parseHashTypeOpt(std::string_view s); -/* And the reverse. */ -std::string printHashType(HashType ht); +/** + * And the reverse. + */ +std::string_view printHashType(HashType ht); union Ctx; diff --git a/src/libutil/hilite.hh b/src/libutil/hilite.hh index f8bdbfc55..2d5cf7c6f 100644 --- a/src/libutil/hilite.hh +++ b/src/libutil/hilite.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -6,11 +7,13 @@ namespace nix { -/* Highlight all the given matches in the given string `s` by wrapping - them between `prefix` and `postfix`. - - If some matches overlap, then their union will be wrapped rather - than the individual matches. */ +/** + * Highlight all the given matches in the given string `s` by wrapping + * them between `prefix` and `postfix`. + * + * If some matches overlap, then their union will be wrapped rather + * than the individual matches. + */ std::string hiliteMatches( std::string_view s, std::vector matches, diff --git a/src/libutil/json-impls.hh b/src/libutil/json-impls.hh index bd75748ad..b26163a04 100644 --- a/src/libutil/json-impls.hh +++ b/src/libutil/json-impls.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "nlohmann/json_fwd.hpp" diff --git a/src/libutil/json-utils.hh b/src/libutil/json-utils.hh index b8a031227..eb00e954f 100644 --- a/src/libutil/json-utils.hh +++ b/src/libutil/json-utils.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 7cac75ce1..5a2dd99af 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -65,9 +65,10 @@ public: switch (lvl) { case lvlError: c = '3'; break; case lvlWarn: c = '4'; break; - case lvlInfo: c = '5'; break; + case lvlNotice: case lvlInfo: c = '5'; break; case lvlTalkative: case lvlChatty: c = '6'; break; - default: c = '7'; + case lvlDebug: case lvlVomit: c = '7'; + default: c = '7'; break; // should not happen, and missing enum case is reported by -Werror=switch-enum } prefix = std::string("<") + c + ">"; } diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 59a707eef..5aa6bee95 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "error.hh" @@ -72,6 +73,9 @@ public: virtual void stop() { }; + virtual void pause() { }; + virtual void resume() { }; + // Whether the logger prints the whole build log virtual bool isVerbose() { return false; } @@ -179,12 +183,17 @@ bool handleJSONLogMessage(const std::string & msg, const Activity & act, std::map & activities, bool trusted); -extern Verbosity verbosity; /* suppress msgs > this */ +/** + * suppress msgs > this + */ +extern Verbosity verbosity; -/* Print a message with the standard ErrorInfo format. - In general, use these 'log' macros for reporting problems that may require user - intervention or that need more explanation. Use the 'print' macros for more - lightweight status messages. */ +/** + * Print a message with the standard ErrorInfo format. + * In general, use these 'log' macros for reporting problems that may require user + * intervention or that need more explanation. Use the 'print' macros for more + * lightweight status messages. + */ #define logErrorInfo(level, errorInfo...) \ do { \ if ((level) <= nix::verbosity) { \ @@ -195,9 +204,11 @@ extern Verbosity verbosity; /* suppress msgs > this */ #define logError(errorInfo...) logErrorInfo(lvlError, errorInfo) #define logWarning(errorInfo...) logErrorInfo(lvlWarn, errorInfo) -/* Print a string message if the current log level is at least the specified - level. Note that this has to be implemented as a macro to ensure that the - arguments are evaluated lazily. */ +/** + * Print a string message if the current log level is at least the specified + * level. Note that this has to be implemented as a macro to ensure that the + * arguments are evaluated lazily. + */ #define printMsgUsing(loggerParam, level, args...) \ do { \ auto __lvl = level; \ @@ -214,7 +225,9 @@ extern Verbosity verbosity; /* suppress msgs > this */ #define debug(args...) printMsg(lvlDebug, args) #define vomit(args...) printMsg(lvlVomit, args) -/* if verbosity >= lvlWarn, print a message with a yellow 'warning:' prefix. */ +/** + * if verbosity >= lvlWarn, print a message with a yellow 'warning:' prefix. + */ template inline void warn(const std::string & fs, const Args & ... args) { diff --git a/src/libutil/lru-cache.hh b/src/libutil/lru-cache.hh index 6ef4a3e06..0e19517ed 100644 --- a/src/libutil/lru-cache.hh +++ b/src/libutil/lru-cache.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -7,7 +8,9 @@ namespace nix { -/* A simple least-recently used cache. Not thread-safe. */ +/** + * A simple least-recently used cache. Not thread-safe. + */ template class LRUCache { @@ -31,7 +34,9 @@ public: LRUCache(size_t capacity) : capacity(capacity) { } - /* Insert or upsert an item in the cache. */ + /** + * Insert or upsert an item in the cache. + */ void upsert(const Key & key, const Value & value) { if (capacity == 0) return; @@ -39,7 +44,9 @@ public: erase(key); if (data.size() >= capacity) { - /* Retire the oldest item. */ + /** + * Retire the oldest item. + */ auto oldest = lru.begin(); data.erase(*oldest); lru.erase(oldest); @@ -63,14 +70,18 @@ public: return true; } - /* Look up an item in the cache. If it exists, it becomes the most - recently used item. */ + /** + * Look up an item in the cache. If it exists, it becomes the most + * recently used item. + * */ std::optional get(const Key & key) { auto i = data.find(key); if (i == data.end()) return {}; - /* Move this item to the back of the LRU list. */ + /** + * Move this item to the back of the LRU list. + */ lru.erase(i->second.first.it); auto j = lru.insert(lru.end(), i); i->second.first.it = j; diff --git a/src/libutil/monitor-fd.hh b/src/libutil/monitor-fd.hh index 9518cf8aa..86d0115fc 100644 --- a/src/libutil/monitor-fd.hh +++ b/src/libutil/monitor-fd.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include diff --git a/src/libutil/namespaces.hh b/src/libutil/namespaces.hh index e82379b9c..0b7eeb66c 100644 --- a/src/libutil/namespaces.hh +++ b/src/libutil/namespaces.hh @@ -1,4 +1,5 @@ #pragma once +///@file namespace nix { diff --git a/src/libutil/pool.hh b/src/libutil/pool.hh index d49067bb9..6247b6125 100644 --- a/src/libutil/pool.hh +++ b/src/libutil/pool.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -11,33 +12,37 @@ namespace nix { -/* This template class implements a simple pool manager of resources - of some type R, such as database connections. It is used as - follows: - - class Connection { ... }; - - Pool pool; - - { - auto conn(pool.get()); - conn->exec("select ..."); - } - - Here, the Connection object referenced by ‘conn’ is automatically - returned to the pool when ‘conn’ goes out of scope. -*/ - +/** + * This template class implements a simple pool manager of resources + * of some type R, such as database connections. It is used as + * follows: + * + * class Connection { ... }; + * + * Pool pool; + * + * { + * auto conn(pool.get()); + * conn->exec("select ..."); + * } + * + * Here, the Connection object referenced by ‘conn’ is automatically + * returned to the pool when ‘conn’ goes out of scope. + */ template class Pool { public: - /* A function that produces new instances of R on demand. */ + /** + * A function that produces new instances of R on demand. + */ typedef std::function()> Factory; - /* A function that checks whether an instance of R is still - usable. Unusable instances are removed from the pool. */ + /** + * A function that checks whether an instance of R is still + * usable. Unusable instances are removed from the pool. + */ typedef std::function &)> Validator; private: diff --git a/src/libutil/ref.hh b/src/libutil/ref.hh index 7d38b059c..af5f8304c 100644 --- a/src/libutil/ref.hh +++ b/src/libutil/ref.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -6,8 +7,10 @@ namespace nix { -/* A simple non-nullable reference-counted pointer. Actually a wrapper - around std::shared_ptr that prevents null constructions. */ +/** + * A simple non-nullable reference-counted pointer. Actually a wrapper + * around std::shared_ptr that prevents null constructions. + */ template class ref { diff --git a/src/libutil/regex-combinators.hh b/src/libutil/regex-combinators.hh index 0b997b25a..87d6aa678 100644 --- a/src/libutil/regex-combinators.hh +++ b/src/libutil/regex-combinators.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 7476e6f6c..6e53239f5 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -186,6 +186,22 @@ static DefaultStackAllocator defaultAllocatorSingleton; StackAllocator *StackAllocator::defaultAllocator = &defaultAllocatorSingleton; +std::shared_ptr (*create_coro_gc_hook)() = []() -> std::shared_ptr { + return {}; +}; + +/* This class is used for entry and exit hooks on coroutines */ +class CoroutineContext { + /* Disable GC when entering the coroutine without the boehm patch, + * since it doesn't find the main thread stack in this case. + * std::shared_ptr performs type-erasure, so it will call the right + * deleter. */ + const std::shared_ptr coro_gc_hook = create_coro_gc_hook(); +public: + CoroutineContext() {}; + ~CoroutineContext() {}; +}; + std::unique_ptr sourceToSink(std::function fun) { struct SourceToSink : FinishSink @@ -206,7 +222,8 @@ std::unique_ptr sourceToSink(std::function fun) if (in.empty()) return; cur = in; - if (!coro) + if (!coro) { + CoroutineContext ctx; coro = coro_t::push_type(VirtualStackAllocator{}, [&](coro_t::pull_type & yield) { LambdaSource source([&](char *out, size_t out_len) { if (cur.empty()) { @@ -223,17 +240,24 @@ std::unique_ptr sourceToSink(std::function fun) }); fun(source); }); + } if (!*coro) { abort(); } - if (!cur.empty()) (*coro)(false); + if (!cur.empty()) { + CoroutineContext ctx; + (*coro)(false); + } } void finish() override { if (!coro) return; if (!*coro) abort(); - (*coro)(true); + { + CoroutineContext ctx; + (*coro)(true); + } if (*coro) abort(); } }; @@ -264,18 +288,23 @@ std::unique_ptr sinkToSource( size_t read(char * data, size_t len) override { - if (!coro) + if (!coro) { + CoroutineContext ctx; coro = coro_t::pull_type(VirtualStackAllocator{}, [&](coro_t::push_type & yield) { LambdaSink sink([&](std::string_view data) { if (!data.empty()) yield(std::string(data)); }); fun(sink); }); + } if (!*coro) { eof(); abort(); } if (pos == cur.size()) { - if (!cur.empty()) (*coro)(); + if (!cur.empty()) { + CoroutineContext ctx; + (*coro)(); + } cur = coro->get(); pos = 0; } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 7da5b07fd..ba6dbf619 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include @@ -10,7 +11,9 @@ namespace boost::context { struct stack_context; } namespace nix { -/* Abstract destination of binary data. */ +/** + * Abstract destination of binary data. + */ struct Sink { virtual ~Sink() { } @@ -18,7 +21,9 @@ struct Sink virtual bool good() { return true; } }; -/* Just throws away data. */ +/** + * Just throws away data. + */ struct NullSink : Sink { void operator () (std::string_view data) override @@ -32,8 +37,10 @@ struct FinishSink : virtual Sink }; -/* A buffered abstract sink. Warning: a BufferedSink should not be - used from multiple threads concurrently. */ +/** + * A buffered abstract sink. Warning: a BufferedSink should not be + * used from multiple threads concurrently. + */ struct BufferedSink : virtual Sink { size_t bufSize, bufPos; @@ -50,19 +57,25 @@ struct BufferedSink : virtual Sink }; -/* Abstract source of binary data. */ +/** + * Abstract source of binary data. + */ struct Source { virtual ~Source() { } - /* Store exactly ‘len’ bytes in the buffer pointed to by ‘data’. - It blocks until all the requested data is available, or throws - an error if it is not going to be available. */ + /** + * Store exactly ‘len’ bytes in the buffer pointed to by ‘data’. + * It blocks until all the requested data is available, or throws + * an error if it is not going to be available. + */ void operator () (char * data, size_t len); - /* Store up to ‘len’ in the buffer pointed to by ‘data’, and - return the number of bytes stored. It blocks until at least - one byte is available. */ + /** + * Store up to ‘len’ in the buffer pointed to by ‘data’, and + * return the number of bytes stored. It blocks until at least + * one byte is available. + */ virtual size_t read(char * data, size_t len) = 0; virtual bool good() { return true; } @@ -73,8 +86,10 @@ struct Source }; -/* A buffered abstract source. Warning: a BufferedSource should not be - used from multiple threads concurrently. */ +/** + * A buffered abstract source. Warning: a BufferedSource should not be + * used from multiple threads concurrently. + */ struct BufferedSource : Source { size_t bufSize, bufPosIn, bufPosOut; @@ -88,12 +103,16 @@ struct BufferedSource : Source bool hasData(); protected: - /* Underlying read call, to be overridden. */ + /** + * Underlying read call, to be overridden. + */ virtual size_t readUnbuffered(char * data, size_t len) = 0; }; -/* A sink that writes data to a file descriptor. */ +/** + * A sink that writes data to a file descriptor. + */ struct FdSink : BufferedSink { int fd; @@ -123,7 +142,9 @@ private: }; -/* A source that reads data from a file descriptor. */ +/** + * A source that reads data from a file descriptor. + */ struct FdSource : BufferedSource { int fd; @@ -149,7 +170,9 @@ private: }; -/* A sink that writes data to a string. */ +/** + * A sink that writes data to a string. + */ struct StringSink : Sink { std::string s; @@ -163,7 +186,9 @@ struct StringSink : Sink }; -/* A source that reads data from a string. */ +/** + * A source that reads data from a string. + */ struct StringSource : Source { std::string_view s; @@ -173,7 +198,9 @@ struct StringSource : Source }; -/* A sink that writes all incoming data to two other sinks. */ +/** + * A sink that writes all incoming data to two other sinks. + */ struct TeeSink : Sink { Sink & sink1, & sink2; @@ -186,7 +213,9 @@ struct TeeSink : Sink }; -/* Adapter class of a Source that saves all data read to a sink. */ +/** + * Adapter class of a Source that saves all data read to a sink. + */ struct TeeSource : Source { Source & orig; @@ -201,7 +230,9 @@ struct TeeSource : Source } }; -/* A reader that consumes the original Source until 'size'. */ +/** + * A reader that consumes the original Source until 'size'. + */ struct SizedSource : Source { Source & orig; @@ -219,7 +250,9 @@ struct SizedSource : Source return n; } - /* Consume the original source until no remain data is left to consume. */ + /** + * Consume the original source until no remain data is left to consume. + */ size_t drainAll() { std::vector buf(8192); @@ -232,7 +265,9 @@ struct SizedSource : Source } }; -/* A sink that that just counts the number of bytes given to it */ +/** + * A sink that that just counts the number of bytes given to it + */ struct LengthSink : Sink { uint64_t length = 0; @@ -243,7 +278,9 @@ struct LengthSink : Sink } }; -/* Convert a function into a sink. */ +/** + * Convert a function into a sink. + */ struct LambdaSink : Sink { typedef std::function lambda_t; @@ -259,7 +296,9 @@ struct LambdaSink : Sink }; -/* Convert a function into a source. */ +/** + * Convert a function into a source. + */ struct LambdaSource : Source { typedef std::function lambda_t; @@ -274,8 +313,10 @@ struct LambdaSource : Source } }; -/* Chain two sources together so after the first is exhausted, the second is - used */ +/** + * Chain two sources together so after the first is exhausted, the second is + * used + */ struct ChainSource : Source { Source & source1, & source2; @@ -289,8 +330,10 @@ struct ChainSource : Source std::unique_ptr sourceToSink(std::function fun); -/* Convert a function that feeds data into a Sink into a Source. The - Source executes the function as a coroutine. */ +/** + * Convert a function that feeds data into a Sink into a Source. The + * Source executes the function as a coroutine. + */ std::unique_ptr sinkToSource( std::function fun, std::function eof = []() { @@ -376,7 +419,9 @@ Source & operator >> (Source & in, bool & b) Error readError(Source & source); -/* An adapter that converts a std::basic_istream into a source. */ +/** + * An adapter that converts a std::basic_istream into a source. + */ struct StreamToSourceAdapter : Source { std::shared_ptr> istream; @@ -399,13 +444,14 @@ struct StreamToSourceAdapter : Source }; -/* A source that reads a distinct format of concatenated chunks back into its - logical form, in order to guarantee a known state to the original stream, - even in the event of errors. - - Use with FramedSink, which also allows the logical stream to be terminated - in the event of an exception. -*/ +/** + * A source that reads a distinct format of concatenated chunks back into its + * logical form, in order to guarantee a known state to the original stream, + * even in the event of errors. + * + * Use with FramedSink, which also allows the logical stream to be terminated + * in the event of an exception. + */ struct FramedSource : Source { Source & from; @@ -450,11 +496,12 @@ struct FramedSource : Source } }; -/* Write as chunks in the format expected by FramedSource. - - The exception_ptr reference can be used to terminate the stream when you - detect that an error has occurred on the remote end. -*/ +/** + * Write as chunks in the format expected by FramedSource. + * + * The exception_ptr reference can be used to terminate the stream when you + * detect that an error has occurred on the remote end. + */ struct FramedSink : nix::BufferedSink { BufferedSink & to; @@ -487,18 +534,27 @@ struct FramedSink : nix::BufferedSink }; }; -/* Stack allocation strategy for sinkToSource. - Mutable to avoid a boehm gc dependency in libutil. - - boost::context doesn't provide a virtual class, so we define our own. +/** + * Stack allocation strategy for sinkToSource. + * Mutable to avoid a boehm gc dependency in libutil. + * + * boost::context doesn't provide a virtual class, so we define our own. */ struct StackAllocator { virtual boost::context::stack_context allocate() = 0; virtual void deallocate(boost::context::stack_context sctx) = 0; - /* The stack allocator to use in sinkToSource and potentially elsewhere. - It is reassigned by the initGC() method in libexpr. */ + /** + * The stack allocator to use in sinkToSource and potentially elsewhere. + * It is reassigned by the initGC() method in libexpr. + */ static StackAllocator *defaultAllocator; }; +/* Disabling GC when entering a coroutine (without the boehm patch). + mutable to avoid boehm gc dependency in libutil. + */ +extern std::shared_ptr (*create_coro_gc_hook)(); + + } diff --git a/src/libutil/split.hh b/src/libutil/split.hh index 87a23b13e..3b9b2b83b 100644 --- a/src/libutil/split.hh +++ b/src/libutil/split.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -7,10 +8,12 @@ namespace nix { -// If `separator` is found, we return the portion of the string before the -// separator, and modify the string argument to contain only the part after the -// separator. Otherwise, we return `std::nullopt`, and we leave the argument -// string alone. +/** + * If `separator` is found, we return the portion of the string before the + * separator, and modify the string argument to contain only the part after the + * separator. Otherwise, we return `std::nullopt`, and we leave the argument + * string alone. + */ static inline std::optional splitPrefixTo(std::string_view & string, char separator) { auto sepInstance = string.find(separator); diff --git a/src/libutil/suggestions.hh b/src/libutil/suggestions.hh index d54dd8e31..9abf5ee5f 100644 --- a/src/libutil/suggestions.hh +++ b/src/libutil/suggestions.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "comparator.hh" #include "types.hh" @@ -13,7 +14,8 @@ int levenshteinDistance(std::string_view first, std::string_view second); */ class Suggestion { public: - int distance; // The smaller the better + /// The smaller the better + int distance; std::string suggestion; std::string to_string() const; @@ -43,7 +45,9 @@ public: std::ostream & operator<<(std::ostream & str, const Suggestion &); std::ostream & operator<<(std::ostream & str, const Suggestions &); -// Either a value of type `T`, or some suggestions +/** + * Either a value of type `T`, or some suggestions + */ template class OrSuggestions { public: diff --git a/src/libutil/sync.hh b/src/libutil/sync.hh index e1d591d77..47e4512b1 100644 --- a/src/libutil/sync.hh +++ b/src/libutil/sync.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -7,22 +8,22 @@ namespace nix { -/* This template class ensures synchronized access to a value of type - T. It is used as follows: - - struct Data { int x; ... }; - - Sync data; - - { - auto data_(data.lock()); - data_->x = 123; - } - - Here, "data" is automatically unlocked when "data_" goes out of - scope. -*/ - +/** + * This template class ensures synchronized access to a value of type + * T. It is used as follows: + * + * struct Data { int x; ... }; + * + * Sync data; + * + * { + * auto data_(data.lock()); + * data_->x = 123; + * } + * + * Here, "data" is automatically unlocked when "data_" goes out of + * scope. + */ template class Sync { diff --git a/src/libutil/tarfile.hh b/src/libutil/tarfile.hh index 4d9141fd4..24afb710a 100644 --- a/src/libutil/tarfile.hh +++ b/src/libutil/tarfile.hh @@ -1,3 +1,6 @@ +#pragma once +///@file + #include "serialise.hh" #include @@ -14,7 +17,7 @@ struct TarArchive { TarArchive(const Path & path); - // disable copy constructor + /// disable copy constructor TarArchive(const TarArchive &) = delete; void close(); diff --git a/src/libutil/tests/canon-path.cc b/src/libutil/tests/canon-path.cc index c1c5adadf..fc94ccc3d 100644 --- a/src/libutil/tests/canon-path.cc +++ b/src/libutil/tests/canon-path.cc @@ -107,15 +107,13 @@ namespace nix { } TEST(CanonPath, within) { - { - ASSERT_TRUE(CanonPath("foo").isWithin(CanonPath("foo"))); - ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("bar"))); - ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("fo"))); - ASSERT_TRUE(CanonPath("foo/bar").isWithin(CanonPath("foo"))); - ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("foo/bar"))); - ASSERT_TRUE(CanonPath("/foo/bar/default.nix").isWithin(CanonPath("/"))); - ASSERT_TRUE(CanonPath("/").isWithin(CanonPath("/"))); - } + ASSERT_TRUE(CanonPath("foo").isWithin(CanonPath("foo"))); + ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("bar"))); + ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("fo"))); + ASSERT_TRUE(CanonPath("foo/bar").isWithin(CanonPath("foo"))); + ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("foo/bar"))); + ASSERT_TRUE(CanonPath("/foo/bar/default.nix").isWithin(CanonPath("/"))); + ASSERT_TRUE(CanonPath("/").isWithin(CanonPath("/"))); } TEST(CanonPath, sort) { @@ -127,29 +125,38 @@ namespace nix { } TEST(CanonPath, allowed) { - { - std::set allowed { - CanonPath("foo/bar"), - CanonPath("foo!"), - CanonPath("xyzzy"), - CanonPath("a/b/c"), - }; + std::set allowed { + CanonPath("foo/bar"), + CanonPath("foo!"), + CanonPath("xyzzy"), + CanonPath("a/b/c"), + }; - ASSERT_TRUE (CanonPath("foo/bar").isAllowed(allowed)); - ASSERT_TRUE (CanonPath("foo/bar/bla").isAllowed(allowed)); - ASSERT_TRUE (CanonPath("foo").isAllowed(allowed)); - ASSERT_FALSE(CanonPath("bar").isAllowed(allowed)); - ASSERT_FALSE(CanonPath("bar/a").isAllowed(allowed)); - ASSERT_TRUE (CanonPath("a").isAllowed(allowed)); - ASSERT_TRUE (CanonPath("a/b").isAllowed(allowed)); - ASSERT_TRUE (CanonPath("a/b/c").isAllowed(allowed)); - ASSERT_TRUE (CanonPath("a/b/c/d").isAllowed(allowed)); - ASSERT_TRUE (CanonPath("a/b/c/d/e").isAllowed(allowed)); - ASSERT_FALSE(CanonPath("a/b/a").isAllowed(allowed)); - ASSERT_FALSE(CanonPath("a/b/d").isAllowed(allowed)); - ASSERT_FALSE(CanonPath("aaa").isAllowed(allowed)); - ASSERT_FALSE(CanonPath("zzz").isAllowed(allowed)); - ASSERT_TRUE (CanonPath("/").isAllowed(allowed)); - } + ASSERT_TRUE (CanonPath("foo/bar").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("foo/bar/bla").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("foo").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("bar").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("bar/a").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("a").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("a/b").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("a/b/c").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("a/b/c/d").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("a/b/c/d/e").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("a/b/a").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("a/b/d").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("aaa").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("zzz").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("/").isAllowed(allowed)); + } + + TEST(CanonPath, makeRelative) { + CanonPath d("/foo/bar"); + ASSERT_EQ(d.makeRelative(CanonPath("/foo/bar")), "."); + ASSERT_EQ(d.makeRelative(CanonPath("/foo")), ".."); + ASSERT_EQ(d.makeRelative(CanonPath("/")), "../.."); + ASSERT_EQ(d.makeRelative(CanonPath("/foo/bar/xyzzy")), "xyzzy"); + ASSERT_EQ(d.makeRelative(CanonPath("/foo/bar/xyzzy/bla")), "xyzzy/bla"); + ASSERT_EQ(d.makeRelative(CanonPath("/foo/xyzzy/bla")), "../xyzzy/bla"); + ASSERT_EQ(d.makeRelative(CanonPath("/xyzzy/bla")), "../../xyzzy/bla"); } } diff --git a/src/libutil/tests/hash.hh b/src/libutil/tests/hash.hh index 9e9650e6e..1f9fa59ae 100644 --- a/src/libutil/tests/hash.hh +++ b/src/libutil/tests/hash.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include diff --git a/src/libutil/thread-pool.hh b/src/libutil/thread-pool.hh index b22e0d162..0e09fae97 100644 --- a/src/libutil/thread-pool.hh +++ b/src/libutil/thread-pool.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "sync.hh" #include "util.hh" @@ -13,8 +14,10 @@ namespace nix { MakeError(ThreadPoolShutDown, Error); -/* A simple thread pool that executes a queue of work items - (lambdas). */ +/** + * A simple thread pool that executes a queue of work items + * (lambdas). + */ class ThreadPool { public: @@ -23,19 +26,30 @@ public: ~ThreadPool(); - // FIXME: use std::packaged_task? + /** + * An individual work item. + * + * \todo use std::packaged_task? + */ typedef std::function work_t; - /* Enqueue a function to be executed by the thread pool. */ + /** + * Enqueue a function to be executed by the thread pool. + */ void enqueue(const work_t & t); - /* Execute work items until the queue is empty. Note that work - items are allowed to add new items to the queue; this is - handled correctly. Queue processing stops prematurely if any - work item throws an exception. This exception is propagated to - the calling thread. If multiple work items throw an exception - concurrently, only one item is propagated; the others are - printed on stderr and otherwise ignored. */ + /** + * Execute work items until the queue is empty. + * + * \note Note that work items are allowed to add new items to the + * queue; this is handled correctly. + * + * Queue processing stops prematurely if any work item throws an + * exception. This exception is propagated to the calling thread. If + * multiple work items throw an exception concurrently, only one + * item is propagated; the others are printed on stderr and + * otherwise ignored. + */ void process(); private: @@ -62,9 +76,11 @@ private: void shutdown(); }; -/* Process in parallel a set of items of type T that have a partial - ordering between them. Thus, any item is only processed after all - its dependencies have been processed. */ +/** + * Process in parallel a set of items of type T that have a partial + * ordering between them. Thus, any item is only processed after all + * its dependencies have been processed. + */ template void processGraph( ThreadPool & pool, diff --git a/src/libutil/topo-sort.hh b/src/libutil/topo-sort.hh index 7418be5e0..a52811fbf 100644 --- a/src/libutil/topo-sort.hh +++ b/src/libutil/topo-sort.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "error.hh" diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 6bcbd7e1d..c86f52175 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "ref.hh" @@ -17,7 +18,9 @@ typedef std::set StringSet; typedef std::map StringMap; typedef std::map StringPairs; -/* Paths are just strings. */ +/** + * Paths are just strings. + */ typedef std::string Path; typedef std::string_view PathView; typedef std::list Paths; @@ -25,15 +28,19 @@ typedef std::set PathSet; typedef std::vector> Headers; -/* Helper class to run code at startup. */ +/** + * Helper class to run code at startup. + */ template struct OnStartup { OnStartup(T && t) { t(); } }; -/* Wrap bools to prevent string literals (i.e. 'char *') from being - cast to a bool in Attr. */ +/** + * Wrap bools to prevent string literals (i.e. 'char *') from being + * cast to a bool in Attr. + */ template struct Explicit { T t; @@ -45,21 +52,25 @@ struct Explicit { }; -/* This wants to be a little bit like rust's Cow type. - Some parts of the evaluator benefit greatly from being able to reuse - existing allocations for strings, but have to be able to also use - newly allocated storage for values. - - We do not define implicit conversions, even with ref qualifiers, - since those can easily become ambiguous to the reader and can degrade - into copying behaviour we want to avoid. */ +/** + * This wants to be a little bit like rust's Cow type. + * Some parts of the evaluator benefit greatly from being able to reuse + * existing allocations for strings, but have to be able to also use + * newly allocated storage for values. + * + * We do not define implicit conversions, even with ref qualifiers, + * since those can easily become ambiguous to the reader and can degrade + * into copying behaviour we want to avoid. + */ class BackedStringView { private: std::variant data; - /* Needed to introduce a temporary since operator-> must return - a pointer. Without this we'd need to store the view object - even when we already own a string. */ + /** + * Needed to introduce a temporary since operator-> must return + * a pointer. Without this we'd need to store the view object + * even when we already own a string. + */ class Ptr { private: std::string_view view; @@ -77,8 +88,10 @@ public: BackedStringView(const BackedStringView &) = delete; BackedStringView & operator=(const BackedStringView &) = delete; - /* We only want move operations defined since the sole purpose of - this type is to avoid copies. */ + /** + * We only want move operations defined since the sole purpose of + * this type is to avoid copies. + */ BackedStringView(BackedStringView && other) = default; BackedStringView & operator=(BackedStringView && other) = default; diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh index d5e6a2736..98162b0f7 100644 --- a/src/libutil/url-parts.hh +++ b/src/libutil/url-parts.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -22,21 +23,22 @@ const static std::string segmentRegex = "(?:" + pcharRegex + "*)"; const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)"; const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)"; -// A Git ref (i.e. branch or tag name). -const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.\\/-]*"; // FIXME: check +/// A Git ref (i.e. branch or tag name). +/// \todo check that this is correct. +const static std::string refRegexS = "[a-zA-Z0-9@][a-zA-Z0-9_.\\/@-]*"; extern std::regex refRegex; -// Instead of defining what a good Git Ref is, we define what a bad Git Ref is -// This is because of the definition of a ref in refs.c in https://github.com/git/git -// See tests/fetchGitRefs.sh for the full definition +/// Instead of defining what a good Git Ref is, we define what a bad Git Ref is +/// This is because of the definition of a ref in refs.c in https://github.com/git/git +/// See tests/fetchGitRefs.sh for the full definition const static std::string badGitRefRegexS = "//|^[./]|/\\.|\\.\\.|[[:cntrl:][:space:]:?^~\[]|\\\\|\\*|\\.lock$|\\.lock/|@\\{|[/.]$|^@$|^$"; extern std::regex badGitRefRegex; -// A Git revision (a SHA-1 commit hash). +/// A Git revision (a SHA-1 commit hash). const static std::string revRegexS = "[0-9a-fA-F]{40}"; extern std::regex revRegex; -// A ref or revision, or a ref followed by a revision. +/// A ref or revision, or a ref followed by a revision. const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegexS + ")(?:/(" + revRegexS + "))?))"; const static std::string flakeIdRegexS = "[a-zA-Z][a-zA-Z0-9_-]*"; diff --git a/src/libutil/url.hh b/src/libutil/url.hh index ddd673d65..d2413ec0e 100644 --- a/src/libutil/url.hh +++ b/src/libutil/url.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "error.hh" @@ -7,7 +8,8 @@ namespace nix { struct ParsedURL { std::string url; - std::string base; // URL without query/fragment + /// URL without query/fragment + std::string base; std::string scheme; std::optional authority; std::string path; @@ -28,7 +30,7 @@ std::map decodeQuery(const std::string & query); ParsedURL parseURL(const std::string & url); -/* +/** * Although that’s not really standardized anywhere, an number of tools * use a scheme of the form 'x+y' in urls, where y is the “transport layer” * scheme, and x is the “application layer” scheme. diff --git a/src/libutil/util.hh b/src/libutil/util.hh index a8e6f9fbb..56160baaf 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "error.hh" @@ -32,77 +33,112 @@ struct Sink; struct Source; -/* The system for which Nix is compiled. */ +/** + * The system for which Nix is compiled. + */ extern const std::string nativeSystem; -/* Return an environment variable. */ +/** + * @return an environment variable. + */ std::optional getEnv(const std::string & key); -/* Return a non empty environment variable. Returns nullopt if the env -variable is set to "" */ +/** + * @return a non empty environment variable. Returns nullopt if the env + * variable is set to "" + */ std::optional getEnvNonEmpty(const std::string & key); -/* Get the entire environment. */ +/** + * Get the entire environment. + */ std::map getEnv(); -/* Clear the environment. */ +/** + * Clear the environment. + */ void clearEnv(); -/* Return an absolutized path, resolving paths relative to the - specified directory, or the current directory otherwise. The path - is also canonicalised. */ +/** + * @return An absolutized path, resolving paths relative to the + * specified directory, or the current directory otherwise. The path + * is also canonicalised. + */ Path absPath(Path path, std::optional dir = {}, bool resolveSymlinks = false); -/* Canonicalise a path by removing all `.' or `..' components and - double or trailing slashes. Optionally resolves all symlink - components such that each component of the resulting path is *not* - a symbolic link. */ +/** + * Canonicalise a path by removing all `.` or `..` components and + * double or trailing slashes. Optionally resolves all symlink + * components such that each component of the resulting path is *not* + * a symbolic link. + */ Path canonPath(PathView path, bool resolveSymlinks = false); -/* Return the directory part of the given canonical path, i.e., - everything before the final `/'. If the path is the root or an - immediate child thereof (e.g., `/foo'), this means `/' - is returned.*/ +/** + * @return The directory part of the given canonical path, i.e., + * everything before the final `/`. If the path is the root or an + * immediate child thereof (e.g., `/foo`), this means `/` + * is returned. + */ Path dirOf(const PathView path); -/* Return the base name of the given canonical path, i.e., everything - following the final `/' (trailing slashes are removed). */ +/** + * @return the base name of the given canonical path, i.e., everything + * following the final `/` (trailing slashes are removed). + */ std::string_view baseNameOf(std::string_view path); -/* Perform tilde expansion on a path. */ +/** + * Perform tilde expansion on a path. + */ std::string expandTilde(std::string_view path); -/* Check whether 'path' is a descendant of 'dir'. Both paths must be - canonicalized. */ +/** + * Check whether 'path' is a descendant of 'dir'. Both paths must be + * canonicalized. + */ bool isInDir(std::string_view path, std::string_view dir); -/* Check whether 'path' is equal to 'dir' or a descendant of - 'dir'. Both paths must be canonicalized. */ +/** + * Check whether 'path' is equal to 'dir' or a descendant of + * 'dir'. Both paths must be canonicalized. + */ bool isDirOrInDir(std::string_view path, std::string_view dir); -/* Get status of `path'. */ +/** + * Get status of `path`. + */ struct stat stat(const Path & path); struct stat lstat(const Path & path); -/* Return true iff the given path exists. */ +/** + * @return true iff the given path exists. + */ bool pathExists(const Path & path); -/* Read the contents (target) of a symbolic link. The result is not - in any way canonicalised. */ +/** + * Read the contents (target) of a symbolic link. The result is not + * in any way canonicalised. + */ Path readLink(const Path & path); bool isLink(const Path & path); -/* Read the contents of a directory. The entries `.' and `..' are - removed. */ +/** + * Read the contents of a directory. The entries `.` and `..` are + * removed. + */ struct DirEntry { std::string name; ino_t ino; - unsigned char type; // one of DT_* + /** + * one of DT_* + */ + unsigned char type; DirEntry(std::string name, ino_t ino, unsigned char type) : name(std::move(name)), ino(ino), type(type) { } }; @@ -113,74 +149,110 @@ DirEntries readDirectory(const Path & path); unsigned char getFileType(const Path & path); -/* Read the contents of a file into a string. */ +/** + * Read the contents of a file into a string. + */ std::string readFile(int fd); std::string readFile(const Path & path); void readFile(const Path & path, Sink & sink); -/* Write a string to a file. */ +/** + * Write a string to a file. + */ void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, bool sync = false); void writeFile(const Path & path, Source & source, mode_t mode = 0666, bool sync = false); -/* Flush a file's parent directory to disk */ +/** + * Flush a file's parent directory to disk + */ void syncParent(const Path & path); -/* Read a line from a file descriptor. */ +/** + * Read a line from a file descriptor. + */ std::string readLine(int fd); -/* Write a line to a file descriptor. */ +/** + * Write a line to a file descriptor. + */ void writeLine(int fd, std::string s); -/* Delete a path; i.e., in the case of a directory, it is deleted - recursively. It's not an error if the path does not exist. The - second variant returns the number of bytes and blocks freed. */ +/** + * Delete a path; i.e., in the case of a directory, it is deleted + * recursively. It's not an error if the path does not exist. The + * second variant returns the number of bytes and blocks freed. + */ void deletePath(const Path & path); void deletePath(const Path & path, uint64_t & bytesFreed); std::string getUserName(); -/* Return the given user's home directory from /etc/passwd. */ +/** + * @return the given user's home directory from /etc/passwd. + */ Path getHomeOf(uid_t userId); -/* Return $HOME or the user's home directory from /etc/passwd. */ +/** + * @return $HOME or the user's home directory from /etc/passwd. + */ Path getHome(); -/* Return $XDG_CACHE_HOME or $HOME/.cache. */ +/** + * @return $XDG_CACHE_HOME or $HOME/.cache. + */ Path getCacheDir(); -/* Return $XDG_CONFIG_HOME or $HOME/.config. */ +/** + * @return $XDG_CONFIG_HOME or $HOME/.config. + */ Path getConfigDir(); -/* Return the directories to search for user configuration files */ +/** + * @return the directories to search for user configuration files + */ std::vector getConfigDirs(); -/* Return $XDG_DATA_HOME or $HOME/.local/share. */ +/** + * @return $XDG_DATA_HOME or $HOME/.local/share. + */ Path getDataDir(); -/* Return the path of the current executable. */ +/** + * @return the path of the current executable. + */ std::optional getSelfExe(); -/* Return $XDG_STATE_HOME or $HOME/.local/state. */ +/** + * @return $XDG_STATE_HOME or $HOME/.local/state. + */ Path getStateDir(); -/* Create the Nix state directory and return the path to it. */ +/** + * Create the Nix state directory and return the path to it. + */ Path createNixStateDir(); -/* Create a directory and all its parents, if necessary. Returns the - list of created directories, in order of creation. */ +/** + * Create a directory and all its parents, if necessary. Returns the + * list of created directories, in order of creation. + */ Paths createDirs(const Path & path); inline Paths createDirs(PathView path) { return createDirs(Path(path)); } -/* Create a symlink. */ +/** + * Create a symlink. + */ void createSymlink(const Path & target, const Path & link, std::optional mtime = {}); -/* Atomically create or replace a symlink. */ +/** + * Atomically create or replace a symlink. + */ void replaceSymlink(const Path & target, const Path & link, std::optional mtime = {}); @@ -196,24 +268,32 @@ void renameFile(const Path & src, const Path & dst); void moveFile(const Path & src, const Path & dst); -/* Wrappers arount read()/write() that read/write exactly the - requested number of bytes. */ +/** + * Wrappers arount read()/write() that read/write exactly the + * requested number of bytes. + */ void readFull(int fd, char * buf, size_t count); void writeFull(int fd, std::string_view s, bool allowInterrupts = true); MakeError(EndOfFile, Error); -/* Read a file descriptor until EOF occurs. */ +/** + * Read a file descriptor until EOF occurs. + */ std::string drainFD(int fd, bool block = true, const size_t reserveSize=0); void drainFD(int fd, Sink & sink, bool block = true); -/* If cgroups are active, attempt to calculate the number of CPUs available. - If cgroups are unavailable or if cpu.max is set to "max", return 0. */ +/** + * If cgroups are active, attempt to calculate the number of CPUs available. + * If cgroups are unavailable or if cpu.max is set to "max", return 0. + */ unsigned int getMaxCPU(); -/* Automatic cleanup of resources. */ +/** + * Automatic cleanup of resources. + */ class AutoDelete @@ -251,11 +331,15 @@ public: }; -/* Create a temporary directory. */ +/** + * Create a temporary directory. + */ Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); -/* Create a temporary file, returning a file handle and its path. */ +/** + * Create a temporary file, returning a file handle and its path. + */ std::pair createTempFile(const Path & prefix = "nix"); @@ -298,27 +382,36 @@ public: }; -/* Kill all processes running under the specified uid by sending them - a SIGKILL. */ +/** + * Kill all processes running under the specified uid by sending them + * a SIGKILL. + */ void killUser(uid_t uid); -/* Fork a process that runs the given function, and return the child - pid to the caller. */ +/** + * Fork a process that runs the given function, and return the child + * pid to the caller. + */ struct ProcessOptions { std::string errorPrefix = ""; bool dieWithParent = true; bool runExitHandlers = false; bool allowVfork = false; - int cloneFlags = 0; // use clone() with the specified flags (Linux only) + /** + * use clone() with the specified flags (Linux only) + */ + int cloneFlags = 0; }; pid_t startProcess(std::function fun, const ProcessOptions & options = ProcessOptions()); -/* Run a program and return its stdout in a string (i.e., like the - shell backtick operator). */ +/** + * Run a program and return its stdout in a string (i.e., like the + * shell backtick operator). + */ std::string runProgram(Path program, bool searchPath = false, const Strings & args = Strings(), const std::optional & input = {}); @@ -343,25 +436,35 @@ std::pair runProgram(RunOptions && options); void runProgram2(const RunOptions & options); -/* Change the stack size. */ +/** + * Change the stack size. + */ void setStackSize(size_t stackSize); -/* Restore the original inherited Unix process context (such as signal - masks, stack size). */ +/** + * Restore the original inherited Unix process context (such as signal + * masks, stack size). + */ void restoreProcessContext(bool restoreMounts = true); -/* Save the current mount namespace. Ignored if called more than - once. */ +/** + * Save the current mount namespace. Ignored if called more than + * once. + */ void saveMountNamespace(); -/* Restore the mount namespace saved by saveMountNamespace(). Ignored - if saveMountNamespace() was never called. */ +/** + * Restore the mount namespace saved by saveMountNamespace(). Ignored + * if saveMountNamespace() was never called. + */ void restoreMountNamespace(); -/* Cause this thread to not share any FS attributes with the main - thread, because this causes setns() in restoreMountNamespace() to - fail. */ +/** + * Cause this thread to not share any FS attributes with the main + * thread, because this causes setns() in restoreMountNamespace() to + * fail. + */ void unshareFilesystem(); @@ -376,16 +479,22 @@ public: { } }; -/* Convert a list of strings to a null-terminated vector of char - *'s. The result must not be accessed beyond the lifetime of the - list of strings. */ +/** + * Convert a list of strings to a null-terminated vector of `char + * *`s. The result must not be accessed beyond the lifetime of the + * list of strings. + */ std::vector stringsToCharPtrs(const Strings & ss); -/* Close all file descriptors except those listed in the given set. - Good practice in child processes. */ +/** + * Close all file descriptors except those listed in the given set. + * Good practice in child processes. + */ void closeMostFDs(const std::set & exceptions); -/* Set the close-on-exec flag for the given file descriptor. */ +/** + * Set the close-on-exec flag for the given file descriptor. + */ void closeOnExec(int fd); @@ -411,12 +520,16 @@ MakeError(Interrupted, BaseError); MakeError(FormatError, Error); -/* String tokenizer. */ +/** + * String tokenizer. + */ template C tokenizeString(std::string_view s, std::string_view separators = " \t\n\r"); -/* Concatenate the given strings with a separator between the - elements. */ +/** + * Concatenate the given strings with a separator between the + * elements. + */ template std::string concatStringsSep(const std::string_view sep, const C & ss) { @@ -441,7 +554,9 @@ auto concatStrings(Parts && ... parts) } -/* Add quotes around a collection of strings. */ +/** + * Add quotes around a collection of strings. + */ template Strings quoteStrings(const C & c) { Strings res; @@ -450,17 +565,23 @@ template Strings quoteStrings(const C & c) return res; } - -/* Remove trailing whitespace from a string. FIXME: return - std::string_view. */ +/** + * Remove trailing whitespace from a string. + * + * \todo return std::string_view. + */ std::string chomp(std::string_view s); -/* Remove whitespace from the start and end of a string. */ +/** + * Remove whitespace from the start and end of a string. + */ std::string trim(std::string_view s, std::string_view whitespace = " \n\r\t"); -/* Replace all occurrences of a string inside another string. */ +/** + * Replace all occurrences of a string inside another string. + */ std::string replaceStrings( std::string s, std::string_view from, @@ -470,14 +591,18 @@ std::string replaceStrings( std::string rewriteStrings(std::string s, const StringMap & rewrites); -/* Convert the exit status of a child as returned by wait() into an - error string. */ +/** + * Convert the exit status of a child as returned by wait() into an + * error string. + */ std::string statusToString(int status); bool statusOk(int status); -/* Parse a string into an integer. */ +/** + * Parse a string into an integer. + */ template std::optional string2Int(const std::string_view s) { @@ -490,8 +615,10 @@ std::optional string2Int(const std::string_view s) } } -/* Like string2Int(), but support an optional suffix 'K', 'M', 'G' or - 'T' denoting a binary unit prefix. */ +/** + * Like string2Int(), but support an optional suffix 'K', 'M', 'G' or + * 'T' denoting a binary unit prefix. + */ template N string2IntWithUnitPrefix(std::string_view s) { @@ -512,7 +639,9 @@ N string2IntWithUnitPrefix(std::string_view s) throw UsageError("'%s' is not an integer", s); } -/* Parse a string into a float. */ +/** + * Parse a string into a float. + */ template std::optional string2Float(const std::string_view s) { @@ -524,7 +653,9 @@ std::optional string2Float(const std::string_view s) } -/* Convert a little-endian integer to host order. */ +/** + * Convert a little-endian integer to host order. + */ template T readLittleEndian(unsigned char * p) { @@ -536,66 +667,90 @@ T readLittleEndian(unsigned char * p) } -/* Return true iff `s' starts with `prefix'. */ +/** + * @return true iff `s` starts with `prefix`. + */ bool hasPrefix(std::string_view s, std::string_view prefix); -/* Return true iff `s' ends in `suffix'. */ +/** + * @return true iff `s` ends in `suffix`. + */ bool hasSuffix(std::string_view s, std::string_view suffix); -/* Convert a string to lower case. */ +/** + * Convert a string to lower case. + */ std::string toLower(const std::string & s); -/* Escape a string as a shell word. */ +/** + * Escape a string as a shell word. + */ std::string shellEscape(const std::string_view s); -/* Exception handling in destructors: print an error message, then - ignore the exception. */ +/** + * Exception handling in destructors: print an error message, then + * ignore the exception. + */ void ignoreException(Verbosity lvl = lvlError); -/* Tree formatting. */ +/** + * Tree formatting. + */ constexpr char treeConn[] = "├───"; constexpr char treeLast[] = "└───"; constexpr char treeLine[] = "│ "; constexpr char treeNull[] = " "; -/* Determine whether ANSI escape sequences are appropriate for the - present output. */ +/** + * Determine whether ANSI escape sequences are appropriate for the + * present output. + */ bool shouldANSI(); -/* Truncate a string to 'width' printable characters. If 'filterAll' - is true, all ANSI escape sequences are filtered out. Otherwise, - some escape sequences (such as colour setting) are copied but not - included in the character count. Also, tabs are expanded to - spaces. */ +/** + * Truncate a string to 'width' printable characters. If 'filterAll' + * is true, all ANSI escape sequences are filtered out. Otherwise, + * some escape sequences (such as colour setting) are copied but not + * included in the character count. Also, tabs are expanded to + * spaces. + */ std::string filterANSIEscapes(std::string_view s, bool filterAll = false, unsigned int width = std::numeric_limits::max()); -/* Base64 encoding/decoding. */ +/** + * Base64 encoding/decoding. + */ std::string base64Encode(std::string_view s); std::string base64Decode(std::string_view s); -/* Remove common leading whitespace from the lines in the string - 's'. For example, if every line is indented by at least 3 spaces, - then we remove 3 spaces from the start of every line. */ +/** + * Remove common leading whitespace from the lines in the string + * 's'. For example, if every line is indented by at least 3 spaces, + * then we remove 3 spaces from the start of every line. + */ std::string stripIndentation(std::string_view s); -/* Get the prefix of 's' up to and excluding the next line break (LF - optionally preceded by CR), and the remainder following the line - break. */ +/** + * Get the prefix of 's' up to and excluding the next line break (LF + * optionally preceded by CR), and the remainder following the line + * break. + */ std::pair getLine(std::string_view s); -/* Get a value for the specified key from an associate container. */ +/** + * Get a value for the specified key from an associate container. + */ template const typename T::mapped_type * get(const T & map, const typename T::key_type & key) { @@ -612,7 +767,9 @@ typename T::mapped_type * get(T & map, const typename T::key_type & key) return &i->second; } -/* Get a value for the specified key from an associate container, or a default value if the key isn't present. */ +/** + * Get a value for the specified key from an associate container, or a default value if the key isn't present. + */ template const typename T::mapped_type & getOr(T & map, const typename T::key_type & key, @@ -623,7 +780,9 @@ const typename T::mapped_type & getOr(T & map, return i->second; } -/* Remove and return the first item from a container. */ +/** + * Remove and return the first item from a container. + */ template std::optional remove_begin(T & c) { @@ -635,7 +794,9 @@ std::optional remove_begin(T & c) } -/* Remove and return the first item from a container. */ +/** + * Remove and return the first item from a container. + */ template std::optional pop(T & c) { @@ -650,8 +811,10 @@ template class Callback; -/* Start a thread that handles various signals. Also block those signals - on the current thread (and thus any threads created by it). */ +/** + * Start a thread that handles various signals. Also block those signals + * on the current thread (and thus any threads created by it). + */ void startSignalHandlerThread(); struct InterruptCallback @@ -659,16 +822,20 @@ struct InterruptCallback virtual ~InterruptCallback() { }; }; -/* Register a function that gets called on SIGINT (in a non-signal - context). */ +/** + * Register a function that gets called on SIGINT (in a non-signal + * context). + */ std::unique_ptr createInterruptCallback( std::function callback); void triggerInterrupt(); -/* A RAII class that causes the current thread to receive SIGUSR1 when - the signal handler thread receives SIGINT. That is, this allows - SIGINT to be multiplexed to multiple threads. */ +/** + * A RAII class that causes the current thread to receive SIGUSR1 when + * the signal handler thread receives SIGINT. That is, this allows + * SIGINT to be multiplexed to multiple threads. + */ struct ReceiveInterrupts { pthread_t target; @@ -682,8 +849,10 @@ struct ReceiveInterrupts -/* A RAII helper that increments a counter on construction and - decrements it on destruction. */ +/** + * A RAII helper that increments a counter on construction and + * decrements it on destruction. + */ template struct MaintainCount { @@ -694,33 +863,50 @@ struct MaintainCount }; -/* Return the number of rows and columns of the terminal. */ +/** + * @return the number of rows and columns of the terminal. + */ std::pair getWindowSize(); -/* Used in various places. */ +/** + * Used in various places. + */ typedef std::function PathFilter; extern PathFilter defaultPathFilter; -/* Common initialisation performed in child processes. */ +/** + * Common initialisation performed in child processes. + */ void commonChildInit(); -/* Create a Unix domain socket. */ +/** + * Create a Unix domain socket. + */ AutoCloseFD createUnixDomainSocket(); -/* Create a Unix domain socket in listen mode. */ +/** + * Create a Unix domain socket in listen mode. + */ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); -/* Bind a Unix domain socket to a path. */ +/** + * Bind a Unix domain socket to a path. + */ void bind(int fd, const std::string & path); -/* Connect to a Unix domain socket. */ +/** + * Connect to a Unix domain socket. + */ void connect(int fd, const std::string & path); -// A Rust/Python-like enumerate() iterator adapter. -// Borrowed from http://reedbeta.com/blog/python-like-enumerate-in-cpp17. +/** + * A Rust/Python-like enumerate() iterator adapter. + * + * Borrowed from http://reedbeta.com/blog/python-like-enumerate-in-cpp17. + */ template ())), typename = decltype(std::end(std::declval()))> @@ -746,7 +932,9 @@ constexpr auto enumerate(T && iterable) } -// C++17 std::visit boilerplate +/** + * C++17 std::visit boilerplate + */ template struct overloaded : Ts... { using Ts::operator()...; }; template overloaded(Ts...) -> overloaded; @@ -754,8 +942,10 @@ template overloaded(Ts...) -> overloaded; std::string showBytes(uint64_t bytes); -/* Provide an addition operator between strings and string_views - inexplicably omitted from the standard library. */ +/** + * Provide an addition operator between strings and string_views + * inexplicably omitted from the standard library. + */ inline std::string operator + (const std::string & s1, std::string_view s2) { auto s = s1; diff --git a/src/libutil/xml-writer.hh b/src/libutil/xml-writer.hh index 4c91adee6..74f53b7ca 100644 --- a/src/libutil/xml-writer.hh +++ b/src/libutil/xml-writer.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index a4b3b1f96..bc7e7eb18 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -440,7 +440,7 @@ static void main_nix_build(int argc, char * * argv) shell = store->printStorePath(shellDrvOutputs.at("out").value()) + "/bin/bash"; } - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { auto resolvedDrv = drv.tryResolve(*store); assert(resolvedDrv && "Successfully resolved the derivation"); drv = *resolvedDrv; diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 338a7d18e..740737ffe 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -168,7 +168,8 @@ static int main_nix_channel(int argc, char ** argv) nixDefExpr = settings.useXDGBaseDirectories ? createNixStateDir() + "/defexpr" : home + "/.nix-defexpr"; // Figure out the name of the channels profile. - profile = profilesDir() + "/channels"; + profile = profilesDir() + "/channels"; + createDirs(dirOf(profile)); enum { cNone, diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 3a012638b..f076ffdb0 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1387,6 +1387,8 @@ static int main_nix_env(int argc, char * * argv) { Strings opFlags, opArgs; Operation op = 0; + std::string opName; + bool showHelp = false; RepairFlag repair = NoRepair; std::string file; @@ -1403,11 +1405,11 @@ static int main_nix_env(int argc, char * * argv) try { createDirs(globals.instSource.nixExprPath); replaceSymlink( - fmt("%s/profiles/per-user/%s/channels", settings.nixStateDir, getUserName()), + defaultChannelsDir(), globals.instSource.nixExprPath + "/channels"); if (getuid() != 0) replaceSymlink( - fmt("%s/profiles/per-user/root/channels", settings.nixStateDir), + rootChannelsDir(), globals.instSource.nixExprPath + "/channels_root"); } catch (Error &) { } } @@ -1426,37 +1428,59 @@ static int main_nix_env(int argc, char * * argv) Operation oldOp = op; if (*arg == "--help") - showManPage("nix-env"); + showHelp = true; else if (*arg == "--version") op = opVersion; - else if (*arg == "--install" || *arg == "-i") + else if (*arg == "--install" || *arg == "-i") { op = opInstall; + opName = "-install"; + } else if (*arg == "--force-name") // undocumented flag for nix-install-package globals.forceName = getArg(*arg, arg, end); - else if (*arg == "--uninstall" || *arg == "-e") + else if (*arg == "--uninstall" || *arg == "-e") { op = opUninstall; - else if (*arg == "--upgrade" || *arg == "-u") + opName = "-uninstall"; + } + else if (*arg == "--upgrade" || *arg == "-u") { op = opUpgrade; - else if (*arg == "--set-flag") + opName = "-upgrade"; + } + else if (*arg == "--set-flag") { op = opSetFlag; - else if (*arg == "--set") + opName = arg->substr(1); + } + else if (*arg == "--set") { op = opSet; - else if (*arg == "--query" || *arg == "-q") + opName = arg->substr(1); + } + else if (*arg == "--query" || *arg == "-q") { op = opQuery; + opName = "-query"; + } else if (*arg == "--profile" || *arg == "-p") globals.profile = absPath(getArg(*arg, arg, end)); else if (*arg == "--file" || *arg == "-f") file = getArg(*arg, arg, end); - else if (*arg == "--switch-profile" || *arg == "-S") + else if (*arg == "--switch-profile" || *arg == "-S") { op = opSwitchProfile; - else if (*arg == "--switch-generation" || *arg == "-G") + opName = "-switch-profile"; + } + else if (*arg == "--switch-generation" || *arg == "-G") { op = opSwitchGeneration; - else if (*arg == "--rollback") + opName = "-switch-generation"; + } + else if (*arg == "--rollback") { op = opRollback; - else if (*arg == "--list-generations") + opName = arg->substr(1); + } + else if (*arg == "--list-generations") { op = opListGenerations; - else if (*arg == "--delete-generations") + opName = arg->substr(1); + } + else if (*arg == "--delete-generations") { op = opDeleteGenerations; + opName = arg->substr(1); + } else if (*arg == "--dry-run") { printInfo("(dry run; not doing anything)"); globals.dryRun = true; @@ -1485,6 +1509,7 @@ static int main_nix_env(int argc, char * * argv) myArgs.parseCmdline(argvToStrings(argc, argv)); + if (showHelp) showManPage("nix-env" + opName); if (!op) throw UsageError("no operation specified"); auto store = openStore(); diff --git a/src/nix-env/user-env.hh b/src/nix-env/user-env.hh index 10646f713..af45d2d85 100644 --- a/src/nix-env/user-env.hh +++ b/src/nix-env/user-env.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "get-drvs.hh" diff --git a/src/nix-store/dotgraph.hh b/src/nix-store/dotgraph.hh index 73b8d06b9..4fd944080 100644 --- a/src/nix-store/dotgraph.hh +++ b/src/nix-store/dotgraph.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" diff --git a/src/nix-store/graphml.cc b/src/nix-store/graphml.cc index 425d61e53..439557658 100644 --- a/src/nix-store/graphml.cc +++ b/src/nix-store/graphml.cc @@ -57,7 +57,7 @@ void printGraphML(ref store, StorePathSet && roots) << "\n" - << "" + << "" << "" << "" << "\n"; diff --git a/src/nix-store/graphml.hh b/src/nix-store/graphml.hh index 78be8a367..bd3a4a37c 100644 --- a/src/nix-store/graphml.hh +++ b/src/nix-store/graphml.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 54479489f..7035e6a7b 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -277,17 +277,17 @@ static void printTree(const StorePath & path, static void opQuery(Strings opFlags, Strings opArgs) { enum QueryType - { qDefault, qOutputs, qRequisites, qReferences, qReferrers + { qOutputs, qRequisites, qReferences, qReferrers , qReferrersClosure, qDeriver, qBinding, qHash, qSize , qTree, qGraph, qGraphML, qResolve, qRoots }; - QueryType query = qDefault; + std::optional query; bool useOutput = false; bool includeOutputs = false; bool forceRealise = false; std::string bindingName; for (auto & i : opFlags) { - QueryType prev = query; + std::optional prev = query; if (i == "--outputs") query = qOutputs; else if (i == "--requisites" || i == "-R") query = qRequisites; else if (i == "--references") query = qReferences; @@ -312,15 +312,15 @@ static void opQuery(Strings opFlags, Strings opArgs) else if (i == "--force-realise" || i == "--force-realize" || i == "-f") forceRealise = true; else if (i == "--include-outputs") includeOutputs = true; else throw UsageError("unknown flag '%1%'", i); - if (prev != qDefault && prev != query) + if (prev && prev != query) throw UsageError("query type '%1%' conflicts with earlier flag", i); } - if (query == qDefault) query = qOutputs; + if (!query) query = qOutputs; RunPager pager; - switch (query) { + switch (*query) { case qOutputs: { for (auto & i : opArgs) { @@ -1024,62 +1024,104 @@ static int main_nix_store(int argc, char * * argv) Strings opFlags, opArgs; Operation op = 0; bool readFromStdIn = false; + std::string opName; + bool showHelp = false; parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { Operation oldOp = op; if (*arg == "--help") - showManPage("nix-store"); + showHelp = true; else if (*arg == "--version") op = opVersion; - else if (*arg == "--realise" || *arg == "--realize" || *arg == "-r") + else if (*arg == "--realise" || *arg == "--realize" || *arg == "-r") { op = opRealise; - else if (*arg == "--add" || *arg == "-A") + opName = "-realise"; + } + else if (*arg == "--add" || *arg == "-A"){ op = opAdd; - else if (*arg == "--add-fixed") + opName = "-add"; + } + else if (*arg == "--add-fixed") { op = opAddFixed; + opName = arg->substr(1); + } else if (*arg == "--print-fixed-path") op = opPrintFixedPath; - else if (*arg == "--delete") + else if (*arg == "--delete") { op = opDelete; - else if (*arg == "--query" || *arg == "-q") + opName = arg->substr(1); + } + else if (*arg == "--query" || *arg == "-q") { op = opQuery; - else if (*arg == "--print-env") + opName = "-query"; + } + else if (*arg == "--print-env") { op = opPrintEnv; - else if (*arg == "--read-log" || *arg == "-l") + opName = arg->substr(1); + } + else if (*arg == "--read-log" || *arg == "-l") { op = opReadLog; - else if (*arg == "--dump-db") + opName = "-read-log"; + } + else if (*arg == "--dump-db") { op = opDumpDB; - else if (*arg == "--load-db") + opName = arg->substr(1); + } + else if (*arg == "--load-db") { op = opLoadDB; + opName = arg->substr(1); + } else if (*arg == "--register-validity") op = opRegisterValidity; else if (*arg == "--check-validity") op = opCheckValidity; - else if (*arg == "--gc") + else if (*arg == "--gc") { op = opGC; - else if (*arg == "--dump") + opName = arg->substr(1); + } + else if (*arg == "--dump") { op = opDump; - else if (*arg == "--restore") + opName = arg->substr(1); + } + else if (*arg == "--restore") { op = opRestore; - else if (*arg == "--export") + opName = arg->substr(1); + } + else if (*arg == "--export") { op = opExport; - else if (*arg == "--import") + opName = arg->substr(1); + } + else if (*arg == "--import") { op = opImport; + opName = arg->substr(1); + } else if (*arg == "--init") op = opInit; - else if (*arg == "--verify") + else if (*arg == "--verify") { op = opVerify; - else if (*arg == "--verify-path") + opName = arg->substr(1); + } + else if (*arg == "--verify-path") { op = opVerifyPath; - else if (*arg == "--repair-path") + opName = arg->substr(1); + } + else if (*arg == "--repair-path") { op = opRepairPath; - else if (*arg == "--optimise" || *arg == "--optimize") + opName = arg->substr(1); + } + else if (*arg == "--optimise" || *arg == "--optimize") { op = opOptimise; - else if (*arg == "--serve") + opName = "-optimise"; + } + else if (*arg == "--serve") { op = opServe; - else if (*arg == "--generate-binary-cache-key") + opName = arg->substr(1); + } + else if (*arg == "--generate-binary-cache-key") { op = opGenerateBinaryCacheKey; + opName = arg->substr(1); + } else if (*arg == "--add-root") gcRoot = absPath(getArg(*arg, arg, end)); else if (*arg == "--stdin" && !isatty(STDIN_FILENO)) @@ -1109,6 +1151,7 @@ static int main_nix_store(int argc, char * * argv) return true; }); + if (showHelp) showManPage("nix-store" + opName); if (!op) throw UsageError("no operation specified"); if (op != opDump && op != opRestore) /* !!! hack */ diff --git a/src/nix/app.cc b/src/nix/app.cc index bfd75e278..fd4569bb4 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -1,5 +1,6 @@ #include "installables.hh" #include "installable-derived-path.hh" +#include "installable-value.hh" #include "store-api.hh" #include "eval-inline.hh" #include "eval-cache.hh" @@ -40,7 +41,7 @@ std::string resolveString( return rewriteStrings(toResolve, rewrites); } -UnresolvedApp Installable::toApp(EvalState & state) +UnresolvedApp InstallableValue::toApp(EvalState & state) { auto cursor = getCursor(state); auto attrPath = cursor->getAttrPath(); diff --git a/src/nix/build.cc b/src/nix/build.cc index bca20e97c..4e133e288 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -1,4 +1,3 @@ -#include "eval.hh" #include "command.hh" #include "common-args.hh" #include "shared.hh" diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 973bbd423..57c355f0c 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -1,14 +1,15 @@ -#include "command.hh" #include "installable-flake.hh" +#include "command-installable-value.hh" #include "common-args.hh" #include "shared.hh" #include "store-api.hh" #include "local-fs-store.hh" #include "fs-accessor.hh" +#include "eval-inline.hh" using namespace nix; -struct CmdBundle : InstallableCommand +struct CmdBundle : InstallableValueCommand { std::string bundler = "github:NixOS/bundlers"; std::optional outLink; @@ -70,7 +71,7 @@ struct CmdBundle : InstallableCommand return res; } - void run(ref store, ref installable) override + void run(ref store, ref installable) override { auto evalState = getEvalState(); diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 7e4a7ba86..7ae7b4ea6 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -1,3 +1,5 @@ +///@file + #include "command.hh" #include "shared.hh" #include "local-store.hh" @@ -34,6 +36,19 @@ using namespace nix; using namespace nix::daemon; +/** + * Settings related to authenticating clients for the Nix daemon. + * + * For pipes we have little good information about the client side, but + * for Unix domain sockets we do. So currently these options implemented + * mandatory access control based on user names and group names (looked + * up and translated to UID/GIDs in the CLI process that runs the code + * in this file). + * + * No code outside of this file knows about these settings (this is not + * exposed in a header); all authentication and authorization happens in + * `daemon.cc`. + */ struct AuthorizationSettings : Config { Setting trustedUsers{ @@ -54,7 +69,9 @@ struct AuthorizationSettings : Config { > directories that are otherwise inacessible to them. )"}; - /* ?Who we trust to use the daemon in safe ways */ + /** + * Who we trust to use the daemon in safe ways + */ Setting allowedUsers{ this, {"*"}, "allowed-users", R"( @@ -112,8 +129,36 @@ static void setSigChldAction(bool autoReap) throw SysError("setting SIGCHLD handler"); } +/** + * @return Is the given user a member of this group? + * + * @param user User specified by username. + * + * @param group Group the user might be a member of. + */ +static bool matchUser(std::string_view user, const struct group & gr) +{ + for (char * * mem = gr.gr_mem; *mem; mem++) + if (user == std::string_view(*mem)) return true; + return false; +} -bool matchUser(const std::string & user, const std::string & group, const Strings & users) + +/** + * Does the given user (specified by user name and primary group name) + * match the given user/group whitelist? + * + * If the list allows all users: Yes. + * + * If the username is in the set: Yes. + * + * If the groupname is in the set: Yes. + * + * If the user is in another group which is in the set: yes. + * + * Otherwise: No. + */ +static bool matchUser(const std::string & user, const std::string & group, const Strings & users) { if (find(users.begin(), users.end(), "*") != users.end()) return true; @@ -126,8 +171,7 @@ bool matchUser(const std::string & user, const std::string & group, const String if (group == i.substr(1)) return true; struct group * gr = getgrnam(i.c_str() + 1); if (!gr) continue; - for (char * * mem = gr->gr_mem; *mem; mem++) - if (user == std::string(*mem)) return true; + if (matchUser(user, *gr)) return true; } return false; @@ -145,7 +189,9 @@ struct PeerInfo }; -// Get the identity of the caller, if possible. +/** + * Get the identity of the caller, if possible. + */ static PeerInfo getPeerInfo(int remote) { PeerInfo peer = { false, 0, false, 0, false, 0 }; @@ -179,6 +225,9 @@ static PeerInfo getPeerInfo(int remote) #define SD_LISTEN_FDS_START 3 +/** + * Open a store without a path info cache. + */ static ref openUncachedStore() { Store::Params params; // FIXME: get params from somewhere @@ -187,7 +236,44 @@ static ref openUncachedStore() return openStore(settings.storeUri, params); } +/** + * Authenticate a potential client + * + * @param peer Information about other end of the connection, the client which + * wants to communicate with us. + * + * @return A pair of a `TrustedFlag`, whether the potential client is trusted, + * and the name of the user (useful for printing messages). + * + * If the potential client is not allowed to talk to us, we throw an `Error`. + */ +static std::pair authPeer(const PeerInfo & peer) +{ + TrustedFlag trusted = NotTrusted; + struct passwd * pw = peer.uidKnown ? getpwuid(peer.uid) : 0; + std::string user = pw ? pw->pw_name : std::to_string(peer.uid); + + struct group * gr = peer.gidKnown ? getgrgid(peer.gid) : 0; + std::string group = gr ? gr->gr_name : std::to_string(peer.gid); + + const Strings & trustedUsers = authorizationSettings.trustedUsers; + const Strings & allowedUsers = authorizationSettings.allowedUsers; + + if (matchUser(user, group, trustedUsers)) + trusted = Trusted; + + if ((!trusted && !matchUser(user, group, allowedUsers)) || group == settings.buildUsersGroup) + throw Error("user '%1%' is not allowed to connect to the Nix daemon", user); + + return { trusted, std::move(user) }; +} + + +/** + * Run a server. The loop opens a socket and accepts new connections from that + * socket. + */ static void daemonLoop() { if (chdir("/") == -1) @@ -231,23 +317,9 @@ static void daemonLoop() closeOnExec(remote.get()); - TrustedFlag trusted = NotTrusted; PeerInfo peer = getPeerInfo(remote.get()); - - struct passwd * pw = peer.uidKnown ? getpwuid(peer.uid) : 0; - std::string user = pw ? pw->pw_name : std::to_string(peer.uid); - - struct group * gr = peer.gidKnown ? getgrgid(peer.gid) : 0; - std::string group = gr ? gr->gr_name : std::to_string(peer.gid); - - Strings trustedUsers = authorizationSettings.trustedUsers; - Strings allowedUsers = authorizationSettings.allowedUsers; - - if (matchUser(user, group, trustedUsers)) - trusted = Trusted; - - if ((!trusted && !matchUser(user, group, allowedUsers)) || group == settings.buildUsersGroup) - throw Error("user '%1%' is not allowed to connect to the Nix daemon", user); + auto [_trusted, user] = authPeer(peer); + auto trusted = _trusted; printInfo((std::string) "accepted connection from pid %1%, user %2%" + (trusted ? " (trusted)" : ""), peer.pidKnown ? std::to_string(peer.pid) : "", @@ -294,45 +366,74 @@ static void daemonLoop() } } +/** + * Forward a standard IO connection to the given remote store. + * + * We just act as a middleman blindly ferry output between the standard + * input/output and the remote store connection, not processing anything. + * + * Loops until standard input disconnects, or an error is encountered. + */ +static void forwardStdioConnection(RemoteStore & store) { + auto conn = store.openConnectionWrapper(); + int from = conn->from.fd; + int to = conn->to.fd; + + auto nfds = std::max(from, STDIN_FILENO) + 1; + while (true) { + fd_set fds; + FD_ZERO(&fds); + FD_SET(from, &fds); + FD_SET(STDIN_FILENO, &fds); + if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1) + throw SysError("waiting for data from client or server"); + if (FD_ISSET(from, &fds)) { + auto res = splice(from, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE); + if (res == -1) + throw SysError("splicing data from daemon socket to stdout"); + else if (res == 0) + throw EndOfFile("unexpected EOF from daemon socket"); + } + if (FD_ISSET(STDIN_FILENO, &fds)) { + auto res = splice(STDIN_FILENO, nullptr, to, nullptr, SSIZE_MAX, SPLICE_F_MOVE); + if (res == -1) + throw SysError("splicing data from stdin to daemon socket"); + else if (res == 0) + return; + } + } +} + +/** + * Process a client connecting to us via standard input/output + * + * Unlike `forwardStdioConnection()` we do process commands ourselves in + * this case, not delegating to another daemon. + * + * @note `Trusted` is unconditionally passed because in this mode we + * blindly trust the standard streams. Limiting access to those is + * explicitly not `nix-daemon`'s responsibility. + */ +static void processStdioConnection(ref store) +{ + FdSource from(STDIN_FILENO); + FdSink to(STDOUT_FILENO); + processConnection(store, from, to, Trusted, NotRecursive); +} + +/** + * Entry point shared between the new CLI `nix daemon` and old CLI + * `nix-daemon`. + */ static void runDaemon(bool stdio) { if (stdio) { - if (auto store = openUncachedStore().dynamic_pointer_cast()) { - auto conn = store->openConnectionWrapper(); - int from = conn->from.fd; - int to = conn->to.fd; + auto store = openUncachedStore(); - auto nfds = std::max(from, STDIN_FILENO) + 1; - while (true) { - fd_set fds; - FD_ZERO(&fds); - FD_SET(from, &fds); - FD_SET(STDIN_FILENO, &fds); - if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1) - throw SysError("waiting for data from client or server"); - if (FD_ISSET(from, &fds)) { - auto res = splice(from, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE); - if (res == -1) - throw SysError("splicing data from daemon socket to stdout"); - else if (res == 0) - throw EndOfFile("unexpected EOF from daemon socket"); - } - if (FD_ISSET(STDIN_FILENO, &fds)) { - auto res = splice(STDIN_FILENO, nullptr, to, nullptr, SSIZE_MAX, SPLICE_F_MOVE); - if (res == -1) - throw SysError("splicing data from stdin to daemon socket"); - else if (res == 0) - return; - } - } - } else { - FdSource from(STDIN_FILENO); - FdSink to(STDOUT_FILENO); - /* Auth hook is empty because in this mode we blindly trust the - standard streams. Limiting access to those is explicitly - not `nix-daemon`'s responsibility. */ - processConnection(openUncachedStore(), from, to, Trusted, NotRecursive); - } + if (auto remoteStore = store.dynamic_pointer_cast()) + forwardStdioConnection(*remoteStore); + else + processStdioConnection(store); } else daemonLoop(); } diff --git a/src/nix/derivation-add.cc b/src/nix/derivation-add.cc new file mode 100644 index 000000000..4d91d4538 --- /dev/null +++ b/src/nix/derivation-add.cc @@ -0,0 +1,45 @@ +// FIXME: rename to 'nix plan add' or 'nix derivation add'? + +#include "command.hh" +#include "common-args.hh" +#include "store-api.hh" +#include "archive.hh" +#include "derivations.hh" +#include + +using namespace nix; +using json = nlohmann::json; + +struct CmdAddDerivation : MixDryRun, StoreCommand +{ + std::string description() override + { + return "Add a store derivation"; + } + + std::string doc() override + { + return + #include "derivation-add.md" + ; + } + + Category category() override { return catUtility; } + + void run(ref store) override + { + auto json = nlohmann::json::parse(drainFD(STDIN_FILENO)); + + auto drv = Derivation::fromJSON(*store, json); + + auto drvPath = writeDerivation(*store, drv, NoRepair, /* read only */ dryRun); + + drv.checkInvariants(*store, drvPath); + + writeDerivation(*store, drv, NoRepair, dryRun); + + logger->cout("%s", store->printStorePath(drvPath)); + } +}; + +static auto rCmdAddDerivation = registerCommand2({"derivation", "add"}); diff --git a/src/nix/derivation-add.md b/src/nix/derivation-add.md new file mode 100644 index 000000000..f116681ab --- /dev/null +++ b/src/nix/derivation-add.md @@ -0,0 +1,18 @@ +R""( + +# Description + +This command reads from standard input a JSON representation of a +[store derivation] to which an [*installable*](./nix.md#installables) evaluates. + +Store derivations are used internally by Nix. They are store paths with +extension `.drv` that represent the build-time dependency graph to which +a Nix expression evaluates. + +[store derivation]: ../../glossary.md#gloss-store-derivation + +The JSON format is documented under the [`derivation show`] command. + +[`derivation show`]: ./nix3-derivation-show.md + +)"" diff --git a/src/nix/show-derivation.cc b/src/nix/derivation-show.cc similarity index 87% rename from src/nix/show-derivation.cc rename to src/nix/derivation-show.cc index 4a406ae08..bf637246d 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/derivation-show.cc @@ -1,5 +1,5 @@ // FIXME: integrate this with nix path-info? -// FIXME: rename to 'nix store show-derivation' or 'nix debug show-derivation'? +// FIXME: rename to 'nix store derivation show' or 'nix debug derivation show'? #include "command.hh" #include "common-args.hh" @@ -33,7 +33,7 @@ struct CmdShowDerivation : InstallablesCommand std::string doc() override { return - #include "show-derivation.md" + #include "derivation-show.md" ; } @@ -61,4 +61,4 @@ struct CmdShowDerivation : InstallablesCommand } }; -static auto rCmdShowDerivation = registerCommand("show-derivation"); +static auto rCmdShowDerivation = registerCommand2({"derivation", "show"}); diff --git a/src/nix/show-derivation.md b/src/nix/derivation-show.md similarity index 86% rename from src/nix/show-derivation.md rename to src/nix/derivation-show.md index 1d37c6f5a..1296e2885 100644 --- a/src/nix/show-derivation.md +++ b/src/nix/derivation-show.md @@ -8,7 +8,7 @@ R""( [store derivation]: ../../glossary.md#gloss-store-derivation ```console - # nix show-derivation nixpkgs#hello + # nix derivation show nixpkgs#hello { "/nix/store/s6rn4jz1sin56rf4qj5b5v8jxjm32hlk-hello-2.10.drv": { … @@ -20,14 +20,14 @@ R""( NixOS system: ```console - # nix show-derivation -r /run/current-system + # nix derivation show -r /run/current-system ``` * Print all files fetched using `fetchurl` by Firefox's dependency graph: ```console - # nix show-derivation -r nixpkgs#firefox \ + # nix derivation show -r nixpkgs#firefox \ | jq -r '.[] | select(.outputs.out.hash and .env.urls) | .env.urls' \ | uniq | sort ``` @@ -39,10 +39,11 @@ R""( # Description This command prints on standard output a JSON representation of the -[store derivation]s to which [*installables*](./nix.md#installables) evaluate. Store derivations -are used internally by Nix. They are store paths with extension `.drv` -that represent the build-time dependency graph to which a Nix -expression evaluates. +[store derivation]s to which [*installables*](./nix.md#installables) evaluate. + +Store derivations are used internally by Nix. They are store paths with +extension `.drv` that represent the build-time dependency graph to which +a Nix expression evaluates. By default, this command only shows top-level derivations, but with `--recursive`, it also shows their dependencies. @@ -51,6 +52,9 @@ The JSON output is a JSON object whose keys are the store paths of the derivations, and whose values are a JSON object with the following fields: +* `name`: The name of the derivation. This is used when calculating the + store paths of the derivation's outputs. + * `outputs`: Information about the output paths of the derivation. This is a JSON object with one member per output, where the key is the output name and the value is a JSON object with these diff --git a/src/nix/derivation.cc b/src/nix/derivation.cc new file mode 100644 index 000000000..cd3975a4f --- /dev/null +++ b/src/nix/derivation.cc @@ -0,0 +1,25 @@ +#include "command.hh" + +using namespace nix; + +struct CmdDerivation : virtual NixMultiCommand +{ + CmdDerivation() : MultiCommand(RegisterCommand::getCommandsFor({"derivation"})) + { } + + std::string description() override + { + return "Work with derivations, Nix's notion of a build plan."; + } + + Category category() override { return catUtility; } + + void run() override + { + if (!command) + throw UsageError("'nix derivation' requires a sub-command."); + command->second->run(); + } +}; + +static auto rCmdDerivation = registerCommand("derivation"); diff --git a/src/nix/describe-stores.cc b/src/nix/describe-stores.cc deleted file mode 100644 index eafcedd1f..000000000 --- a/src/nix/describe-stores.cc +++ /dev/null @@ -1,44 +0,0 @@ -#include "command.hh" -#include "common-args.hh" -#include "shared.hh" -#include "store-api.hh" - -#include - -using namespace nix; - -struct CmdDescribeStores : Command, MixJSON -{ - std::string description() override - { - return "show registered store types and their available options"; - } - - Category category() override { return catUtility; } - - void run() override - { - auto res = nlohmann::json::object(); - for (auto & implem : *Implementations::registered) { - auto storeConfig = implem.getConfig(); - auto storeName = storeConfig->name(); - res[storeName] = storeConfig->toJSON(); - } - if (json) { - logger->cout("%s", res); - } else { - for (auto & [storeName, storeConfig] : res.items()) { - std::cout << "## " << storeName << std::endl << std::endl; - for (auto & [optionName, optionDesc] : storeConfig.items()) { - std::cout << "### " << optionName << std::endl << std::endl; - std::cout << optionDesc["description"].get() << std::endl; - std::cout << "default: " << optionDesc["defaultValue"] << std::endl <("describe-stores"); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index f06ade008..9e2dcff61 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -1,6 +1,6 @@ #include "eval.hh" -#include "command.hh" #include "installable-flake.hh" +#include "command-installable-value.hh" #include "common-args.hh" #include "shared.hh" #include "store-api.hh" @@ -208,7 +208,7 @@ static StorePath getDerivationEnvironment(ref store, ref evalStore drv.name += "-env"; drv.env.emplace("name", drv.name); drv.inputSrcs.insert(std::move(getEnvShPath)); - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { for (auto & output : drv.outputs) { output.second = DerivationOutput::Deferred {}, drv.env[output.first] = hashPlaceholder(output.first); @@ -252,7 +252,7 @@ static StorePath getDerivationEnvironment(ref store, ref evalStore throw Error("get-env.sh failed to produce an environment"); } -struct Common : InstallableCommand, MixProfile +struct Common : InstallableValueCommand, MixProfile { std::set ignoreVars{ "BASHOPTS", @@ -374,7 +374,7 @@ struct Common : InstallableCommand, MixProfile return res; } - StorePath getShellOutPath(ref store, ref installable) + StorePath getShellOutPath(ref store, ref installable) { auto path = installable->getStorePath(); if (path && hasSuffix(path->to_string(), "-env")) @@ -393,7 +393,7 @@ struct Common : InstallableCommand, MixProfile } std::pair - getBuildEnvironment(ref store, ref installable) + getBuildEnvironment(ref store, ref installable) { auto shellOutPath = getShellOutPath(store, installable); @@ -481,7 +481,7 @@ struct CmdDevelop : Common, MixEnvironment ; } - void run(ref store, ref installable) override + void run(ref store, ref installable) override { auto [buildEnvironment, gcroot] = getBuildEnvironment(store, installable); @@ -605,7 +605,7 @@ struct CmdPrintDevEnv : Common, MixJSON Category category() override { return catUtility; } - void run(ref store, ref installable) override + void run(ref store, ref installable) override { auto buildEnvironment = getBuildEnvironment(store, installable).first; diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc index 7da4549a1..1aa6831d3 100644 --- a/src/nix/doctor.cc +++ b/src/nix/doctor.cc @@ -33,12 +33,24 @@ bool checkFail(const std::string & msg) { return false; } +void checkInfo(const std::string & msg) { + notice(ANSI_BLUE "[INFO] " ANSI_NORMAL + msg); +} + } struct CmdDoctor : StoreCommand { bool success = true; + /** + * This command is stable before the others + */ + std::optional experimentalFeature() override + { + return std::nullopt; + } + std::string description() override { return "check your system for potential problems and print a PASS or FAIL for each check"; @@ -55,6 +67,7 @@ struct CmdDoctor : StoreCommand success &= checkProfileRoots(store); } success &= checkStoreProtocol(store->getProtocol()); + checkTrustedUser(store); if (!success) throw Exit(2); @@ -130,6 +143,14 @@ struct CmdDoctor : StoreCommand return checkPass("Client protocol matches store protocol."); } + + void checkTrustedUser(ref store) + { + std::string_view trusted = store->isTrustedClient() + ? "trusted" + : "not trusted"; + checkInfo(fmt("You are %s by store uri: %s", trusted, store->getUri())); + } }; static auto rCmdDoctor = registerCommand("doctor"); diff --git a/src/nix/edit.cc b/src/nix/edit.cc index c46c1c23c..66629fab0 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -1,4 +1,4 @@ -#include "command.hh" +#include "command-installable-value.hh" #include "shared.hh" #include "eval.hh" #include "attr-path.hh" @@ -9,7 +9,7 @@ using namespace nix; -struct CmdEdit : InstallableCommand +struct CmdEdit : InstallableValueCommand { std::string description() override { @@ -25,7 +25,7 @@ struct CmdEdit : InstallableCommand Category category() override { return catSecondary; } - void run(ref store, ref installable) override + void run(ref store, ref installable) override { auto state = getEvalState(); diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 6c2b60427..43db5150c 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -1,4 +1,4 @@ -#include "command.hh" +#include "command-installable-value.hh" #include "common-args.hh" #include "shared.hh" #include "store-api.hh" @@ -11,13 +11,13 @@ using namespace nix; -struct CmdEval : MixJSON, InstallableCommand, MixReadOnlyOption +struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption { bool raw = false; std::optional apply; std::optional writeTo; - CmdEval() : InstallableCommand() + CmdEval() : InstallableValueCommand() { addFlag({ .longName = "raw", @@ -54,7 +54,7 @@ struct CmdEval : MixJSON, InstallableCommand, MixReadOnlyOption Category category() override { return catSecondary; } - void run(ref store, ref installable) override + void run(ref store, ref installable) override { if (raw && json) throw UsageError("--raw and --json are mutually exclusive"); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 0a6616396..cd4ee5921 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1026,36 +1026,43 @@ struct CmdFlakeShow : FlakeCommand, MixJSON auto visitor2 = visitor.getAttr(attrName); - if ((attrPathS[0] == "apps" - || attrPathS[0] == "checks" - || attrPathS[0] == "devShells" - || attrPathS[0] == "legacyPackages" - || attrPathS[0] == "packages") - && (attrPathS.size() == 1 || attrPathS.size() == 2)) { - for (const auto &subAttr : visitor2->getAttrs()) { - if (hasContent(*visitor2, attrPath2, subAttr)) { - return true; + try { + if ((attrPathS[0] == "apps" + || attrPathS[0] == "checks" + || attrPathS[0] == "devShells" + || attrPathS[0] == "legacyPackages" + || attrPathS[0] == "packages") + && (attrPathS.size() == 1 || attrPathS.size() == 2)) { + for (const auto &subAttr : visitor2->getAttrs()) { + if (hasContent(*visitor2, attrPath2, subAttr)) { + return true; + } } + return false; } - return false; - } - if ((attrPathS.size() == 1) - && (attrPathS[0] == "formatter" - || attrPathS[0] == "nixosConfigurations" - || attrPathS[0] == "nixosModules" - || attrPathS[0] == "overlays" - )) { - for (const auto &subAttr : visitor2->getAttrs()) { - if (hasContent(*visitor2, attrPath2, subAttr)) { - return true; + if ((attrPathS.size() == 1) + && (attrPathS[0] == "formatter" + || attrPathS[0] == "nixosConfigurations" + || attrPathS[0] == "nixosModules" + || attrPathS[0] == "overlays" + )) { + for (const auto &subAttr : visitor2->getAttrs()) { + if (hasContent(*visitor2, attrPath2, subAttr)) { + return true; + } } + return false; } - return false; - } - // If we don't recognize it, it's probably content - return true; + // If we don't recognize it, it's probably content + return true; + } catch (EvalError & e) { + // Some attrs may contain errors, eg. legacyPackages of + // nixpkgs. We still want to recurse into it, instead of + // skipping it at all. + return true; + } }; std::functionsecond->run(); } }; diff --git a/src/nix/flake.md b/src/nix/flake.md index 8eaa41b96..d70f34eeb 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -221,11 +221,46 @@ Currently the `type` attribute can be one of the following: commit hash (`rev`). Note that unlike Git, GitHub allows fetching by commit hash without specifying a branch or tag. + You can also specify `host` as a parameter, to point to a custom GitHub + Enterprise server. + Some examples: * `github:edolstra/dwarffs` * `github:edolstra/dwarffs/unstable` * `github:edolstra/dwarffs/d3f2baba8f425779026c6ec04021b2e927f61e31` + * `github:internal/project?host=company-github.example.org` + +* `gitlab`: Similar to `github`, is a more efficient way to fetch + GitLab repositories. The following attributes are required: + + * `owner`: The owner of the repository. + + * `repo`: The name of the repository. + + Like `github`, these are downloaded as tarball archives. + + The URL syntax for `gitlab` flakes is: + + `gitlab:/(/)?(\?)?` + + `` works the same as `github`. Either a branch or tag name + (`ref`), or a commit hash (`rev`) can be specified. + + Since GitLab allows for self-hosting, you can specify `host` as + a parameter, to point to any instances other than `gitlab.com`. + + Some examples: + + * `gitlab:veloren/veloren` + * `gitlab:veloren/veloren/master` + * `gitlab:veloren/veloren/80a4d7f13492d916e47d6195be23acae8001985a` + * `gitlab:openldap/openldap?host=git.openldap.org` + + When accessing a project in a (nested) subgroup, make sure to URL-encode any + slashes, i.e. replace `/` with `%2F`: + + * `gitlab:veloren%2Fdev/rfcs` * `sourcehut`: Similar to `github`, is a more efficient way to fetch SourceHut repositories. The following attributes are required: @@ -317,6 +352,8 @@ The following attributes are supported in `flake.nix`: also contains some metadata about the inputs. These are: * `outPath`: The path in the Nix store of the flake's source tree. + This way, the attribute set can be passed to `import` as if it was a path, + as in the example above (`import nixpkgs`). * `rev`: The commit hash of the flake's repository, if applicable. diff --git a/src/nix/fmt.cc b/src/nix/fmt.cc index 6f6a4a632..c85eacded 100644 --- a/src/nix/fmt.cc +++ b/src/nix/fmt.cc @@ -1,4 +1,5 @@ #include "command.hh" +#include "installable-value.hh" #include "run.hh" using namespace nix; @@ -31,8 +32,9 @@ struct CmdFmt : SourceExprCommand { auto evalState = getEvalState(); auto evalStore = getEvalStore(); - auto installable = parseInstallable(store, "."); - auto app = installable->toApp(*evalState).resolve(evalStore, store); + auto installable_ = parseInstallable(store, "."); + auto & installable = InstallableValue::require(*installable_); + auto app = installable.toApp(*evalState).resolve(evalStore, store); Strings programArgs{app.program}; diff --git a/src/nix/help-stores.md b/src/nix/help-stores.md new file mode 100644 index 000000000..47ba9b94d --- /dev/null +++ b/src/nix/help-stores.md @@ -0,0 +1,46 @@ +R"( + +Nix supports different types of stores. These are described below. + +## Store URL format + +Stores are specified using a URL-like syntax. For example, the command + +```console +# nix path-info --store https://cache.nixos.org/ --json \ + /nix/store/a7gvj343m05j2s32xcnwr35v31ynlypr-coreutils-9.1 +``` + +fetches information about a store path in the HTTP binary cache +located at https://cache.nixos.org/, which is a type of store. + +Store URLs can specify **store settings** using URL query strings, +i.e. by appending `?name1=value1&name2=value2&...` to the URL. For +instance, + +``` +--store ssh://machine.example.org?ssh-key=/path/to/my/key +``` + +tells Nix to access the store on a remote machine via the SSH +protocol, using `/path/to/my/key` as the SSH private key. The +supported settings for each store type are documented below. + +The special store URL `auto` causes Nix to automatically select a +store as follows: + +* Use the [local store](#local-store) `/nix/store` if `/nix/var/nix` + is writable by the current user. + +* Otherwise, if `/nix/var/nix/daemon-socket/socket` exists, [connect + to the Nix daemon listening on that socket](#local-daemon-store). + +* Otherwise, on Linux only, use the [local chroot store](#local-store) + `~/.local/share/nix/root`, which will be created automatically if it + does not exist. + +* Otherwise, use the [local store](#local-store) `/nix/store`. + +@stores@ + +)" diff --git a/src/nix/main.cc b/src/nix/main.cc index 7b715f281..f943f77bb 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -54,17 +54,17 @@ static bool haveInternet() std::string programPath; -struct HelpRequested { }; - struct NixArgs : virtual MultiCommand, virtual MixCommonArgs { bool useNet = true; bool refresh = false; + bool helpRequested = false; bool showVersion = false; NixArgs() : MultiCommand(RegisterCommand::getCommandsFor({})), MixCommonArgs("nix") { categories.clear(); + categories[catHelp] = "Help commands"; categories[Command::catDefault] = "Main commands"; categories[catSecondary] = "Infrequently used commands"; categories[catUtility] = "Utility/scripting commands"; @@ -74,7 +74,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs .longName = "help", .description = "Show usage information.", .category = miscCategory, - .handler = {[&]() { throw HelpRequested(); }}, + .handler = {[this]() { this->helpRequested = true; }}, }); addFlag({ @@ -83,6 +83,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs .description = "Print full build logs on standard error.", .category = loggingCategory, .handler = {[&]() { logger->setPrintBuildLogs(true); }}, + .experimentalFeature = Xp::NixCommand, }); addFlag({ @@ -98,6 +99,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs .description = "Disable substituters and consider all previously downloaded files up-to-date.", .category = miscCategory, .handler = {[&]() { useNet = false; }}, + .experimentalFeature = Xp::NixCommand, }); addFlag({ @@ -105,6 +107,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs .description = "Consider all previously downloaded files out-of-date.", .category = miscCategory, .handler = {[&]() { refresh = true; }}, + .experimentalFeature = Xp::NixCommand, }); } @@ -124,6 +127,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs {"optimise-store", {"store", "optimise"}}, {"ping-store", {"store", "ping"}}, {"sign-paths", {"store", "sign"}}, + {"show-derivation", {"derivation", "show"}}, {"to-base16", {"hash", "to-base16"}}, {"to-base32", {"hash", "to-base32"}}, {"to-base64", {"hash", "to-base64"}}, @@ -164,11 +168,29 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs { commands = RegisterCommand::getCommandsFor({}); } + + std::string dumpCli() + { + auto res = nlohmann::json::object(); + + res["args"] = toJSON(); + + auto stores = nlohmann::json::object(); + for (auto & implem : *Implementations::registered) { + auto storeConfig = implem.getConfig(); + auto storeName = storeConfig->name(); + stores[storeName]["doc"] = storeConfig->doc(); + stores[storeName]["settings"] = storeConfig->toJSON(); + } + res["stores"] = std::move(stores); + + return res.dump(); + } }; /* Render the help for the specified subcommand to stdout using lowdown. */ -static void showHelp(std::vector subcommand, MultiCommand & toplevel) +static void showHelp(std::vector subcommand, NixArgs & toplevel) { auto mdName = subcommand.empty() ? "nix" : fmt("nix3-%s", concatStringsSep("-", subcommand)); @@ -189,11 +211,11 @@ static void showHelp(std::vector subcommand, MultiCommand & topleve , "/"), *vUtils); - auto attrs = state.buildBindings(16); - attrs.alloc("toplevel").mkString(toplevel.toJSON().dump()); + auto vDump = state.allocValue(); + vDump->mkString(toplevel.dumpCli()); auto vRes = state.allocValue(); - state.callFunction(*vGenerateManpage, state.allocValue()->mkAttrs(attrs), *vRes, noPos); + state.callFunction(*vGenerateManpage, *vDump, *vRes, noPos); auto attr = vRes->attrs->get(state.symbols.create(mdName + ".md")); if (!attr) @@ -205,6 +227,14 @@ static void showHelp(std::vector subcommand, MultiCommand & topleve std::cout << renderMarkdownToTerminal(markdown) << "\n"; } +static NixArgs & getNixArgs(Command & cmd) +{ + assert(cmd.parent); + MultiCommand * toplevel = cmd.parent; + while (toplevel->parent) toplevel = toplevel->parent; + return dynamic_cast(*toplevel); +} + struct CmdHelp : Command { std::vector subcommand; @@ -229,17 +259,43 @@ struct CmdHelp : Command ; } + Category category() override { return catHelp; } + void run() override { assert(parent); MultiCommand * toplevel = parent; while (toplevel->parent) toplevel = toplevel->parent; - showHelp(subcommand, *toplevel); + showHelp(subcommand, getNixArgs(*this)); } }; static auto rCmdHelp = registerCommand("help"); +struct CmdHelpStores : Command +{ + std::string description() override + { + return "show help about store types and their settings"; + } + + std::string doc() override + { + return + #include "help-stores.md" + ; + } + + Category category() override { return catHelp; } + + void run() override + { + showHelp({"help-stores"}, getNixArgs(*this)); + } +}; + +static auto rCmdHelpStores = registerCommand("help-stores"); + void mainWrapped(int argc, char * * argv) { savedArgv = argv; @@ -291,13 +347,16 @@ void mainWrapped(int argc, char * * argv) NixArgs args; - if (argc == 2 && std::string(argv[1]) == "__dump-args") { - logger->cout("%s", args.toJSON()); + if (argc == 2 && std::string(argv[1]) == "__dump-cli") { + logger->cout(args.dumpCli()); return; } if (argc == 2 && std::string(argv[1]) == "__dump-builtins") { - settings.experimentalFeatures = {Xp::Flakes, Xp::FetchClosure}; + experimentalFeatureSettings.experimentalFeatures = { + Xp::Flakes, + Xp::FetchClosure, + }; evalSettings.pureEval = false; EvalState state({}, openStore("dummy://")); auto res = nlohmann::json::object(); @@ -334,7 +393,11 @@ void mainWrapped(int argc, char * * argv) try { args.parseCmdline(argvToStrings(argc, argv)); - } catch (HelpRequested &) { + } catch (UsageError &) { + if (!args.helpRequested && !completions) throw; + } + + if (args.helpRequested) { std::vector subcommand; MultiCommand * command = &args; while (command) { @@ -346,8 +409,6 @@ void mainWrapped(int argc, char * * argv) } showHelp(subcommand, args); return; - } catch (UsageError &) { - if (!completions) throw; } if (completions) { @@ -363,10 +424,8 @@ void mainWrapped(int argc, char * * argv) if (!args.command) throw UsageError("no subcommand specified"); - if (args.command->first != "repl" - && args.command->first != "doctor" - && args.command->first != "upgrade-nix") - settings.requireExperimentalFeature(Xp::NixCommand); + experimentalFeatureSettings.require( + args.command->second->experimentalFeature()); if (args.useNet && !haveInternet()) { warn("you don't have Internet access; disabling some network-dependent features"); diff --git a/src/nix/nix.md b/src/nix/nix.md index 0a90fa6c9..e1865b31c 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -220,8 +220,7 @@ operate are determined as follows: # Nix stores -Most `nix` subcommands operate on a *Nix store*. - -TODO: list store types, options +Most `nix` subcommands operate on a *Nix store*. These are documented +in [`nix help-stores`](./nix3-help-stores.md). )"" diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc index 5c44510ab..ec450e8e0 100644 --- a/src/nix/ping-store.cc +++ b/src/nix/ping-store.cc @@ -28,15 +28,20 @@ struct CmdPingStore : StoreCommand, MixJSON store->connect(); if (auto version = store->getVersion()) notice("Version: %s", *version); + if (auto trusted = store->isTrustedClient()) + notice("Trusted: %s", *trusted); } else { nlohmann::json res; Finally printRes([&]() { logger->cout("%s", res); }); + res["url"] = store->getUri(); store->connect(); if (auto version = store->getVersion()) res["version"] = *version; + if (auto trusted = store->isTrustedClient()) + res["trusted"] = *trusted; } } }; diff --git a/src/nix/profile.cc b/src/nix/profile.cc index d72dd1a13..88d4a3ce5 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -254,13 +254,19 @@ struct ProfileManifest } }; -static std::map> +static std::map>> builtPathsPerInstallable( const std::vector, BuiltPathWithResult>> & builtPaths) { - std::map> res; + std::map>> res; for (auto & [installable, builtPath] : builtPaths) { - auto & r = res[&*installable]; + auto & r = res.insert({ + &*installable, + { + {}, + make_ref(), + } + }).first->second; /* Note that there could be conflicting info (e.g. meta.priority fields) if the installable returned multiple derivations. So pick one arbitrarily. FIXME: @@ -307,14 +313,16 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile for (auto & installable : installables) { ProfileElement element; - auto & [res, info] = builtPaths[&*installable]; + auto iter = builtPaths.find(&*installable); + if (iter == builtPaths.end()) continue; + auto & [res, info] = iter->second; - if (info.originalRef && info.resolvedRef && info.attrPath && info.extendedOutputsSpec) { + if (auto * info2 = dynamic_cast(&*info)) { element.source = ProfileElementSource { - .originalRef = *info.originalRef, - .resolvedRef = *info.resolvedRef, - .attrPath = *info.attrPath, - .outputs = *info.extendedOutputsSpec, + .originalRef = info2->flake.originalRef, + .resolvedRef = info2->flake.resolvedRef, + .attrPath = info2->value.attrPath, + .outputs = info2->value.extendedOutputsSpec, }; } @@ -323,7 +331,12 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile element.priority = priority ? *priority - : info.priority.value_or(defaultPriority); + : ({ + auto * info2 = dynamic_cast(&*info); + info2 + ? info2->value.priority.value_or(defaultPriority) + : defaultPriority; + }); element.updateStorePaths(getEvalStore(), store, res); @@ -541,19 +554,20 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf auto derivedPaths = installable->toDerivedPaths(); if (derivedPaths.empty()) continue; - auto & info = derivedPaths[0].info; + auto * infop = dynamic_cast(&*derivedPaths[0].info); + // `InstallableFlake` should use `ExtraPathInfoFlake`. + assert(infop); + auto & info = *infop; - assert(info.resolvedRef && info.attrPath); - - if (element.source->resolvedRef == info.resolvedRef) continue; + if (element.source->resolvedRef == info.flake.resolvedRef) continue; printInfo("upgrading '%s' from flake '%s' to '%s'", - element.source->attrPath, element.source->resolvedRef, *info.resolvedRef); + element.source->attrPath, element.source->resolvedRef, info.flake.resolvedRef); element.source = ProfileElementSource { .originalRef = installable->flakeRef, - .resolvedRef = *info.resolvedRef, - .attrPath = *info.attrPath, + .resolvedRef = info.flake.resolvedRef, + .attrPath = info.value.attrPath, .outputs = installable->extendedOutputsSpec, }; @@ -582,7 +596,10 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf for (size_t i = 0; i < installables.size(); ++i) { auto & installable = installables.at(i); auto & element = manifest.elements[indices.at(i)]; - element.updateStorePaths(getEvalStore(), store, builtPaths[&*installable].first); + element.updateStorePaths( + getEvalStore(), + store, + builtPaths.find(&*installable)->second.first); } updateProfile(manifest.build(store)); diff --git a/src/nix/profile.md b/src/nix/profile.md index 273e02280..bf61ef4b9 100644 --- a/src/nix/profile.md +++ b/src/nix/profile.md @@ -12,7 +12,7 @@ them to be rolled back easily. The default profile used by `nix profile` is `$HOME/.nix-profile`, which, if it does not exist, is created as a symlink to `/nix/var/nix/profiles/default` if Nix is invoked by the -`root` user, or `/nix/var/nix/profiles/per-user/`*username* otherwise. +`root` user, or `${XDG_STATE_HOME-$HOME/.local/state}/nix/profiles/profile` otherwise. You can specify another profile location using `--profile` *path*. @@ -24,11 +24,11 @@ the profile. In turn, *path*`-`*N* is a symlink to a path in the Nix store. For example: ```console -$ ls -l /nix/var/nix/profiles/per-user/alice/profile* -lrwxrwxrwx 1 alice users 14 Nov 25 14:35 /nix/var/nix/profiles/per-user/alice/profile -> profile-7-link -lrwxrwxrwx 1 alice users 51 Oct 28 16:18 /nix/var/nix/profiles/per-user/alice/profile-5-link -> /nix/store/q69xad13ghpf7ir87h0b2gd28lafjj1j-profile -lrwxrwxrwx 1 alice users 51 Oct 29 13:20 /nix/var/nix/profiles/per-user/alice/profile-6-link -> /nix/store/6bvhpysd7vwz7k3b0pndn7ifi5xr32dg-profile -lrwxrwxrwx 1 alice users 51 Nov 25 14:35 /nix/var/nix/profiles/per-user/alice/profile-7-link -> /nix/store/mp0x6xnsg0b8qhswy6riqvimai4gm677-profile +$ ls -l ~alice/.local/state/nix/profiles/profile* +lrwxrwxrwx 1 alice users 14 Nov 25 14:35 /home/alice/.local/state/nix/profiles/profile -> profile-7-link +lrwxrwxrwx 1 alice users 51 Oct 28 16:18 /home/alice/.local/state/nix/profiles/profile-5-link -> /nix/store/q69xad13ghpf7ir87h0b2gd28lafjj1j-profile +lrwxrwxrwx 1 alice users 51 Oct 29 13:20 /home/alice/.local/state/nix/profiles/profile-6-link -> /nix/store/6bvhpysd7vwz7k3b0pndn7ifi5xr32dg-profile +lrwxrwxrwx 1 alice users 51 Nov 25 14:35 /home/alice/.local/state/nix/profiles/profile-7-link -> /nix/store/mp0x6xnsg0b8qhswy6riqvimai4gm677-profile ``` Each of these symlinks is a root for the Nix garbage collector. @@ -38,20 +38,20 @@ profile is a tree of symlinks to the files of the installed packages, e.g. ```console -$ ll -R /nix/var/nix/profiles/per-user/eelco/profile-7-link/ -/nix/var/nix/profiles/per-user/eelco/profile-7-link/: +$ ll -R ~eelco/.local/state/nix/profiles/profile-7-link/ +/home/eelco/.local/state/nix/profiles/profile-7-link/: total 20 dr-xr-xr-x 2 root root 4096 Jan 1 1970 bin -r--r--r-- 2 root root 1402 Jan 1 1970 manifest.json dr-xr-xr-x 4 root root 4096 Jan 1 1970 share -/nix/var/nix/profiles/per-user/eelco/profile-7-link/bin: +/home/eelco/.local/state/nix/profiles/profile-7-link/bin: total 20 lrwxrwxrwx 5 root root 79 Jan 1 1970 chromium -> /nix/store/ijm5k0zqisvkdwjkc77mb9qzb35xfi4m-chromium-86.0.4240.111/bin/chromium lrwxrwxrwx 7 root root 87 Jan 1 1970 spotify -> /nix/store/w9182874m1bl56smps3m5zjj36jhp3rn-spotify-1.1.26.501.gbe11e53b-15/bin/spotify lrwxrwxrwx 3 root root 79 Jan 1 1970 zoom-us -> /nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927/bin/zoom-us -/nix/var/nix/profiles/per-user/eelco/profile-7-link/share/applications: +/home/eelco/.local/state/nix/profiles/profile-7-link/share/applications: total 12 lrwxrwxrwx 4 root root 120 Jan 1 1970 chromium-browser.desktop -> /nix/store/4cf803y4vzfm3gyk3vzhzb2327v0kl8a-chromium-unwrapped-86.0.4240.111/share/applications/chromium-browser.desktop lrwxrwxrwx 7 root root 110 Jan 1 1970 spotify.desktop -> /nix/store/w9182874m1bl56smps3m5zjj36jhp3rn-spotify-1.1.26.501.gbe11e53b-15/share/applications/spotify.desktop diff --git a/src/nix/realisation.cc b/src/nix/realisation.cc index 13db80282..e19e93219 100644 --- a/src/nix/realisation.cc +++ b/src/nix/realisation.cc @@ -45,7 +45,7 @@ struct CmdRealisationInfo : BuiltPathsCommand, MixJSON void run(ref store, BuiltPaths && paths) override { - settings.requireExperimentalFeature(Xp::CaDerivations); + experimentalFeatureSettings.require(Xp::CaDerivations); RealisedPath::Set realisations; for (auto & builtPath : paths) { diff --git a/src/nix/registry.cc b/src/nix/registry.cc index 1f4f820ac..cb94bbd31 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -224,7 +224,7 @@ struct CmdRegistry : virtual NixMultiCommand void run() override { - settings.requireExperimentalFeature(Xp::Flakes); + experimentalFeatureSettings.require(Xp::Flakes); if (!command) throw UsageError("'nix registry' requires a sub-command."); command->second->run(); diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 51d3074b4..bb14f3f99 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -1,6 +1,7 @@ #include "eval.hh" #include "globals.hh" #include "command.hh" +#include "installable-value.hh" #include "repl.hh" namespace nix { @@ -11,6 +12,14 @@ struct CmdRepl : RawInstallablesCommand evalSettings.pureEval = false; } + /** + * This command is stable before the others + */ + std::optional experimentalFeature() override + { + return std::nullopt; + } + std::vector files; Strings getDefaultFlakeAttrPaths() override @@ -37,7 +46,7 @@ struct CmdRepl : RawInstallablesCommand void applyDefaultInstallables(std::vector & rawInstallables) override { - if (!settings.isExperimentalFeatureEnabled(Xp::ReplFlake) && !(file) && rawInstallables.size() >= 1) { + if (!experimentalFeatureSettings.isEnabled(Xp::ReplFlake) && !(file) && rawInstallables.size() >= 1) { warn("future versions of Nix will require using `--file` to load a file"); if (rawInstallables.size() > 1) warn("more than one input file is not currently supported"); @@ -57,11 +66,12 @@ struct CmdRepl : RawInstallablesCommand auto getValues = [&]()->AbstractNixRepl::AnnotatedValues{ auto installables = parseInstallables(store, rawInstallables); AbstractNixRepl::AnnotatedValues values; - for (auto & installable: installables){ - auto what = installable->what(); + for (auto & installable_: installables){ + auto & installable = InstallableValue::require(*installable_); + auto what = installable.what(); if (file){ - auto [val, pos] = installable->toValue(*state); - auto what = installable->what(); + auto [val, pos] = installable.toValue(*state); + auto what = installable.what(); state->forceValue(*val, pos); auto autoArgs = getAutoArgs(*state); auto valPost = state->allocValue(); @@ -69,7 +79,7 @@ struct CmdRepl : RawInstallablesCommand state->forceValue(*valPost, pos); values.push_back( {valPost, what }); } else { - auto [val, pos] = installable->toValue(*state); + auto [val, pos] = installable.toValue(*state); values.push_back( {val, what} ); } } diff --git a/src/nix/run.cc b/src/nix/run.cc index 56605d9d5..1baf299ab 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -1,5 +1,5 @@ #include "run.hh" -#include "command.hh" +#include "command-installable-value.hh" #include "common-args.hh" #include "shared.hh" #include "store-api.hh" @@ -137,7 +137,7 @@ struct CmdShell : InstallablesCommand, MixEnvironment static auto rCmdShell = registerCommand("shell"); -struct CmdRun : InstallableCommand +struct CmdRun : InstallableValueCommand { using InstallableCommand::run; @@ -183,7 +183,7 @@ struct CmdRun : InstallableCommand return res; } - void run(ref store, ref installable) override + void run(ref store, ref installable) override { auto state = getEvalState(); diff --git a/src/nix/run.hh b/src/nix/run.hh index fed360158..97ddef19b 100644 --- a/src/nix/run.hh +++ b/src/nix/run.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" diff --git a/src/nix/search.cc b/src/nix/search.cc index 994ec44c2..c92ed1663 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -1,4 +1,4 @@ -#include "command.hh" +#include "command-installable-value.hh" #include "globals.hh" #include "eval.hh" #include "eval-inline.hh" @@ -22,7 +22,7 @@ std::string wrap(std::string prefix, std::string s) return concatStrings(prefix, s, ANSI_NORMAL); } -struct CmdSearch : InstallableCommand, MixJSON +struct CmdSearch : InstallableValueCommand, MixJSON { std::vector res; std::vector excludeRes; @@ -61,7 +61,7 @@ struct CmdSearch : InstallableCommand, MixJSON }; } - void run(ref store, ref installable) override + void run(ref store, ref installable) override { settings.readOnlyMode = true; evalSettings.enableImportFromDerivation.setDefault(false); diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 17796d6b8..2295d86d0 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -32,6 +32,14 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand }); } + /** + * This command is stable before the others + */ + std::optional experimentalFeature() override + { + return std::nullopt; + } + std::string description() override { return "upgrade Nix to the stable version declared in Nixpkgs"; diff --git a/src/nix/upgrade-nix.md b/src/nix/upgrade-nix.md index 084c80ba2..08757aebd 100644 --- a/src/nix/upgrade-nix.md +++ b/src/nix/upgrade-nix.md @@ -11,7 +11,7 @@ R""( * Upgrade Nix in a specific profile: ```console - # nix upgrade-nix -p /nix/var/nix/profiles/per-user/alice/profile + # nix upgrade-nix -p ~alice/.local/state/nix/profiles/profile ``` # Description diff --git a/tests/ca/build.sh b/tests/ca/build.sh index 98e1c5125..7754ad276 100644 --- a/tests/ca/build.sh +++ b/tests/ca/build.sh @@ -3,7 +3,7 @@ source common.sh drv=$(nix-instantiate ./content-addressed.nix -A rootCA --arg seed 1) -nix show-derivation "$drv" --arg seed 1 +nix derivation show "$drv" --arg seed 1 buildAttr () { local derivationPath=$1 diff --git a/tests/ca/derivation-json.sh b/tests/ca/derivation-json.sh new file mode 100644 index 000000000..3615177e9 --- /dev/null +++ b/tests/ca/derivation-json.sh @@ -0,0 +1,26 @@ +source common.sh + +export NIX_TESTS_CA_BY_DEFAULT=1 + +drvPath=$(nix-instantiate ../simple.nix) + +nix derivation show $drvPath | jq .[] > $TEST_HOME/simple.json + +drvPath2=$(nix derivation add < $TEST_HOME/simple.json) + +[[ "$drvPath" = "$drvPath2" ]] + +# Content-addressed derivations can be renamed. +jq '.name = "foo"' < $TEST_HOME/simple.json > $TEST_HOME/foo.json +drvPath3=$(nix derivation add --dry-run < $TEST_HOME/foo.json) +# With --dry-run nothing is actually written +[[ ! -e "$drvPath3" ]] + +# Without --dry-run it is actually written +drvPath4=$(nix derivation add < $TEST_HOME/foo.json) +[[ "$drvPath4" = "$drvPath3" ]] +[[ -e "$drvPath3" ]] + +# The modified derivation read back as JSON matches +nix derivation show $drvPath3 | jq .[] > $TEST_HOME/foo-read.json +diff $TEST_HOME/foo.json $TEST_HOME/foo-read.json diff --git a/tests/derivation-json.sh b/tests/derivation-json.sh new file mode 100644 index 000000000..b6be5d977 --- /dev/null +++ b/tests/derivation-json.sh @@ -0,0 +1,12 @@ +source common.sh + +drvPath=$(nix-instantiate simple.nix) + +nix derivation show $drvPath | jq .[] > $TEST_HOME/simple.json + +drvPath2=$(nix derivation add < $TEST_HOME/simple.json) + +[[ "$drvPath" = "$drvPath2" ]] + +# Input addressed derivations cannot be renamed. +jq '.name = "foo"' < $TEST_HOME/simple.json | expectStderr 1 nix derivation add | grepQuiet "has incorrect output" diff --git a/tests/describe-stores.sh b/tests/describe-stores.sh deleted file mode 100644 index 3fea61483..000000000 --- a/tests/describe-stores.sh +++ /dev/null @@ -1,8 +0,0 @@ -source common.sh - -# Query an arbitrary value in `nix describe-stores --json`'s output just to -# check that it has the right structure -[[ $(nix --experimental-features 'nix-command flakes' describe-stores --json | jq '.["SSH Store"]["compress"]["defaultValue"]') == false ]] - -# Ensure that the output of `nix describe-stores` isn't empty -[[ -n $(nix --experimental-features 'nix-command flakes' describe-stores) ]] diff --git a/tests/experimental-features.sh b/tests/experimental-features.sh new file mode 100644 index 000000000..a4d55f5f4 --- /dev/null +++ b/tests/experimental-features.sh @@ -0,0 +1,40 @@ +source common.sh + +# Without flakes, flake options should not show up +# With flakes, flake options should show up + +function both_ways { + nix --experimental-features 'nix-command' "$@" | grepQuietInverse flake + nix --experimental-features 'nix-command flakes' "$@" | grepQuiet flake + + # Also, the order should not matter + nix "$@" --experimental-features 'nix-command' | grepQuietInverse flake + nix "$@" --experimental-features 'nix-command flakes' | grepQuiet flake +} + +# Simple case, the configuration effects the running command +both_ways show-config + +# Skipping for now, because we actually *do* want these to show up in +# the manual, just be marked experimental. Will reenable once the manual +# generation takes advantage of the JSON metadata on this. + +# both_ways store gc --help + +expect 1 nix --experimental-features 'nix-command' show-config --flake-registry 'https://no' +nix --experimental-features 'nix-command flakes' show-config --flake-registry 'https://no' + +# Double check these are stable +nix --experimental-features '' --help +nix --experimental-features '' doctor --help +nix --experimental-features '' repl --help +nix --experimental-features '' upgrade-nix --help + +# These 3 arguments are currently given to all commands, which is wrong (as not +# all care). To deal with fixing later, we simply make them require the +# nix-command experimental features --- it so happens that the commands we wish +# stabilizing to do not need them anyways. +for arg in '--print-build-logs' '--offline' '--refresh'; do + nix --experimental-features 'nix-command' "$arg" --help + ! nix --experimental-features '' "$arg" --help +done diff --git a/tests/flakes/flakes.sh b/tests/flakes/flakes.sh index 5c922d7c5..f2e216435 100644 --- a/tests/flakes/flakes.sh +++ b/tests/flakes/flakes.sh @@ -96,7 +96,9 @@ json=$(nix flake metadata flake1 --json | jq .) hash1=$(echo "$json" | jq -r .revision) echo -n '# foo' >> $flake1Dir/flake.nix +flake1OriginalCommit=$(git -C $flake1Dir rev-parse HEAD) git -C $flake1Dir commit -a -m 'Foo' +flake1NewCommit=$(git -C $flake1Dir rev-parse HEAD) hash2=$(nix flake metadata flake1 --json --refresh | jq -r .revision) [[ $hash1 != $hash2 ]] @@ -491,3 +493,14 @@ nix store delete $(nix store add-path $badFlakeDir) [[ $(nix-instantiate --eval flake:git+file://$flake3Dir -A x) = 123 ]] [[ $(nix-instantiate -I flake3=flake:flake3 --eval '' -A x) = 123 ]] [[ $(NIX_PATH=flake3=flake:flake3 nix-instantiate --eval '' -A x) = 123 ]] + +# Test alternate lockfile paths. +nix flake lock $flake2Dir --output-lock-file $TEST_ROOT/flake2.lock +cmp $flake2Dir/flake.lock $TEST_ROOT/flake2.lock >/dev/null # lockfiles should be identical, since we're referencing flake2's original one + +nix flake lock $flake2Dir --output-lock-file $TEST_ROOT/flake2-overridden.lock --override-input flake1 git+file://$flake1Dir?rev=$flake1OriginalCommit +expectStderr 1 cmp $flake2Dir/flake.lock $TEST_ROOT/flake2-overridden.lock +nix flake metadata $flake2Dir --reference-lock-file $TEST_ROOT/flake2-overridden.lock | grepQuiet $flake1OriginalCommit + +# reference-lock-file can only be used if allow-dirty is set. +expectStderr 1 nix flake metadata $flake2Dir --no-allow-dirty --reference-lock-file $TEST_ROOT/flake2-overridden.lock diff --git a/tests/flakes/show.sh b/tests/flakes/show.sh index dd13264b9..a3d300552 100644 --- a/tests/flakes/show.sh +++ b/tests/flakes/show.sh @@ -64,3 +64,24 @@ in assert show_output == { }; true ' + +# Test that attributes with errors are handled correctly. +# nixpkgs.legacyPackages is a particularly prominent instance of this. +cat >flake.nix < show-output.json +nix eval --impure --expr ' +let show_output = builtins.fromJSON (builtins.readFile ./show-output.json); +in +assert show_output.legacyPackages.${builtins.currentSystem}.AAAAAASomeThingsFailToEvaluate == { }; +assert show_output.legacyPackages.${builtins.currentSystem}.simple.name == "simple"; +true +' diff --git a/tests/impure-derivations.sh b/tests/impure-derivations.sh index 7595fdd35..c7dadf397 100644 --- a/tests/impure-derivations.sh +++ b/tests/impure-derivations.sh @@ -37,8 +37,8 @@ path4=$(nix build -L --no-link --json --file ./impure-derivations.nix impureOnIm (! nix build -L --no-link --json --file ./impure-derivations.nix inputAddressed 2>&1) | grep 'depends on impure derivation' drvPath=$(nix eval --json --file ./impure-derivations.nix impure.drvPath | jq -r .) -[[ $(nix show-derivation $drvPath | jq ".[\"$drvPath\"].outputs.out.impure") = true ]] -[[ $(nix show-derivation $drvPath | jq ".[\"$drvPath\"].outputs.stuff.impure") = true ]] +[[ $(nix derivation show $drvPath | jq ".[\"$drvPath\"].outputs.out.impure") = true ]] +[[ $(nix derivation show $drvPath | jq ".[\"$drvPath\"].outputs.stuff.impure") = true ]] # Fixed-output derivations *can* depend on impure derivations. path5=$(nix build -L --no-link --json --file ./impure-derivations.nix contentAddressed | jq -r .[].outputs.out) diff --git a/tests/installer/default.nix b/tests/installer/default.nix index 31d83699d..49cfd2bcc 100644 --- a/tests/installer/default.nix +++ b/tests/installer/default.nix @@ -17,7 +17,7 @@ let script = '' tar -xf ./nix.tar.xz mv ./nix-* nix - ./nix/install --no-daemon + ./nix/install --no-daemon --no-channel-add ''; }; @@ -30,6 +30,14 @@ let }; }; + mockChannel = pkgs: + pkgs.runCommandNoCC "mock-channel" {} '' + mkdir nixexprs + mkdir -p $out/channel + echo -n 'someContent' > nixexprs/someFile + tar cvf - nixexprs | bzip2 > $out/channel/nixexprs.tar.bz2 + ''; + disableSELinux = "sudo setenforce 0"; images = { @@ -189,6 +197,11 @@ let echo "Running installer..." $ssh "set -eux; $installScript" + echo "Copying the mock channel" + # `scp -r` doesn't seem to work properly on some rhel instances, so let's + # use a plain tarpipe instead + tar -C ${mockChannel pkgs} -c channel | ssh -p 20022 $ssh_opts vagrant@localhost tar x -f- + echo "Testing Nix installation..." $ssh < \$out"]; }') [[ \$(cat \$out) = foobar ]] + + if pgrep nix-daemon; then + MAYBESUDO="sudo" + else + MAYBESUDO="" + fi + + + $MAYBESUDO \$(which nix-channel) --add file://\$HOME/channel myChannel + $MAYBESUDO \$(which nix-channel) --update + [[ \$(nix-instantiate --eval --expr 'builtins.readFile ') = '"someContent"' ]] EOF echo "Done!" diff --git a/tests/lang.sh b/tests/lang.sh index cdb4174eb..8170cb39d 100644 --- a/tests/lang.sh +++ b/tests/lang.sh @@ -5,12 +5,19 @@ export NIX_REMOTE=dummy:// export NIX_STORE_DIR=/nix/store nix-instantiate --eval -E 'builtins.trace "Hello" 123' 2>&1 | grepQuiet Hello +nix-instantiate --eval -E 'builtins.trace "Hello" 123' 2>/dev/null | grepQuiet 123 nix-instantiate --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1 nix-instantiate --trace-verbose --eval -E 'builtins.traceVerbose "Hello" 123' 2>&1 | grepQuiet Hello nix-instantiate --eval -E 'builtins.traceVerbose "Hello" 123' 2>&1 | grepQuietInverse Hello nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1 | grepQuietInverse Hello expectStderr 1 nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" (throw "Foo")' | grepQuiet Hello +nix-instantiate --eval -E 'let x = builtins.trace { x = x; } true; in x' \ + 2>&1 | grepQuiet -E 'trace: { x = «potential infinite recursion»; }' + +nix-instantiate --eval -E 'let x = { repeating = x; tracing = builtins.trace x true; }; in x.tracing'\ + 2>&1 | grepQuiet -F 'trace: { repeating = «repeated»; tracing = «potential infinite recursion»; }' + set +x fail=0 diff --git a/tests/legacy-ssh-store.sh b/tests/legacy-ssh-store.sh new file mode 100644 index 000000000..71b716b84 --- /dev/null +++ b/tests/legacy-ssh-store.sh @@ -0,0 +1,4 @@ +source common.sh + +# Check that store ping trusted doesn't yet work with ssh:// +nix --store ssh://localhost?remote-store=$TEST_ROOT/other-store store ping --json | jq -e 'has("trusted") | not' diff --git a/tests/local-store.sh b/tests/local-store.sh index 0247346f1..89502f864 100644 --- a/tests/local-store.sh +++ b/tests/local-store.sh @@ -17,3 +17,6 @@ PATH2=$(nix path-info --store "$PWD/x" $CORRECT_PATH) PATH3=$(nix path-info --store "local?root=$PWD/x" $CORRECT_PATH) [ $CORRECT_PATH == $PATH3 ] + +# Ensure store ping trusted works with local store +nix --store ./x store ping --json | jq -e '.trusted' diff --git a/tests/local.mk b/tests/local.mk index 05894727c..9a2bc5ac7 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -17,7 +17,9 @@ nix_tests = \ ca/gc.sh \ gc.sh \ remote-store.sh \ + legacy-ssh-store.sh \ lang.sh \ + experimental-features.sh \ fetchMercurial.sh \ gc-auto.sh \ user-envs.sh \ @@ -100,6 +102,8 @@ nix_tests = \ eval-store.sh \ why-depends.sh \ ca/why-depends.sh \ + derivation-json.sh \ + ca/derivation-json.sh \ import-derivation.sh \ ca/import-derivation.sh \ nix_path.sh \ @@ -115,7 +119,6 @@ nix_tests = \ db-migration.sh \ bash-profile.sh \ pass-as-file.sh \ - describe-stores.sh \ nix-profile.sh \ suggestions.sh \ store-ping.sh \ diff --git a/tests/nixos/nix-copy.nix b/tests/nixos/nix-copy.nix new file mode 100644 index 000000000..ee8b77100 --- /dev/null +++ b/tests/nixos/nix-copy.nix @@ -0,0 +1,85 @@ +# Test that ‘nix copy’ works over ssh. + +{ lib, config, nixpkgs, hostPkgs, ... }: + +let + pkgs = config.nodes.client.nixpkgs.pkgs; + + pkgA = pkgs.cowsay; + pkgB = pkgs.wget; + pkgC = pkgs.hello; + pkgD = pkgs.tmux; + +in { + name = "nix-copy"; + + enableOCR = true; + + nodes = + { client = + { config, lib, pkgs, ... }: + { virtualisation.writableStore = true; + virtualisation.additionalPaths = [ pkgA pkgD.drvPath ]; + nix.settings.substituters = lib.mkForce [ ]; + nix.settings.experimental-features = [ "nix-command" ]; + services.getty.autologinUser = "root"; + }; + + server = + { config, pkgs, ... }: + { services.openssh.enable = true; + services.openssh.permitRootLogin = "yes"; + users.users.root.password = "foobar"; + virtualisation.writableStore = true; + virtualisation.additionalPaths = [ pkgB pkgC ]; + }; + }; + + testScript = { nodes }: '' + # fmt: off + import subprocess + + # Create an SSH key on the client. + subprocess.run([ + "${pkgs.openssh}/bin/ssh-keygen", "-t", "ed25519", "-f", "key", "-N", "" + ], capture_output=True, check=True) + + start_all() + + server.wait_for_unit("sshd") + client.wait_for_unit("network.target") + client.wait_for_unit("getty@tty1.service") + client.wait_for_text("]#") + + # Copy the closure of package A from the client to the server using password authentication, + # and check that all prompts are visible + server.fail("nix-store --check-validity ${pkgA}") + client.send_chars("nix copy --to ssh://server ${pkgA} >&2; echo done\n") + client.wait_for_text("continue connecting") + client.send_chars("yes\n") + client.wait_for_text("Password:") + client.send_chars("foobar\n") + client.wait_for_text("done") + server.succeed("nix-store --check-validity ${pkgA}") + + client.copy_from_host("key", "/root/.ssh/id_ed25519") + client.succeed("chmod 600 /root/.ssh/id_ed25519") + + # Install the SSH key on the server. + server.copy_from_host("key.pub", "/root/.ssh/authorized_keys") + server.succeed("systemctl restart sshd") + client.succeed(f"ssh -o StrictHostKeyChecking=no {server.name} 'echo hello world'") + + # Copy the closure of package B from the server to the client, using ssh-ng. + client.fail("nix-store --check-validity ${pkgB}") + # Shouldn't download untrusted paths by default + client.fail("nix copy --from ssh-ng://server ${pkgB} >&2") + client.succeed("nix copy --no-check-sigs --from ssh-ng://server ${pkgB} >&2") + client.succeed("nix-store --check-validity ${pkgB}") + + # Copy the derivation of package D's derivation from the client to the server. + server.fail("nix-store --check-validity ${pkgD.drvPath}") + client.succeed("nix copy --derivation --to ssh://server ${pkgD.drvPath} >&2") + server.succeed("nix-store --check-validity ${pkgD.drvPath}") + ''; +} diff --git a/tests/remote-store.sh b/tests/remote-store.sh index 1ae126794..ea32a20d3 100644 --- a/tests/remote-store.sh +++ b/tests/remote-store.sh @@ -5,8 +5,19 @@ clearStore # Ensure "fake ssh" remote store works just as legacy fake ssh would. nix --store ssh-ng://localhost?remote-store=$TEST_ROOT/other-store doctor +# Ensure that store ping trusted works with ssh-ng:// +nix --store ssh-ng://localhost?remote-store=$TEST_ROOT/other-store store ping --json | jq -e '.trusted' + startDaemon +if isDaemonNewer "2.15pre0"; then + # Ensure that ping works trusted with new daemon + nix store ping --json | jq -e '.trusted' +else + # And the the field is absent with the old daemon + nix store ping --json | jq -e 'has("trusted") | not' +fi + # Test import-from-derivation through the daemon. [[ $(nix eval --impure --raw --expr ' with import ./config.nix;