diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index be96bb484..addafb9f8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,9 +7,16 @@ on: permissions: read-all jobs: + eval: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: cachix/install-nix-action@v30 + - run: nix --experimental-features 'nix-command flakes' flake show --all-systems --json tests: - needs: [check_secrets] strategy: fail-fast: false matrix: @@ -26,86 +33,21 @@ jobs: extra_nix_config: | sandbox = true max-jobs = 1 - - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/cachix-action@v15 - if: needs.check_secrets.outputs.cachix == 'true' + - uses: DeterminateSystems/magic-nix-cache-action@main + # Since ubuntu 22.30, unprivileged usernamespaces are no longer allowed to map to the root user: + # https://ubuntu.com/blog/ubuntu-23-10-restricted-unprivileged-user-namespaces + - run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + if: matrix.os == 'ubuntu-latest' + - run: scripts/build-checks + - run: scripts/prepare-installer-for-github-actions + - name: Upload installer tarball + uses: actions/upload-artifact@v4 with: - name: '${{ env.CACHIX_NAME }}' - signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' - authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - - if: matrix.os == 'ubuntu-latest' - run: | - free -h - swapon --show - swap=$(swapon --show --noheadings | head -n 1 | awk '{print $1}') - echo "Found swap: $swap" - sudo swapoff $swap - # resize it (fallocate) - sudo fallocate -l 10G $swap - sudo mkswap $swap - sudo swapon $swap - free -h - ( - while sleep 60; do - free -h - done - ) & - - run: nix --experimental-features 'nix-command flakes' flake check -L - - run: nix --experimental-features 'nix-command flakes' flake show --all-systems --json - - # Steps to test CI automation in your own fork. - # Cachix: - # 1. Sign-up for https://www.cachix.org/ - # 2. Create a cache for $githubuser-nix-install-tests - # 3. Create a cachix auth token and save it in https://github.com/$githubuser/nix/settings/secrets/actions in "Repository secrets" as CACHIX_AUTH_TOKEN - # Dockerhub: - # 1. Sign-up for https://hub.docker.com/ - # 2. Store your dockerhub username as DOCKERHUB_USERNAME in "Repository secrets" of your fork repository settings (https://github.com/$githubuser/nix/settings/secrets/actions) - # 3. Create an access token in https://hub.docker.com/settings/security and store it as DOCKERHUB_TOKEN in "Repository secrets" of your fork - check_secrets: - permissions: - contents: none - name: Check Cachix and Docker secrets present for installer tests - runs-on: ubuntu-latest - outputs: - cachix: ${{ steps.secret.outputs.cachix }} - docker: ${{ steps.secret.outputs.docker }} - steps: - - name: Check for secrets - id: secret - env: - _CACHIX_SECRETS: ${{ secrets.CACHIX_SIGNING_KEY }}${{ secrets.CACHIX_AUTH_TOKEN }} - _DOCKER_SECRETS: ${{ secrets.DOCKERHUB_USERNAME }}${{ secrets.DOCKERHUB_TOKEN }} - run: | - echo "::set-output name=cachix::${{ env._CACHIX_SECRETS != '' }}" - echo "::set-output name=docker::${{ env._DOCKER_SECRETS != '' }}" - - installer: - needs: [tests, check_secrets] - if: github.event_name == 'push' && needs.check_secrets.outputs.cachix == 'true' - runs-on: ubuntu-latest - outputs: - installerURL: ${{ steps.prepare-installer.outputs.installerURL }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v30 - with: - install_url: https://releases.nixos.org/nix/nix-2.20.3/install - - uses: cachix/cachix-action@v15 - with: - name: '${{ env.CACHIX_NAME }}' - signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' - authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - cachixArgs: '-v' - - id: prepare-installer - run: scripts/prepare-installer-for-github-actions + name: installer-${{matrix.os}} + path: out/* installer_test: - needs: [installer, check_secrets] - if: github.event_name == 'push' && needs.check_secrets.outputs.cachix == 'true' + needs: [tests] strategy: fail-fast: false matrix: @@ -113,11 +55,18 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV + - name: Download installer tarball + uses: actions/download-artifact@v4 + with: + name: installer-${{matrix.os}} + path: out + - name: Serving installer + id: serving_installer + run: ./scripts/serve-installer-for-github-actions - uses: cachix/install-nix-action@v30 with: - install_url: '${{needs.installer.outputs.installerURL}}' - install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" + install_url: 'http://localhost:8126/install' + install_options: "--tarball-url-prefix http://localhost:8126/" - run: sudo apt install fish zsh if: matrix.os == 'ubuntu-latest' - run: brew install fish @@ -129,32 +78,50 @@ jobs: - run: exec bash -c "nix-channel --add https://releases.nixos.org/nixos/unstable/nixos-23.05pre466020.60c1d71f2ba nixpkgs" - run: exec bash -c "nix-channel --update && nix-env -iA nixpkgs.hello && hello" + # Steps to test CI automation in your own fork. + # 1. Sign-up for https://hub.docker.com/ + # 2. Store your dockerhub username as DOCKERHUB_USERNAME in "Repository secrets" of your fork repository settings (https://github.com/$githubuser/nix/settings/secrets/actions) + # 3. Create an access token in https://hub.docker.com/settings/security and store it as DOCKERHUB_TOKEN in "Repository secrets" of your fork + check_secrets: + permissions: + contents: none + name: Check Docker secrets present for installer tests + runs-on: ubuntu-latest + outputs: + docker: ${{ steps.secret.outputs.docker }} + steps: + - name: Check for secrets + id: secret + env: + _DOCKER_SECRETS: ${{ secrets.DOCKERHUB_USERNAME }}${{ secrets.DOCKERHUB_TOKEN }} + run: | + echo "::set-output name=docker::${{ env._DOCKER_SECRETS != '' }}" + docker_push_image: - needs: [check_secrets, tests, vm_tests] + needs: [tests, vm_tests, check_secrets] permissions: contents: read packages: write if: >- + needs.check_secrets.outputs.docker == 'true' && github.event_name == 'push' && - github.ref_name == 'master' && - needs.check_secrets.outputs.cachix == 'true' && - needs.check_secrets.outputs.docker == 'true' + github.ref_name == 'master' runs-on: ubuntu-latest steps: + - name: Check for secrets + id: secret + env: + _DOCKER_SECRETS: ${{ secrets.DOCKERHUB_USERNAME }}${{ secrets.DOCKERHUB_TOKEN }} + run: | + echo "::set-output name=docker::${{ env._DOCKER_SECRETS != '' }}" - uses: actions/checkout@v4 with: fetch-depth: 0 - uses: cachix/install-nix-action@v30 with: install_url: https://releases.nixos.org/nix/nix-2.20.3/install - - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV + - uses: DeterminateSystems/magic-nix-cache-action@main - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#nix.version | tr -d \")" >> $GITHUB_ENV - - uses: cachix/cachix-action@v15 - if: needs.check_secrets.outputs.cachix == 'true' - with: - name: '${{ env.CACHIX_NAME }}' - signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' - authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - run: nix --experimental-features 'nix-command flakes' build .#dockerImage -L - run: docker load -i ./result/image.tar.gz - run: docker tag nix:$NIX_VERSION ${{ secrets.DOCKERHUB_USERNAME }}/nix:$NIX_VERSION diff --git a/.gitignore b/.gitignore index de1183977..337a7c154 100644 --- a/.gitignore +++ b/.gitignore @@ -1,110 +1,12 @@ -Makefile.config -perl/Makefile.config - -# / -/aclocal.m4 -/autom4te.cache -/precompiled-headers.h.gch -/config.* -/configure -/stamp-h1 -/svn-revision -/libtool -/config/config.* # Default meson build dir /build -# /doc/manual/ -/doc/manual/*.1 -/doc/manual/*.5 -/doc/manual/*.8 -/doc/manual/generated/* -/doc/manual/nix.json -/doc/manual/conf-file.json -/doc/manual/language.json -/doc/manual/xp-features.json -/doc/manual/source/SUMMARY.md -/doc/manual/source/SUMMARY-rl-next.md -/doc/manual/source/store/types/* -!/doc/manual/source/store/types/index.md.in -/doc/manual/source/command-ref/new-cli -/doc/manual/source/command-ref/conf-file.md -/doc/manual/source/command-ref/experimental-features-shortlist.md -/doc/manual/source/contributing/experimental-feature-descriptions.md -/doc/manual/source/language/builtins.md -/doc/manual/source/language/builtin-constants.md -/doc/manual/source/release-notes/rl-next.md - -# /scripts/ -/scripts/nix-profile.sh -/scripts/nix-profile-daemon.sh -/scripts/nix-profile.fish -/scripts/nix-profile-daemon.fish - -# /src/libexpr/ -/src/libexpr/lexer-tab.cc -/src/libexpr/lexer-tab.hh -/src/libexpr/parser-tab.cc -/src/libexpr/parser-tab.hh -/src/libexpr/parser-tab.output -/src/libexpr/nix.tbl -/src/libexpr/tests -/src/libexpr-tests/libnixexpr-tests - -# /src/libfetchers -/src/libfetchers-tests/libnixfetchers-tests - -# /src/libflake -/src/libflake-tests/libnixflake-tests - -# /src/libstore/ -*.gen.* -/src/libstore/tests -/src/libstore-tests/libnixstore-tests - -# /src/libutil/ -/src/libutil/tests -/src/libutil-tests/libnixutil-tests - -/src/nix/nix - -/src/nix/generated-doc - -# /src/nix-env/ -/src/nix-env/nix-env - -# /src/nix-instantiate/ -/src/nix-instantiate/nix-instantiate - -# /src/nix-store/ -/src/nix-store/nix-store - -/src/nix-prefetch-url/nix-prefetch-url - -/src/nix-collect-garbage/nix-collect-garbage - -# /src/nix-channel/ -/src/nix-channel/nix-channel - -# /src/nix-build/ -/src/nix-build/nix-build - -/src/nix-copy-closure/nix-copy-closure - -/src/error-demo/error-demo - -/src/build-remote/build-remote - # /tests/functional/ -/tests/functional/test-tmp /tests/functional/common/subst-vars.sh -/tests/functional/result* /tests/functional/restricted-innocent -/tests/functional/shell -/tests/functional/shell.drv -/tests/functional/repl-result-out /tests/functional/debugger-test-out /tests/functional/test-libstoreconsumer/test-libstoreconsumer +/tests/functional/nix-shell # /tests/functional/lang/ /tests/functional/lang/*.out @@ -112,27 +14,9 @@ perl/Makefile.config /tests/functional/lang/*.err /tests/functional/lang/*.ast -/perl/lib/Nix/Config.pm -/perl/lib/Nix/Store.cc - -/misc/systemd/nix-daemon.service -/misc/systemd/nix-daemon.socket -/misc/systemd/nix-daemon.conf -/misc/upstart/nix-daemon.conf - outputs/ -*.a -*.o -*.o.tmp -*.so -*.dylib -*.dll -*.exe -*.dep *~ -*.pc -*.plist # GNU Global GPATH @@ -147,8 +31,6 @@ GTAGS compile_commands.json *.compile_commands.json -nix-rust/target - result result-* @@ -163,3 +45,5 @@ result-* # Mac OS .DS_Store + +flake-regressions diff --git a/.mergify.yml b/.mergify.yml index c545bbe6a..70fccae49 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -4,8 +4,9 @@ queue_rules: merge_conditions: - check-success=tests (macos-latest) - check-success=tests (ubuntu-latest) + - check-success=installer_test (macos-latest) + - check-success=installer_test (ubuntu-latest) - check-success=vm_tests - merge_method: rebase batch_size: 5 pull_request_rules: diff --git a/doc/manual/meson.build b/doc/manual/meson.build index 3630e2dc8..c4cc9b893 100644 --- a/doc/manual/meson.build +++ b/doc/manual/meson.build @@ -199,6 +199,7 @@ nix3_manpages = [ 'nix3-build', 'nix3-bundle', 'nix3-config', + 'nix3-config-check', 'nix3-config-show', 'nix3-copy', 'nix3-daemon', @@ -206,8 +207,8 @@ nix3_manpages = [ 'nix3-derivation', 'nix3-derivation-show', 'nix3-develop', - #'nix3-doctor', 'nix3-edit', + 'nix3-env-shell', 'nix3-eval', 'nix3-flake-archive', 'nix3-flake-check', @@ -224,6 +225,7 @@ nix3_manpages = [ 'nix3-fmt', 'nix3-hash-file', 'nix3-hash', + 'nix3-hash-convert', 'nix3-hash-path', 'nix3-hash-to-base16', 'nix3-hash-to-base32', @@ -238,6 +240,7 @@ nix3_manpages = [ 'nix3-nar-cat', 'nix3-nar-dump-path', 'nix3-nar-ls', + 'nix3-nar-pack', 'nix3-nar', 'nix3-path-info', 'nix3-print-dev-env', @@ -260,7 +263,7 @@ nix3_manpages = [ 'nix3-repl', 'nix3-run', 'nix3-search', - #'nix3-shell', + 'nix3-store-add', 'nix3-store-add-file', 'nix3-store-add-path', 'nix3-store-cat', @@ -270,6 +273,7 @@ nix3_manpages = [ 'nix3-store-diff-closures', 'nix3-store-dump-path', 'nix3-store-gc', + 'nix3-store-info', 'nix3-store-ls', 'nix3-store-make-content-addressed', 'nix3-store', diff --git a/doc/manual/rl-next/ignore-local-registries.md b/doc/manual/rl-next/ignore-local-registries.md new file mode 100644 index 000000000..8d5e333dd --- /dev/null +++ b/doc/manual/rl-next/ignore-local-registries.md @@ -0,0 +1,22 @@ +--- +synopsis: "Flake lock file generation now ignores local registries" +prs: [12019] +--- + +When resolving indirect flake references like `nixpkgs` in `flake.nix` files, Nix will no longer use the system and user flake registries. It will only use the global flake registry and overrides given on the command line via `--override-flake`. + +This avoids accidents where users have local registry overrides that map `nixpkgs` to a `path:` flake in the local file system, which then end up in committed lock files pushed to other users. + +In the future, we may remove the use of the registry during lock file generation altogether. It's better to explicitly specify the URL of a flake input. For example, instead of +```nix +{ + outputs = { self, nixpkgs }: { ... }; +} +``` +write +```nix +{ + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11"; + outputs = { self, nixpkgs }: { ... }; +} +``` diff --git a/doc/manual/rl-next/nix-sshopts-parsing.md b/doc/manual/rl-next/nix-sshopts-parsing.md new file mode 100644 index 000000000..65fe6f562 --- /dev/null +++ b/doc/manual/rl-next/nix-sshopts-parsing.md @@ -0,0 +1,21 @@ +--- +synopsis: "Improved `NIX_SSHOPTS` parsing for better SSH option handling" +issues: [5181] +prs: [12020] +--- + +The parsing of the `NIX_SSHOPTS` environment variable has been improved to handle spaces and quotes correctly. +Previously, incorrectly split SSH options could cause failures in CLIs like `nix-copy-closure`, +especially when using complex ssh invocations such as `-o ProxyCommand="ssh -W %h:%p ..."`. + +This change introduces a `shellSplitString` function to ensure +that `NIX_SSHOPTS` is parsed in a manner consistent with shell +behavior, addressing common parsing errors. + +For example, the following now works as expected: + +```bash +export NIX_SSHOPTS='-o ProxyCommand="ssh -W %h:%p ..."' +``` + +This update improves the reliability of SSH-related operations using `NIX_SSHOPTS` across Nix CLIs. diff --git a/flake.nix b/flake.nix index 794736af4..44a764dd7 100644 --- a/flake.nix +++ b/flake.nix @@ -186,7 +186,7 @@ }; checks = forAllSystems (system: { - binaryTarball = self.hydraJobs.binaryTarball.${system}; + installerScriptForGHA = self.hydraJobs.installerScriptForGHA.${system}; installTests = self.hydraJobs.installTests.${system}; nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system}; rl-next = @@ -201,11 +201,7 @@ # Some perl dependencies are broken on i686-linux. # Since the support is only best-effort there, disable the perl # bindings - - # Temporarily disabled because GitHub Actions OOM issues. Once - # the old build system is gone and we are back to one build - # system, we should reenable this. - #perlBindings = self.hydraJobs.perlBindings.${system}; + perlBindings = self.hydraJobs.perlBindings.${system}; } # Add "passthru" tests // flatMapAttrs ({ @@ -237,6 +233,8 @@ inherit (nixpkgsFor.${system}.native) changelog-d; default = self.packages.${system}.nix; + installerScriptForGHA = self.hydraJobs.installerScriptForGHA.${system}; + binaryTarball = self.hydraJobs.binaryTarball.${system}; # TODO probably should be `nix-cli` nix = self.packages.${system}.nix-everything; nix-manual = nixpkgsFor.${system}.native.nixComponents.nix-manual; diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index 1d4e85c8c..1a134b91a 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -12,6 +12,8 @@ hooks = { clang-format = { enable = true; + # https://github.com/cachix/git-hooks.nix/pull/532 + package = pkgs.llvmPackages_latest.clang-tools; excludes = [ # We don't want to format test data # ''tests/(?!nixos/).*\.nix'' diff --git a/misc/launchd/meson.build b/misc/launchd/meson.build new file mode 100644 index 000000000..5168131d1 --- /dev/null +++ b/misc/launchd/meson.build @@ -0,0 +1,13 @@ +configure_file( + input : 'org.nixos.nix-daemon.plist.in', + output : 'org.nixos.nix-daemon.plist', + install : true, + install_dir : get_option('prefix') / 'Library/LaunchDaemons', + install_mode : 'rw-r--r--', + configuration : { + # TODO: unhardcode paths with something like: + # 'storedir' : store_dir, + # 'localstatedir' : localstatedir, + # 'bindir' : bindir, + }, +) diff --git a/misc/meson.build b/misc/meson.build index a6d1f944b..82f2b0c65 100644 --- a/misc/meson.build +++ b/misc/meson.build @@ -2,4 +2,10 @@ subdir('bash') subdir('fish') subdir('zsh') -subdir('systemd') +if host_machine.system() == 'linux' + subdir('systemd') +endif + +if host_machine.system() == 'darwin' + subdir('launchd') +endif diff --git a/build-utils-meson/common/meson.build b/nix-meson-build-support/common/meson.build similarity index 100% rename from build-utils-meson/common/meson.build rename to nix-meson-build-support/common/meson.build diff --git a/build-utils-meson/deps-lists/meson.build b/nix-meson-build-support/deps-lists/meson.build similarity index 100% rename from build-utils-meson/deps-lists/meson.build rename to nix-meson-build-support/deps-lists/meson.build diff --git a/build-utils-meson/export-all-symbols/meson.build b/nix-meson-build-support/export-all-symbols/meson.build similarity index 100% rename from build-utils-meson/export-all-symbols/meson.build rename to nix-meson-build-support/export-all-symbols/meson.build diff --git a/build-utils-meson/export/meson.build b/nix-meson-build-support/export/meson.build similarity index 100% rename from build-utils-meson/export/meson.build rename to nix-meson-build-support/export/meson.build diff --git a/build-utils-meson/generate-header/meson.build b/nix-meson-build-support/generate-header/meson.build similarity index 100% rename from build-utils-meson/generate-header/meson.build rename to nix-meson-build-support/generate-header/meson.build diff --git a/build-utils-meson/libatomic/meson.build b/nix-meson-build-support/libatomic/meson.build similarity index 100% rename from build-utils-meson/libatomic/meson.build rename to nix-meson-build-support/libatomic/meson.build diff --git a/build-utils-meson/subprojects/meson.build b/nix-meson-build-support/subprojects/meson.build similarity index 100% rename from build-utils-meson/subprojects/meson.build rename to nix-meson-build-support/subprojects/meson.build diff --git a/build-utils-meson/windows-version/meson.build b/nix-meson-build-support/windows-version/meson.build similarity index 100% rename from build-utils-meson/windows-version/meson.build rename to nix-meson-build-support/windows-version/meson.build diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 81406a249..5b1e47559 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -18,12 +18,8 @@ let testNixVersions = pkgs: daemon: pkgs.nixComponents.nix-functional-tests.override { - pname = - "nix-tests" - + lib.optionalString - (lib.versionAtLeast daemon.version "2.4pre20211005" && - lib.versionAtLeast pkgs.nix.version "2.4pre20211005") - "-${pkgs.nix.version}-against-${daemon.version}"; + pname = "nix-daemon-compat-tests"; + version = "${pkgs.nix.version}-with-daemon-${daemon.version}"; test-daemon = daemon; }; @@ -127,15 +123,10 @@ in self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu" ]; - installerScriptForGHA = installScriptFor [ - # Native - self.hydraJobs.binaryTarball."x86_64-linux" - self.hydraJobs.binaryTarball."aarch64-darwin" - # Cross - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" - self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu" - ]; + + installerScriptForGHA = forAllSystems (system: nixpkgsFor.${system}.native.callPackage ../scripts/installer.nix { + tarballs = [ self.hydraJobs.binaryTarball.${system} ]; + }); # docker image with Nix inside dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); diff --git a/scripts/binary-tarball.nix b/scripts/binary-tarball.nix index 671c8e96e..9de90b7fb 100644 --- a/scripts/binary-tarball.nix +++ b/scripts/binary-tarball.nix @@ -65,7 +65,7 @@ runCommand "nix-binary-tarball-${version}" env '' fn=$out/$dir.tar.xz mkdir -p $out/nix-support echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products - tar cvfJ $fn \ + tar cfJ $fn \ --owner=0 --group=0 --mode=u+rw,uga+r \ --mtime='1970-01-01' \ --absolute-names \ diff --git a/scripts/build-checks b/scripts/build-checks new file mode 100755 index 000000000..e0ee70631 --- /dev/null +++ b/scripts/build-checks @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail +system=$(nix eval --raw --impure --expr builtins.currentSystem) +nix eval --json ".#checks.$system" --apply builtins.attrNames | \ + jq -r '.[]' | \ + xargs -P0 -I '{}' sh -c "nix build -L .#checks.$system.{} || { echo 'FAILED: \033[0;31mnix build -L .#checks.$system.{}\\033[0m'; kill 0; }" diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh index 103e1e391..7a61764d4 100755 --- a/scripts/create-darwin-volume.sh +++ b/scripts/create-darwin-volume.sh @@ -463,7 +463,7 @@ EOF EDITOR="$SCRATCH/ex_cleanroom_wrapper" _sudo "to add nix to fstab" "$@" < "$SCRATCH/.nix-channels" _sudo "to set up the default system channel (part 1)" \ - install -m 0664 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels" + install -m 0644 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels" fi } @@ -964,7 +969,7 @@ $NIX_EXTRA_CONF build-users-group = $NIX_BUILD_GROUP_NAME EOF _sudo "to place the default nix daemon configuration (part 2)" \ - install -m 0664 "$SCRATCH/nix.conf" /etc/nix/nix.conf + install -m 0644 "$SCRATCH/nix.conf" /etc/nix/nix.conf } diff --git a/scripts/install-nix-from-tarball.sh b/scripts/install-nix-from-tarball.sh index 007fe85ee..8d127a9c5 100644 --- a/scripts/install-nix-from-tarball.sh +++ b/scripts/install-nix-from-tarball.sh @@ -9,6 +9,8 @@ self="$(dirname "$0")" nix="@nix@" cacert="@cacert@" +# allow to override identity change command +readonly NIX_BECOME="${NIX_BECOME:-sudo}" if ! [ -e "$self/.reginfo" ]; then echo "$0: incomplete installer (.reginfo is missing)" >&2 @@ -63,7 +65,6 @@ while [ $# -gt 0 ]; do exit 1 fi INSTALL_MODE=no-daemon - # intentional tail space ACTION=install ;; --yes) @@ -135,8 +136,8 @@ echo "performing a single-user installation of Nix..." >&2 if ! [ -e "$dest" ]; then cmd="mkdir -m 0755 $dest && chown $USER $dest" - echo "directory $dest does not exist; creating it by running '$cmd' using sudo" >&2 - if ! sudo sh -c "$cmd"; then + echo "directory $dest does not exist; creating it by running '$cmd' using $NIX_BECOME" >&2 + if ! $NIX_BECOME sh -c "$cmd"; then echo "$0: please manually run '$cmd' as root to create $dest" >&2 exit 1 fi diff --git a/scripts/prepare-installer-for-github-actions b/scripts/prepare-installer-for-github-actions index 4b994a753..0fbecf25c 100755 --- a/scripts/prepare-installer-for-github-actions +++ b/scripts/prepare-installer-for-github-actions @@ -1,10 +1,11 @@ #!/usr/bin/env bash -set -e +set -euo pipefail -script=$(nix-build -A outputs.hydraJobs.installerScriptForGHA --no-out-link) -installerHash=$(echo "$script" | cut -b12-43 -) +nix build -L ".#installerScriptForGHA" ".#binaryTarball" -installerURL=https://$CACHIX_NAME.cachix.org/serve/$installerHash/install - -echo "::set-output name=installerURL::$installerURL" +mkdir -p out +cp ./result/install "out/install" +name="$(basename "$(realpath ./result-1)")" +# everything before the first dash +cp -r ./result-1 "out/${name%%-*}" diff --git a/scripts/serve-installer-for-github-actions b/scripts/serve-installer-for-github-actions new file mode 100755 index 000000000..2efd2aa32 --- /dev/null +++ b/scripts/serve-installer-for-github-actions @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -euo pipefail +if [[ ! -d out ]]; then + echo "run prepare-installer-for-github-actions first" + exit 1 +fi +cd out +PORT=${PORT:-8126} +nohup python -m http.server "$PORT" >/dev/null 2>&1 & +pid=$! + +while ! curl -s "http://localhost:$PORT"; do + sleep 1 + if ! kill -0 $pid; then + echo "Failed to start http server" + exit 1 + fi +done + +echo 'To install nix, run the following command:' +echo "sh <(curl http://localhost:$PORT/install) --tarball-url-prefix http://localhost:$PORT" diff --git a/src/libcmd/build-utils-meson b/src/libcmd/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libcmd/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build index 1f27c1614..222817c81 100644 --- a/src/libcmd/meson.build +++ b/src/libcmd/meson.build @@ -14,7 +14,7 @@ project('nix-cmd', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -28,7 +28,7 @@ deps_public_maybe_subproject = [ dependency('nix-flake'), dependency('nix-main'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') deps_public += nlohmann_json @@ -70,7 +70,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'built-path.cc', @@ -125,4 +125,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libcmd/nix-meson-build-support b/src/libcmd/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libcmd/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libcmd/package.nix b/src/libcmd/package.nix index 244179ee4..53e54d2f8 100644 --- a/src/libcmd/package.nix +++ b/src/libcmd/package.nix @@ -39,8 +39,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libexpr-c/build-utils-meson b/src/libexpr-c/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libexpr-c/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libexpr-c/meson.build b/src/libexpr-c/meson.build index 5bcca29e0..1556dae51 100644 --- a/src/libexpr-c/meson.build +++ b/src/libexpr-c/meson.build @@ -14,7 +14,7 @@ project('nix-expr-c', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -27,7 +27,7 @@ deps_public_maybe_subproject = [ dependency('nix-util-c'), dependency('nix-store-c'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') # TODO rename, because it will conflict with downstream projects configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) @@ -53,7 +53,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'nix_api_expr.cc', @@ -72,8 +72,8 @@ headers = [config_h] + files( # TODO move this header to libexpr, maybe don't use it in tests? headers += files('nix_api_expr_internal.h') -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') this_library = library( 'nixexprc', @@ -89,4 +89,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libexpr-c/nix-meson-build-support b/src/libexpr-c/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libexpr-c/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libexpr-c/package.nix b/src/libexpr-c/package.nix index df49a8bdc..727b3a811 100644 --- a/src/libexpr-c/package.nix +++ b/src/libexpr-c/package.nix @@ -20,8 +20,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libexpr-test-support/build-utils-meson b/src/libexpr-test-support/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libexpr-test-support/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libexpr-test-support/meson.build b/src/libexpr-test-support/meson.build index 33d9e17a6..64d4fe218 100644 --- a/src/libexpr-test-support/meson.build +++ b/src/libexpr-test-support/meson.build @@ -14,7 +14,7 @@ project('nix-expr-test-support', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') deps_private_maybe_subproject = [ ] @@ -26,7 +26,7 @@ deps_public_maybe_subproject = [ dependency('nix-expr'), dependency('nix-expr-c'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') rapidcheck = dependency('rapidcheck') deps_public += rapidcheck @@ -40,7 +40,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'tests/value/context.cc', @@ -54,8 +54,8 @@ headers = files( 'tests/value/context.hh', ) -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') this_library = library( 'nix-expr-test-support', @@ -73,4 +73,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libexpr-test-support/nix-meson-build-support b/src/libexpr-test-support/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libexpr-test-support/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libexpr-test-support/package.nix b/src/libexpr-test-support/package.nix index 7e92d145f..4842f5f17 100644 --- a/src/libexpr-test-support/package.nix +++ b/src/libexpr-test-support/package.nix @@ -23,8 +23,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libexpr-tests/build-utils-meson b/src/libexpr-tests/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libexpr-tests/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libexpr-tests/error_traces.cc b/src/libexpr-tests/error_traces.cc index be379a909..2aa13cf62 100644 --- a/src/libexpr-tests/error_traces.cc +++ b/src/libexpr-tests/error_traces.cc @@ -691,15 +691,15 @@ namespace nix { ASSERT_TRACE2("elemAt \"foo\" (-1)", TypeError, HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.elemAt")); + HintFmt("while evaluating the first argument passed to 'builtins.elemAt'")); ASSERT_TRACE1("elemAt [] (-1)", Error, - HintFmt("list index %d is out of bounds", -1)); + HintFmt("'builtins.elemAt' called with index %d on a list of size %d", -1, 0)); ASSERT_TRACE1("elemAt [\"foo\"] 3", Error, - HintFmt("list index %d is out of bounds", 3)); + HintFmt("'builtins.elemAt' called with index %d on a list of size %d", 3, 1)); } @@ -708,11 +708,11 @@ namespace nix { ASSERT_TRACE2("head 1", TypeError, HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.elemAt")); + HintFmt("while evaluating the first argument passed to 'builtins.head'")); ASSERT_TRACE1("head []", Error, - HintFmt("list index %d is out of bounds", 0)); + HintFmt("'builtins.head' called on an empty list")); } @@ -721,11 +721,11 @@ namespace nix { ASSERT_TRACE2("tail 1", TypeError, HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.tail")); + HintFmt("while evaluating the first argument passed to 'builtins.tail'")); ASSERT_TRACE1("tail []", Error, - HintFmt("'tail' called on an empty list")); + HintFmt("'builtins.tail' called on an empty list")); } diff --git a/src/libexpr-tests/meson.build b/src/libexpr-tests/meson.build index b50c18c9c..f37e85e57 100644 --- a/src/libexpr-tests/meson.build +++ b/src/libexpr-tests/meson.build @@ -14,7 +14,7 @@ project('nix-expr-tests', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') deps_private_maybe_subproject = [ dependency('nix-expr'), @@ -23,10 +23,10 @@ deps_private_maybe_subproject = [ ] deps_public_maybe_subproject = [ ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') rapidcheck = dependency('rapidcheck') deps_private += rapidcheck @@ -49,7 +49,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'derived-path.cc', diff --git a/src/libexpr-tests/nix-meson-build-support b/src/libexpr-tests/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libexpr-tests/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libexpr-tests/package.nix b/src/libexpr-tests/package.nix index 959d6b84e..70e497b7e 100644 --- a/src/libexpr-tests/package.nix +++ b/src/libexpr-tests/package.nix @@ -27,8 +27,8 @@ mkMesonExecutable (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libexpr/build-utils-meson b/src/libexpr/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libexpr/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 05f58957e..fe5f05ab8 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -347,6 +347,16 @@ void EvalState::allowPath(const StorePath & storePath) rootFS2->allowPrefix(CanonPath(store->toRealPath(storePath))); } +void EvalState::allowClosure(const StorePath & storePath) +{ + if (!rootFS.dynamic_pointer_cast()) return; + + StorePathSet closure; + store->computeFSClosure(storePath, closure); + for (auto & p : closure) + allowPath(p); +} + void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & v) { allowPath(storePath); @@ -3099,10 +3109,7 @@ std::optional EvalState::resolveLookupPathPath(const LookupPath::Pat allowPath(path.path.abs()); if (store->isInStore(path.path.abs())) { try { - StorePathSet closure; - store->computeFSClosure(store->toStorePath(path.path.abs()).first, closure); - for (auto & p : closure) - allowPath(p); + allowClosure(store->toStorePath(path.path.abs()).first); } catch (InvalidPath &) { } } } @@ -3178,5 +3185,14 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) { return v.print(str); } +void forceNoNullByte(std::string_view s) +{ + if (s.find('\0') != s.npos) { + using namespace std::string_view_literals; + auto str = replaceStrings(std::string(s), "\0"sv, "␀"sv); + throw Error("input string '%s' cannot be represented as Nix string because it contains null bytes", str); + } +} + } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 3ac3c8a8a..84b7d823c 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -400,6 +400,11 @@ public: */ void allowPath(const StorePath & storePath); + /** + * Allow access to the closure of a store path. + */ + void allowClosure(const StorePath & storePath); + /** * Allow access to a store path and return it as a string. */ diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 9ac56541a..17cab7ad5 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -50,6 +50,7 @@ class JSONSax : nlohmann::json_sax { public: void key(string_t & name, EvalState & state) { + forceNoNullByte(name); attrs.insert_or_assign(state.symbols.create(name), &value(state)); } }; @@ -122,6 +123,7 @@ public: bool string(string_t & val) override { + forceNoNullByte(val); rs->value(state).mkString(val); rs->add(); return true; diff --git a/src/libexpr/lexer-helpers.hh b/src/libexpr/lexer-helpers.hh index caba6e18f..d40f7b874 100644 --- a/src/libexpr/lexer-helpers.hh +++ b/src/libexpr/lexer-helpers.hh @@ -1,5 +1,13 @@ #pragma once +#include + +// inluding the generated headers twice leads to errors +#ifndef BISON_HEADER +# include "lexer-tab.hh" +# include "parser-tab.hh" +#endif + namespace nix::lexer::internal { void initLoc(YYLTYPE * loc); diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index 28318579e..b3c559ba7 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -14,7 +14,7 @@ project('nix-expr', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -25,7 +25,7 @@ deps_public_maybe_subproject = [ dependency('nix-store'), dependency('nix-fetchers'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') boost = dependency( 'boost', @@ -77,7 +77,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') parser_tab = custom_target( input : 'parser.y', @@ -121,7 +121,7 @@ lexer_tab = custom_target( install_dir : get_option('includedir') / 'nix', ) -subdir('build-utils-meson/generate-header') +subdir('nix-meson-build-support/generate-header') generated_headers = [] foreach header : [ @@ -205,4 +205,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libexpr/nix-meson-build-support b/src/libexpr/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libexpr/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 2950ff1fd..a7ad580d2 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -168,7 +168,7 @@ struct ExprVar : Expr the set stored in the environment that is `level` levels up from the current one.*/ Level level; - Displacement displ; + Displacement displ = 0; ExprVar(Symbol name) : name(name) { }; ExprVar(const PosIdx & pos, Symbol name) : pos(pos), name(name) { }; @@ -242,7 +242,7 @@ struct ExprAttrs : Expr Kind kind; Expr * e; PosIdx pos; - Displacement displ; // displacement + Displacement displ = 0; // displacement AttrDef(Expr * e, const PosIdx & pos, Kind kind = Kind::Plain) : kind(kind), e(e), pos(pos) { }; AttrDef() { }; diff --git a/src/libexpr/package.nix b/src/libexpr/package.nix index d97e7f3a8..5171d70fd 100644 --- a/src/libexpr/package.nix +++ b/src/libexpr/package.nix @@ -40,8 +40,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 7e13e945c..c1f1cf4c1 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -119,11 +119,9 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS if (store != buildStore) copyClosure(*buildStore, *store, outputsToCopyAndAllow); if (isIFD) { - for (auto & outputPath : outputsToCopyAndAllow) { - /* Add the output of this derivations to the allowed - paths. */ - allowPath(outputPath); - } + /* Allow access to the output closures of this derivation. */ + for (auto & outputPath : outputsToCopyAndAllow) + allowClosure(outputPath); } return res; @@ -1100,7 +1098,7 @@ static RegisterPrimOp primop_warn({ .name = "__warn", .args = {"e1", "e2"}, .doc = R"( - Evaluate *e1*, which must be a string and print iton standard error as a warning. + Evaluate *e1*, which must be a string, and print it on standard error as a warning. Then return *e2*. This function is useful for non-critical situations where attention is advisable. @@ -3259,23 +3257,19 @@ static RegisterPrimOp primop_isList({ .fun = prim_isList, }); -static void elemAt(EvalState & state, const PosIdx pos, Value & list, int n, Value & v) -{ - state.forceList(list, pos, "while evaluating the first argument passed to builtins.elemAt"); - if (n < 0 || (unsigned int) n >= list.listSize()) - state.error( - "list index %1% is out of bounds", - n - ).atPos(pos).debugThrow(); - state.forceValue(*list.listElems()[n], pos); - v = *list.listElems()[n]; -} - /* Return the n-1'th element of a list. */ static void prim_elemAt(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - NixInt::Inner elem = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.elemAt").value; - elemAt(state, pos, *args[0], elem, v); + NixInt::Inner n = state.forceInt(*args[1], pos, "while evaluating the second argument passed to 'builtins.elemAt'").value; + state.forceList(*args[0], pos, "while evaluating the first argument passed to 'builtins.elemAt'"); + if (n < 0 || (unsigned int) n >= args[0]->listSize()) + state.error( + "'builtins.elemAt' called with index %d on a list of size %d", + n, + args[0]->listSize() + ).atPos(pos).debugThrow(); + state.forceValue(*args[0]->listElems()[n], pos); + v = *args[0]->listElems()[n]; } static RegisterPrimOp primop_elemAt({ @@ -3291,7 +3285,13 @@ static RegisterPrimOp primop_elemAt({ /* Return the first element of a list. */ static void prim_head(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - elemAt(state, pos, *args[0], 0, v); + state.forceList(*args[0], pos, "while evaluating the first argument passed to 'builtins.head'"); + if (args[0]->listSize() == 0) + state.error( + "'builtins.head' called on an empty list" + ).atPos(pos).debugThrow(); + state.forceValue(*args[0]->listElems()[0], pos); + v = *args[0]->listElems()[0]; } static RegisterPrimOp primop_head({ @@ -3310,9 +3310,9 @@ static RegisterPrimOp primop_head({ don't want to use it! */ static void prim_tail(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceList(*args[0], pos, "while evaluating the first argument passed to builtins.tail"); + state.forceList(*args[0], pos, "while evaluating the first argument passed to 'builtins.tail'"); if (args[0]->listSize() == 0) - state.error("'tail' called on an empty list").atPos(pos).debugThrow(); + state.error("'builtins.tail' called on an empty list").atPos(pos).debugThrow(); auto list = state.buildList(args[0]->listSize() - 1); for (const auto & [n, v] : enumerate(list)) diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index 264046711..404425054 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -28,8 +28,10 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V auto attrs = state.buildBindings(size); - for(auto & elem : table) + for(auto & elem : table) { + forceNoNullByte(elem.first); visit(attrs.alloc(elem.first), elem.second); + } v.mkAttrs(attrs); } @@ -54,7 +56,11 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V v.mkFloat(toml::get(t)); break;; case toml::value_t::string: - v.mkString(toml::get(t)); + { + auto s = toml::get(t); + forceNoNullByte(s); + v.mkString(s); + } break;; case toml::value_t::local_datetime: case toml::value_t::offset_datetime: @@ -66,7 +72,9 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V attrs.alloc("_type").mkString("timestamp"); std::ostringstream s; s << t; - attrs.alloc("value").mkString(toView(s)); + auto str = toView(s); + forceNoNullByte(str); + attrs.alloc("value").mkString(str); v.mkAttrs(attrs); } else { throw std::runtime_error("Dates and times are not supported"); diff --git a/src/libexpr/print-options.hh b/src/libexpr/print-options.hh index 080ba26b8..9ad54e532 100644 --- a/src/libexpr/print-options.hh +++ b/src/libexpr/print-options.hh @@ -5,6 +5,7 @@ */ #include +#include namespace nix { diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index d98161488..88fcae986 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -510,4 +510,6 @@ typedef std::shared_ptr RootValue; RootValue allocRootValue(Value * v); +void forceNoNullByte(std::string_view s); + } diff --git a/src/libfetchers-tests/build-utils-meson b/src/libfetchers-tests/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libfetchers-tests/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libfetchers-tests/meson.build b/src/libfetchers-tests/meson.build index fdab6ba6c..3e82c6111 100644 --- a/src/libfetchers-tests/meson.build +++ b/src/libfetchers-tests/meson.build @@ -14,7 +14,7 @@ project('nix-fetchers-tests', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') deps_private_maybe_subproject = [ dependency('nix-store-test-support'), @@ -22,10 +22,10 @@ deps_private_maybe_subproject = [ ] deps_public_maybe_subproject = [ ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') rapidcheck = dependency('rapidcheck') deps_private += rapidcheck @@ -42,7 +42,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'public-key.cc', diff --git a/src/libfetchers-tests/nix-meson-build-support b/src/libfetchers-tests/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libfetchers-tests/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libfetchers-tests/package.nix b/src/libfetchers-tests/package.nix index 7b2ba8f2c..e18d644ed 100644 --- a/src/libfetchers-tests/package.nix +++ b/src/libfetchers-tests/package.nix @@ -26,8 +26,8 @@ mkMesonExecutable (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libfetchers/build-utils-meson b/src/libfetchers/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libfetchers/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index e15a460d0..b105c252a 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -113,7 +113,15 @@ Input Input::fromAttrs(const Settings & settings, Attrs && attrs) std::optional Input::getFingerprint(ref store) const { - return scheme ? scheme->getFingerprint(store, *this) : std::nullopt; + if (!scheme) return std::nullopt; + + if (cachedFingerprint) return *cachedFingerprint; + + auto fingerprint = scheme->getFingerprint(store, *this); + + cachedFingerprint = fingerprint; + + return fingerprint; } ParsedURL Input::toURL() const @@ -307,7 +315,7 @@ std::pair, Input> Input::getAccessorUnchecked(ref sto auto accessor = makeStorePathAccessor(store, storePath); - accessor->fingerprint = scheme->getFingerprint(store, *this); + accessor->fingerprint = getFingerprint(store); return {accessor, *this}; } catch (Error & e) { @@ -318,7 +326,7 @@ std::pair, Input> Input::getAccessorUnchecked(ref sto auto [accessor, result] = scheme->getAccessor(store, *this); assert(!accessor->fingerprint); - accessor->fingerprint = scheme->getFingerprint(store, result); + accessor->fingerprint = result.getFingerprint(store); return {accessor, std::move(result)}; } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index b28ec4568..841a44041 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -46,6 +46,11 @@ struct Input */ std::optional parent; + /** + * Cached result of getFingerprint(). + */ + mutable std::optional> cachedFingerprint; + public: /** * Create an `Input` from a URL. @@ -104,6 +109,11 @@ public: bool operator ==(const Input & other) const noexcept; + bool operator <(const Input & other) const + { + return attrs < other.attrs; + } + bool contains(const Input & other) const; /** diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 8aac2361c..69af966e0 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -6,6 +6,7 @@ #include "signals.hh" #include "users.hh" #include "fs-sink.hh" +#include "sync.hh" #include #include @@ -438,7 +439,12 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this { if (!(statusFlags & GIT_STATUS_INDEX_DELETED) && !(statusFlags & GIT_STATUS_WT_DELETED)) + { info.files.insert(CanonPath(path)); + if (statusFlags != GIT_STATUS_CURRENT) + info.dirtyFiles.insert(CanonPath(path)); + } else + info.deletedFiles.insert(CanonPath(path)); if (statusFlags != GIT_STATUS_CURRENT) info.isDirty = true; return 0; @@ -1311,4 +1317,17 @@ ref getTarballCache() return GitRepo::openRepo(repoDir, true, true); } +GitRepo::WorkdirInfo GitRepo::getCachedWorkdirInfo(const std::filesystem::path & path) +{ + static Sync> _cache; + { + auto cache(_cache.lock()); + auto i = cache->find(path); + if (i != cache->end()) return i->second; + } + auto workdirInfo = GitRepo::openRepo(path)->getWorkdirInfo(); + _cache.lock()->emplace(path, workdirInfo); + return workdirInfo; +} + } diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh index 795af470e..ffa23678d 100644 --- a/src/libfetchers/git-utils.hh +++ b/src/libfetchers/git-utils.hh @@ -59,12 +59,20 @@ struct GitRepo modified or added, but excluding deleted files. */ std::set files; + /* All modified or added files. */ + std::set dirtyFiles; + + /* The deleted files. */ + std::set deletedFiles; + /* The submodules listed in .gitmodules of this workdir. */ std::vector submodules; }; virtual WorkdirInfo getWorkdirInfo() = 0; + static WorkdirInfo getCachedWorkdirInfo(const std::filesystem::path & path); + /* Get the ref that HEAD points to. */ virtual std::optional getWorkdirRef() = 0; diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 6704ace9f..79f2faa98 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -15,6 +15,7 @@ #include "finally.hh" #include "fetch-settings.hh" #include "json-utils.hh" +#include "archive.hh" #include #include @@ -438,7 +439,7 @@ struct GitInputScheme : InputScheme // If this is a local directory and no ref or revision is // given, then allow the use of an unclean working tree. if (!input.getRef() && !input.getRev() && repoInfo.isLocal) - repoInfo.workdirInfo = GitRepo::openRepo(repoInfo.url)->getWorkdirInfo(); + repoInfo.workdirInfo = GitRepo::getCachedWorkdirInfo(repoInfo.url); return repoInfo; } @@ -803,12 +804,33 @@ struct GitInputScheme : InputScheme std::optional getFingerprint(ref store, const Input & input) const override { - if (auto rev = input.getRev()) + auto makeFingerprint = [&](const Hash & rev) { - return rev->gitRev() + (getSubmodulesAttr(input) ? ";s" : "") + (getExportIgnoreAttr(input) ? ";e" : "") + (getLfsAttr(input) ? ";l" : ""); - } - else + return rev.gitRev() + (getSubmodulesAttr(input) ? ";s" : "") + (getExportIgnoreAttr(input) ? ";e" : "") + (getLfsAttr(input) ? ";l" : ""); + }; + + if (auto rev = input.getRev()) + return makeFingerprint(*rev); + else { + auto repoInfo = getRepoInfo(input); + if (repoInfo.isLocal && repoInfo.workdirInfo.headRev && repoInfo.workdirInfo.submodules.empty()) { + /* Calculate a fingerprint that takes into account the + deleted and modified/added files. */ + HashSink hashSink{HashAlgorithm::SHA512}; + for (auto & file : repoInfo.workdirInfo.dirtyFiles) { + writeString("modified:", hashSink); + writeString(file.abs(), hashSink); + dumpPath(repoInfo.url + "/" + file.abs(), hashSink); + } + for (auto & file : repoInfo.workdirInfo.deletedFiles) { + writeString("deleted:", hashSink); + writeString(file.abs(), hashSink); + } + return makeFingerprint(*repoInfo.workdirInfo.headRev) + + ";d=" + hashSink.finish().first.to_string(HashFormat::Base16, false); + } return std::nullopt; + } } bool isLocked(const Input & input) const override diff --git a/src/libfetchers/meson.build b/src/libfetchers/meson.build index 73295e561..ae4cab4c7 100644 --- a/src/libfetchers/meson.build +++ b/src/libfetchers/meson.build @@ -14,7 +14,7 @@ project('nix-fetchers', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -24,7 +24,7 @@ deps_public_maybe_subproject = [ dependency('nix-util'), dependency('nix-store'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') deps_public += nlohmann_json @@ -44,7 +44,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'attrs.cc', @@ -94,4 +94,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libfetchers/nix-meson-build-support b/src/libfetchers/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libfetchers/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libfetchers/package.nix b/src/libfetchers/package.nix index 4229998d9..bcbde67d1 100644 --- a/src/libfetchers/package.nix +++ b/src/libfetchers/package.nix @@ -23,8 +23,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index c761028ab..171afcea7 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -178,7 +178,8 @@ Registries getRegistries(const Settings & settings, ref store) std::pair lookupInRegistries( ref store, - const Input & _input) + const Input & _input, + const RegistryFilter & filter) { Attrs extraAttrs; int n = 0; @@ -190,6 +191,7 @@ std::pair lookupInRegistries( if (n > 100) throw Error("cycle detected in flake registry for '%s'", input.to_string()); for (auto & registry : getRegistries(*input.settings, store)) { + if (filter && !filter(registry->type)) continue; // FIXME: O(n) for (auto & entry : registry->entries) { if (entry.exact) { diff --git a/src/libfetchers/registry.hh b/src/libfetchers/registry.hh index 0d68ac395..8f47e1590 100644 --- a/src/libfetchers/registry.hh +++ b/src/libfetchers/registry.hh @@ -65,8 +65,15 @@ void overrideRegistry( const Input & to, const Attrs & extraAttrs); +using RegistryFilter = std::function; + +/** + * Rewrite a flakeref using the registries. If `filter` is set, only + * use the registries for which the filter function returns true. + */ std::pair lookupInRegistries( ref store, - const Input & input); + const Input & input, + const RegistryFilter & filter = {}); } diff --git a/src/libflake-c/build-utils-meson b/src/libflake-c/build-utils-meson deleted file mode 120000 index 91937f183..000000000 --- a/src/libflake-c/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson/ \ No newline at end of file diff --git a/src/libflake-c/meson.build b/src/libflake-c/meson.build index 00d9650e7..b7669fe97 100644 --- a/src/libflake-c/meson.build +++ b/src/libflake-c/meson.build @@ -14,7 +14,7 @@ project('nix-flake-c', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -29,7 +29,7 @@ deps_public_maybe_subproject = [ dependency('nix-store-c'), dependency('nix-expr-c'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') # TODO rename, because it will conflict with downstream projects configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) @@ -58,7 +58,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'nix_api_flake.cc', @@ -73,8 +73,8 @@ headers = [config_h] + files( # TODO move this header to libexpr, maybe don't use it in tests? headers += files('nix_api_flake.h') -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') this_library = library( 'nixflakec', @@ -90,4 +90,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libflake-c/nix-meson-build-support b/src/libflake-c/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libflake-c/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libflake-c/package.nix b/src/libflake-c/package.nix index a70cbf94e..7425d6140 100644 --- a/src/libflake-c/package.nix +++ b/src/libflake-c/package.nix @@ -21,8 +21,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libflake-tests/build-utils-meson b/src/libflake-tests/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libflake-tests/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libflake-tests/meson.build b/src/libflake-tests/meson.build index c494c414e..5c3c58e53 100644 --- a/src/libflake-tests/meson.build +++ b/src/libflake-tests/meson.build @@ -14,7 +14,7 @@ project('nix-flake-tests', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') deps_private_maybe_subproject = [ dependency('nix-expr-test-support'), @@ -23,10 +23,10 @@ deps_private_maybe_subproject = [ ] deps_public_maybe_subproject = [ ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') rapidcheck = dependency('rapidcheck') deps_private += rapidcheck @@ -43,7 +43,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'flakeref.cc', diff --git a/src/libflake-tests/nix-meson-build-support b/src/libflake-tests/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libflake-tests/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libflake-tests/package.nix b/src/libflake-tests/package.nix index b3a8ac466..f1abbb32d 100644 --- a/src/libflake-tests/package.nix +++ b/src/libflake-tests/package.nix @@ -27,8 +27,8 @@ mkMesonExecutable (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libflake/build-utils-meson b/src/libflake/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libflake/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index 19b622a34..29090b900 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -21,23 +21,23 @@ using namespace flake; namespace flake { -typedef std::pair FetchedFlake; -typedef std::vector> FlakeCache; +struct FetchedFlake +{ + FlakeRef lockedRef; + StorePath storePath; +}; + +typedef std::map FlakeCache; static std::optional lookupInFlakeCache( const FlakeCache & flakeCache, const FlakeRef & flakeRef) { - // FIXME: inefficient. - for (auto & i : flakeCache) { - if (flakeRef == i.first) { - debug("mapping '%s' to previously seen input '%s' -> '%s", - flakeRef, i.first, i.second.second); - return i.second; - } - } - - return std::nullopt; + auto i = flakeCache.find(flakeRef); + if (i == flakeCache.end()) return std::nullopt; + debug("mapping '%s' to previously seen input '%s' -> '%s", + flakeRef, i->first, i->second.lockedRef); + return i->second; } static std::tuple fetchOrSubstituteTree( @@ -51,32 +51,39 @@ static std::tuple fetchOrSubstituteTree( if (!fetched) { if (originalRef.input.isDirect()) { - fetched.emplace(originalRef.fetchTree(state.store)); + auto [storePath, lockedRef] = originalRef.fetchTree(state.store); + fetched.emplace(FetchedFlake{.lockedRef = lockedRef, .storePath = storePath}); } else { if (allowLookup) { - resolvedRef = originalRef.resolve(state.store); - auto fetchedResolved = lookupInFlakeCache(flakeCache, originalRef); - if (!fetchedResolved) fetchedResolved.emplace(resolvedRef.fetchTree(state.store)); - flakeCache.push_back({resolvedRef, *fetchedResolved}); - fetched.emplace(*fetchedResolved); + resolvedRef = originalRef.resolve( + state.store, + [](fetchers::Registry::RegistryType type) { + /* Only use the global registry and CLI flags + to resolve indirect flakerefs. */ + return type == fetchers::Registry::Flag || type == fetchers::Registry::Global; + }); + fetched = lookupInFlakeCache(flakeCache, originalRef); + if (!fetched) { + auto [storePath, lockedRef] = resolvedRef.fetchTree(state.store); + fetched.emplace(FetchedFlake{.lockedRef = lockedRef, .storePath = storePath}); + } + flakeCache.insert_or_assign(resolvedRef, *fetched); } else { throw Error("'%s' is an indirect flake reference, but registry lookups are not allowed", originalRef); } } - flakeCache.push_back({originalRef, *fetched}); + flakeCache.insert_or_assign(originalRef, *fetched); } - auto [storePath, lockedRef] = *fetched; - debug("got tree '%s' from '%s'", - state.store->printStorePath(storePath), lockedRef); + state.store->printStorePath(fetched->storePath), fetched->lockedRef); - state.allowPath(storePath); + state.allowPath(fetched->storePath); - assert(!originalRef.input.getNarHash() || storePath == originalRef.input.computeStorePath(*state.store)); + assert(!originalRef.input.getNarHash() || fetched->storePath == originalRef.input.computeStorePath(*state.store)); - return {std::move(storePath), resolvedRef, lockedRef}; + return {fetched->storePath, resolvedRef, fetched->lockedRef}; } static void forceTrivialValue(EvalState & state, Value & value, const PosIdx pos) @@ -672,6 +679,8 @@ LockedFlake lockFlake( if (lockFlags.writeLockFile) { if (sourcePath || lockFlags.outputLockFilePath) { if (auto unlockedInput = newLockFile.isUnlocked()) { + if (lockFlags.failOnUnlocked) + throw Error("cannot write lock file of flake '%s' because it has an unlocked input ('%s').\n", topRef, *unlockedInput); if (state.fetchSettings.warnDirty) warn("will not write lock file of flake '%s' because it has an unlocked input ('%s')", topRef, *unlockedInput); } else { diff --git a/src/libflake/flake/flake.hh b/src/libflake/flake/flake.hh index cc2bea76e..0dfd9440d 100644 --- a/src/libflake/flake/flake.hh +++ b/src/libflake/flake/flake.hh @@ -156,6 +156,11 @@ struct LockFlags */ bool writeLockFile = true; + /** + * Throw an exception when the flake has an unlocked input. + */ + bool failOnUnlocked = false; + /** * Whether to use the registries to lookup indirect flake * references like 'nixpkgs'. diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flake/flakeref.cc index 9616fe0ea..ab882fdab 100644 --- a/src/libflake/flake/flakeref.cc +++ b/src/libflake/flake/flakeref.cc @@ -3,7 +3,6 @@ #include "url.hh" #include "url-parts.hh" #include "fetchers.hh" -#include "registry.hh" namespace nix { @@ -36,7 +35,9 @@ std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef) return str; } -FlakeRef FlakeRef::resolve(ref store) const +FlakeRef FlakeRef::resolve( + ref store, + const fetchers::RegistryFilter & filter) const { auto [input2, extraAttrs] = lookupInRegistries(store, input); return FlakeRef(std::move(input2), fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir)); diff --git a/src/libflake/flake/flakeref.hh b/src/libflake/flake/flakeref.hh index 1064538a7..ec755399d 100644 --- a/src/libflake/flake/flakeref.hh +++ b/src/libflake/flake/flakeref.hh @@ -6,6 +6,7 @@ #include "types.hh" #include "fetchers.hh" #include "outputs-spec.hh" +#include "registry.hh" namespace nix { @@ -48,6 +49,11 @@ struct FlakeRef bool operator ==(const FlakeRef & other) const = default; + bool operator <(const FlakeRef & other) const + { + return std::tie(input, subdir) < std::tie(other.input, other.subdir); + } + FlakeRef(fetchers::Input && input, const Path & subdir) : input(std::move(input)), subdir(subdir) { } @@ -57,7 +63,9 @@ struct FlakeRef fetchers::Attrs toAttrs() const; - FlakeRef resolve(ref store) const; + FlakeRef resolve( + ref store, + const fetchers::RegistryFilter & filter = {}) const; static FlakeRef fromAttrs( const fetchers::Settings & fetchSettings, diff --git a/src/libflake/meson.build b/src/libflake/meson.build index 2c1a70a18..f9e217729 100644 --- a/src/libflake/meson.build +++ b/src/libflake/meson.build @@ -14,7 +14,7 @@ project('nix-flake', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') deps_private_maybe_subproject = [ ] @@ -24,7 +24,7 @@ deps_public_maybe_subproject = [ dependency('nix-fetchers'), dependency('nix-expr'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') deps_public += nlohmann_json @@ -39,7 +39,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'flake/config.cc', @@ -72,4 +72,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libflake/nix-meson-build-support b/src/libflake/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libflake/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libflake/package.nix b/src/libflake/package.nix index fff481720..92445739f 100644 --- a/src/libflake/package.nix +++ b/src/libflake/package.nix @@ -23,8 +23,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libmain-c/build-utils-meson b/src/libmain-c/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libmain-c/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libmain-c/meson.build b/src/libmain-c/meson.build index 3cb1e4baa..5a5684b8d 100644 --- a/src/libmain-c/meson.build +++ b/src/libmain-c/meson.build @@ -14,7 +14,7 @@ project('nix-main-c', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -27,7 +27,7 @@ deps_public_maybe_subproject = [ dependency('nix-util-c'), dependency('nix-store-c'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') # TODO rename, because it will conflict with downstream projects configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) @@ -53,7 +53,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'nix_api_main.cc', @@ -65,8 +65,8 @@ headers = [config_h] + files( 'nix_api_main.h', ) -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') this_library = library( 'nixmainc', @@ -82,4 +82,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libmain-c/nix-meson-build-support b/src/libmain-c/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libmain-c/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libmain-c/package.nix b/src/libmain-c/package.nix index 5522037f3..d65792921 100644 --- a/src/libmain-c/package.nix +++ b/src/libmain-c/package.nix @@ -22,8 +22,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libmain/build-utils-meson b/src/libmain/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libmain/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libmain/meson.build b/src/libmain/meson.build index 6c6298e2b..87fc8b8d2 100644 --- a/src/libmain/meson.build +++ b/src/libmain/meson.build @@ -14,7 +14,7 @@ project('nix-main', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -24,7 +24,7 @@ deps_public_maybe_subproject = [ dependency('nix-util'), dependency('nix-store'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') pubsetbuf_test = ''' #include @@ -58,7 +58,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'common-args.cc', @@ -96,4 +96,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libmain/nix-meson-build-support b/src/libmain/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libmain/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libmain/package.nix b/src/libmain/package.nix index 7e7b80472..7d9d99b61 100644 --- a/src/libmain/package.nix +++ b/src/libmain/package.nix @@ -22,8 +22,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libstore-c/build-utils-meson b/src/libstore-c/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libstore-c/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libstore-c/meson.build b/src/libstore-c/meson.build index 44b5fe11d..1ac331ad0 100644 --- a/src/libstore-c/meson.build +++ b/src/libstore-c/meson.build @@ -14,7 +14,7 @@ project('nix-store-c', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -25,7 +25,7 @@ deps_private_maybe_subproject = [ deps_public_maybe_subproject = [ dependency('nix-util-c'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') # TODO rename, because it will conflict with downstream projects configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) @@ -49,7 +49,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'nix_api_store.cc', @@ -64,8 +64,8 @@ headers = [config_h] + files( # TODO don't install this once tests don't use it and/or move the header into `libstore`, non-`c` headers += files('nix_api_store_internal.h') -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') this_library = library( 'nixstorec', @@ -81,4 +81,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libstore-c/nix-meson-build-support b/src/libstore-c/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libstore-c/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libstore-c/nix_api_store.cc b/src/libstore-c/nix_api_store.cc index fb7391276..bc306e0d0 100644 --- a/src/libstore-c/nix_api_store.cc +++ b/src/libstore-c/nix_api_store.cc @@ -67,6 +67,17 @@ nix_err nix_store_get_uri(nix_c_context * context, Store * store, nix_get_string NIXC_CATCH_ERRS } +nix_err +nix_store_get_storedir(nix_c_context * context, Store * store, nix_get_string_callback callback, void * user_data) +{ + if (context) + context->last_err_code = NIX_OK; + try { + return call_nix_get_string_callback(store->ptr->storeDir, callback, user_data); + } + NIXC_CATCH_ERRS +} + nix_err nix_store_get_version(nix_c_context * context, Store * store, nix_get_string_callback callback, void * user_data) { @@ -89,6 +100,18 @@ bool nix_store_is_valid_path(nix_c_context * context, Store * store, StorePath * NIXC_CATCH_ERRS_RES(false); } +nix_err nix_store_real_path( + nix_c_context * context, Store * store, StorePath * path, nix_get_string_callback callback, void * user_data) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto res = store->ptr->toRealPath(path->path); + return call_nix_get_string_callback(res, callback, user_data); + } + NIXC_CATCH_ERRS +} + StorePath * nix_store_parse_path(nix_c_context * context, Store * store, const char * path) { if (context) diff --git a/src/libstore-c/nix_api_store.h b/src/libstore-c/nix_api_store.h index 282ccc285..e55bc3f59 100644 --- a/src/libstore-c/nix_api_store.h +++ b/src/libstore-c/nix_api_store.h @@ -48,12 +48,27 @@ nix_err nix_libstore_init_no_load_config(nix_c_context * context); * Store instances may share state and resources behind the scenes. * * @param[out] context Optional, stores error information - * @param[in] uri URI of the Nix store, copied. See [*Store URL format* in the Nix Reference + * + * @param[in] uri @parblock + * URI of the Nix store, copied. + * + * If `NULL`, the store from the settings will be used. + * Note that `"auto"` holds a strange middle ground, reading part of the general environment, but not all of it. It + * ignores `NIX_REMOTE` and the `store` option. For this reason, `NULL` is most likely the better choice. + * + * For supported store URLs, see [*Store URL format* in the Nix Reference * Manual](https://nixos.org/manual/nix/stable/store/types/#store-url-format). - * @param[in] params optional, null-terminated array of key-value pairs, e.g. {{"endpoint", - * "https://s3.local"}}. See [*Store Types* in the Nix Reference - * Manual](https://nixos.org/manual/nix/stable/store/types). + * @endparblock + * + * @param[in] params @parblock + * optional, null-terminated array of key-value pairs, e.g. {{"endpoint", + * "https://s3.local"}}. + * + * See [*Store Types* in the Nix Reference Manual](https://nixos.org/manual/nix/stable/store/types). + * @endparblock + * * @return a Store pointer, NULL in case of errors + * * @see nix_store_free */ Store * nix_store_open(nix_c_context * context, const char * uri, const char *** params); @@ -78,7 +93,18 @@ void nix_store_free(Store * store); */ nix_err nix_store_get_uri(nix_c_context * context, Store * store, nix_get_string_callback callback, void * user_data); -// returns: owned StorePath* +/** + * @brief get the storeDir of a Nix store, typically `"/nix/store"` + * @param[out] context Optional, stores error information + * @param[in] store nix store reference + * @param[in] callback Called with the URI. + * @param[in] user_data optional, arbitrary data, passed to the callback when it's called. + * @see nix_get_string_callback + * @return error code, NIX_OK on success. + */ +nix_err +nix_store_get_storedir(nix_c_context * context, Store * store, nix_get_string_callback callback, void * user_data); + /** * @brief Parse a Nix store path into a StorePath * @@ -123,6 +149,26 @@ void nix_store_path_free(StorePath * p); * @return true or false, error info in context */ bool nix_store_is_valid_path(nix_c_context * context, Store * store, StorePath * path); + +/** + * @brief Get the physical location of a store path + * + * A store may reside at a different location than its `storeDir` suggests. + * This situation is called a relocated store. + * Relocated stores are used during NixOS installation, as well as in restricted computing environments that don't offer + * a writable `/nix/store`. + * + * Not all types of stores support this operation. + * + * @param[in] context Optional, stores error information + * @param[in] store nix store reference + * @param[in] path the path to get the real path from + * @param[in] callback called with the real path + * @param[in] user_data arbitrary data, passed to the callback when it's called. + */ +nix_err nix_store_real_path( + nix_c_context * context, Store * store, StorePath * path, nix_get_string_callback callback, void * user_data); + // nix_err nix_store_ensure(Store*, const char*); // nix_err nix_store_build_paths(Store*); /** diff --git a/src/libstore-c/package.nix b/src/libstore-c/package.nix index 896a1a39f..351d4510a 100644 --- a/src/libstore-c/package.nix +++ b/src/libstore-c/package.nix @@ -20,8 +20,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libstore-test-support/build-utils-meson b/src/libstore-test-support/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libstore-test-support/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libstore-test-support/meson.build b/src/libstore-test-support/meson.build index 1f230914f..2a07e56ac 100644 --- a/src/libstore-test-support/meson.build +++ b/src/libstore-test-support/meson.build @@ -14,7 +14,7 @@ project('nix-store-test-support', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') deps_private_maybe_subproject = [ ] @@ -24,7 +24,7 @@ deps_public_maybe_subproject = [ dependency('nix-store'), dependency('nix-store-c'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') rapidcheck = dependency('rapidcheck') deps_public += rapidcheck @@ -37,7 +37,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'tests/derived-path.cc', @@ -56,8 +56,8 @@ headers = files( 'tests/protocol.hh', ) -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') this_library = library( 'nix-store-test-support', @@ -75,4 +75,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libstore-test-support/nix-meson-build-support b/src/libstore-test-support/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libstore-test-support/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libstore-test-support/package.nix b/src/libstore-test-support/package.nix index 2543049fe..62f783c4f 100644 --- a/src/libstore-test-support/package.nix +++ b/src/libstore-test-support/package.nix @@ -23,8 +23,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libstore-test-support/tests/libstore.hh b/src/libstore-test-support/tests/libstore.hh index 84be52c23..699ba957e 100644 --- a/src/libstore-test-support/tests/libstore.hh +++ b/src/libstore-test-support/tests/libstore.hh @@ -19,12 +19,12 @@ public: protected: LibStoreTest() : store(openStore({ - .variant = - StoreReference::Specified{ - .scheme = "dummy", - }, - .params = {}, - })) + .variant = + StoreReference::Specified{ + .scheme = "dummy", + }, + .params = {}, + })) { } diff --git a/src/libstore-tests/build-utils-meson b/src/libstore-tests/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libstore-tests/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index fc9152f2f..b706fa12c 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -14,19 +14,21 @@ project('nix-store-tests', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') + +nix_store = dependency('nix-store') deps_private_maybe_subproject = [ - dependency('nix-store'), + nix_store, dependency('nix-store-c'), dependency('nix-store-test-support'), ] deps_public_maybe_subproject = [ ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') sqlite = dependency('sqlite3', 'sqlite', version : '>=3.6.19') deps_private += sqlite @@ -50,7 +52,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'common-protocol.cc', @@ -90,6 +92,7 @@ this_exe = executable( include_directories : include_dirs, # TODO: -lrapidcheck, see ../libutil-support/build.meson link_args: linker_export_flags + ['-lrapidcheck'], + cpp_args : [ '-DNIX_STORE_DIR="' + nix_store.get_variable('storedir') + '"' ], # get main from gtest install : true, ) diff --git a/src/libstore-tests/nix-meson-build-support b/src/libstore-tests/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libstore-tests/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libstore-tests/nix_api_store.cc b/src/libstore-tests/nix_api_store.cc index 7c6ec0780..a8b7b8e5f 100644 --- a/src/libstore-tests/nix_api_store.cc +++ b/src/libstore-tests/nix_api_store.cc @@ -24,6 +24,39 @@ TEST_F(nix_api_store_test, nix_store_get_uri) ASSERT_STREQ("local", str.c_str()); } +TEST_F(nix_api_util_context, nix_store_get_storedir_default) +{ + if (nix::getEnv("HOME").value_or("") == "/homeless-shelter") { + // skipping test in sandbox because nix_store_open tries to create /nix/var/nix/profiles + GTEST_SKIP(); + } + nix_libstore_init(ctx); + Store * store = nix_store_open(ctx, nullptr, nullptr); + assert_ctx_ok(); + ASSERT_NE(store, nullptr); + + std::string str; + auto ret = nix_store_get_storedir(ctx, store, OBSERVE_STRING(str)); + assert_ctx_ok(); + ASSERT_EQ(NIX_OK, ret); + + // These tests run with a unique storeDir, but not a relocated store + ASSERT_STREQ(NIX_STORE_DIR, str.c_str()); + + nix_store_free(store); +} + +TEST_F(nix_api_store_test, nix_store_get_storedir) +{ + std::string str; + auto ret = nix_store_get_storedir(ctx, store, OBSERVE_STRING(str)); + assert_ctx_ok(); + ASSERT_EQ(NIX_OK, ret); + + // These tests run with a unique storeDir, but not a relocated store + ASSERT_STREQ(nixStoreDir.c_str(), str.c_str()); +} + TEST_F(nix_api_store_test, InvalidPathFails) { nix_store_parse_path(ctx, store, "invalid-path"); @@ -86,4 +119,84 @@ TEST_F(nix_api_store_test, nix_store_is_valid_path_not_in_store) ASSERT_EQ(false, nix_store_is_valid_path(ctx, store, path)); } +TEST_F(nix_api_store_test, nix_store_real_path) +{ + StorePath * path = nix_store_parse_path(ctx, store, (nixStoreDir + PATH_SUFFIX).c_str()); + std::string rp; + auto ret = nix_store_real_path(ctx, store, path, OBSERVE_STRING(rp)); + assert_ctx_ok(); + ASSERT_EQ(NIX_OK, ret); + // Assumption: we're not testing with a relocated store + ASSERT_STREQ((nixStoreDir + PATH_SUFFIX).c_str(), rp.c_str()); + + nix_store_path_free(path); } + +TEST_F(nix_api_util_context, nix_store_real_path_relocated) +{ + if (nix::getEnv("HOME").value_or("") == "/homeless-shelter") { + // Can't open default store from within sandbox + GTEST_SKIP(); + } + auto tmp = nix::createTempDir(); + std::string storeRoot = tmp + "/store"; + std::string stateDir = tmp + "/state"; + std::string logDir = tmp + "/log"; + const char * rootkv[] = {"root", storeRoot.c_str()}; + const char * statekv[] = {"state", stateDir.c_str()}; + const char * logkv[] = {"log", logDir.c_str()}; + // const char * rokv[] = {"read-only", "true"}; + const char ** kvs[] = {rootkv, statekv, logkv, NULL}; + + nix_libstore_init(ctx); + assert_ctx_ok(); + + Store * store = nix_store_open(ctx, "local", kvs); + assert_ctx_ok(); + ASSERT_NE(store, nullptr); + + std::string nixStoreDir; + auto ret = nix_store_get_storedir(ctx, store, OBSERVE_STRING(nixStoreDir)); + ASSERT_EQ(NIX_OK, ret); + ASSERT_STREQ(NIX_STORE_DIR, nixStoreDir.c_str()); + + StorePath * path = nix_store_parse_path(ctx, store, (nixStoreDir + PATH_SUFFIX).c_str()); + assert_ctx_ok(); + ASSERT_NE(path, nullptr); + + std::string rp; + ret = nix_store_real_path(ctx, store, path, OBSERVE_STRING(rp)); + assert_ctx_ok(); + ASSERT_EQ(NIX_OK, ret); + + // Assumption: we're not testing with a relocated store + ASSERT_STREQ((storeRoot + NIX_STORE_DIR + PATH_SUFFIX).c_str(), rp.c_str()); + + nix_store_path_free(path); +} + +TEST_F(nix_api_util_context, nix_store_real_path_binary_cache) +{ + if (nix::getEnv("HOME").value_or("") == "/homeless-shelter") { + // TODO: override NIX_CACHE_HOME? + // skipping test in sandbox because narinfo cache can't be written + GTEST_SKIP(); + } + + Store * store = nix_store_open(ctx, "https://cache.nixos.org", nullptr); + assert_ctx_ok(); + ASSERT_NE(store, nullptr); + + std::string path_raw = std::string(NIX_STORE_DIR) + PATH_SUFFIX; + StorePath * path = nix_store_parse_path(ctx, store, path_raw.c_str()); + assert_ctx_ok(); + ASSERT_NE(path, nullptr); + + std::string rp; + auto ret = nix_store_real_path(ctx, store, path, OBSERVE_STRING(rp)); + assert_ctx_ok(); + ASSERT_EQ(NIX_OK, ret); + ASSERT_STREQ(path_raw.c_str(), rp.c_str()); +} + +} // namespace nixC diff --git a/src/libstore-tests/package.nix b/src/libstore-tests/package.nix index 3704d8c5c..4937b5329 100644 --- a/src/libstore-tests/package.nix +++ b/src/libstore-tests/package.nix @@ -28,8 +28,8 @@ mkMesonExecutable (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index e8c8892b3..896779f85 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -39,15 +39,13 @@ BinaryCacheStore::BinaryCacheStore(const Params & params) void BinaryCacheStore::init() { - std::string cacheInfoFile = "nix-cache-info"; - - auto cacheInfo = getFile(cacheInfoFile); + auto cacheInfo = getNixCacheInfo(); if (!cacheInfo) { upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", "text/x-nix-cache-info"); } else { for (auto & line : tokenizeString(*cacheInfo, "\n")) { - size_t colon= line.find(':'); - if (colon ==std::string::npos) continue; + size_t colon = line.find(':'); + if (colon == std::string::npos) continue; auto name = line.substr(0, colon); auto value = trim(line.substr(colon + 1, std::string::npos)); if (name == "StoreDir") { @@ -63,6 +61,11 @@ void BinaryCacheStore::init() } } +std::optional BinaryCacheStore::getNixCacheInfo() +{ + return getFile(cacheInfoFile); +} + void BinaryCacheStore::upsertFile(const std::string & path, std::string && data, const std::string & mimeType) diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 695bc9252..6bd7fd14a 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -64,6 +64,8 @@ protected: // The prefix under which realisation infos will be stored const std::string realisationsPrefix = "realisations"; + const std::string cacheInfoFile = "nix-cache-info"; + BinaryCacheStore(const Params & params); public: @@ -84,6 +86,12 @@ public: */ virtual void getFile(const std::string & path, Sink & sink); + /** + * Get the contents of /nix-cache-info. Return std::nullopt if it + * doesn't exist. + */ + virtual std::optional getNixCacheInfo(); + /** * Fetch the specified file and call the specified callback with * the result. A subclass may implement this asynchronously. diff --git a/src/libstore/build-utils-meson b/src/libstore/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libstore/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index bf1a25db1..2ff0ef92f 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1161,7 +1161,7 @@ HookReply DerivationGoal::tryBuildHook() throw; } }(); - if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true)) + if (handleJSONLogMessage(s, worker.act, worker.hook->activities, "the build hook", true)) ; else if (s.substr(0, 2) == "# ") { reply = s.substr(2); @@ -1346,9 +1346,9 @@ void DerivationGoal::handleChildOutput(Descriptor fd, std::string_view data) if (hook && fd == hook->fromHook.readSide.get()) { for (auto c : data) if (c == '\n') { - auto json = parseJSONMessage(currentHookLine); + auto json = parseJSONMessage(currentHookLine, "the derivation builder"); if (json) { - auto s = handleJSONLogMessage(*json, worker.act, hook->activities, true); + auto s = handleJSONLogMessage(*json, worker.act, hook->activities, "the derivation builder", true); // ensure that logs from a builder using `ssh-ng://` as protocol // are also available to `nix log`. if (s && !isWrittenToLog && logSink) { @@ -1390,7 +1390,7 @@ void DerivationGoal::handleEOF(Descriptor fd) void DerivationGoal::flushLine() { - if (handleJSONLogMessage(currentLogLine, *act, builderActivities, false)) + if (handleJSONLogMessage(currentLogLine, *act, builderActivities, "the derivation builder", false)) ; else { diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index fc7ac2dea..f32616f94 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -194,6 +194,19 @@ protected: } } + std::optional getNixCacheInfo() override + { + try { + auto result = getFileTransfer()->download(makeRequest(cacheInfoFile)); + return result.data; + } catch (FileTransferError & e) { + if (e.error == FileTransfer::NotFound) + return std::nullopt; + maybeDisable(); + throw; + } + } + /** * This isn't actually necessary read only. We support "upsert" now, so we * have a notion of authentication via HTTP POST/PUT. diff --git a/src/libstore/meson.build b/src/libstore/meson.build index f836b8d4f..12a0e6376 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -15,7 +15,7 @@ project('nix-store', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -29,7 +29,7 @@ deps_private_maybe_subproject = [ deps_public_maybe_subproject = [ dependency('nix-util'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') run_command('ln', '-s', meson.project_build_root() / '__nothing_link_target', @@ -81,7 +81,7 @@ if host_machine.system() == 'windows' deps_other += [wsock32] endif -subdir('build-utils-meson/libatomic') +subdir('nix-meson-build-support/libatomic') boost = dependency( 'boost', @@ -131,7 +131,7 @@ if aws_s3.found() endif deps_other += aws_s3 -subdir('build-utils-meson/generate-header') +subdir('nix-meson-build-support/generate-header') generated_headers = [] foreach header : [ @@ -179,7 +179,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'binary-cache-store.cc', @@ -416,8 +416,8 @@ foreach name, value : cpp_str_defines ] endforeach -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') this_library = library( 'nixstore', @@ -446,4 +446,4 @@ if host_machine.system() != 'darwin' } endif -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libstore/nix-meson-build-support b/src/libstore/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libstore/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libstore/package.nix b/src/libstore/package.nix index f04e3b95f..47a203f83 100644 --- a/src/libstore/package.nix +++ b/src/libstore/package.nix @@ -32,8 +32,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libstore/path-regex.hh b/src/libstore/path-regex.hh index 56c2cfc1d..e34a305c5 100644 --- a/src/libstore/path-regex.hh +++ b/src/libstore/path-regex.hh @@ -1,6 +1,8 @@ #pragma once ///@file +#include + namespace nix { diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index bf351a56d..cfa713b00 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -48,7 +48,11 @@ R && checkAws(std::string_view s, Aws::Utils::Outcome && outcome) if (!outcome.IsSuccess()) throw S3Error( outcome.GetError().GetErrorType(), - s + ": " + outcome.GetError().GetMessage()); + fmt( + "%s: %s (request id: %s)", + s, + outcome.GetError().GetMessage(), + outcome.GetError().GetRequestId())); return outcome.GetResultWithOwnership(); } @@ -121,9 +125,10 @@ class RetryStrategy : public Aws::Client::DefaultRetryStrategy checkInterrupt(); auto retry = Aws::Client::DefaultRetryStrategy::ShouldRetry(error, attemptedRetries); if (retry) - printError("AWS error '%s' (%s), will retry in %d ms", + printError("AWS error '%s' (%s; request id: %s), will retry in %d ms", error.GetExceptionName(), error.GetMessage(), + error.GetRequestId(), CalculateDelayBeforeNextRetry(error, attemptedRetries)); return retry; } diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index dec733fd5..116a480ba 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -41,8 +41,17 @@ void SSHMaster::addCommonSSHOpts(Strings & args) { auto state(state_.lock()); - for (auto & i : tokenizeString(getEnv("NIX_SSHOPTS").value_or(""))) - args.push_back(i); + std::string sshOpts = getEnv("NIX_SSHOPTS").value_or(""); + + try { + std::list opts = shellSplitString(sshOpts); + for (auto & i : opts) + args.push_back(i); + } catch (Error & e) { + e.addTrace({}, "while splitting NIX_SSHOPTS '%s'", sshOpts); + throw; + } + if (!keyFile.empty()) args.insert(args.end(), {"-i", keyFile}); if (!sshPublicHostKey.empty()) { diff --git a/src/libstore/windows/pathlocks.cc b/src/libstore/windows/pathlocks.cc index 00761a8c3..197f5a1c4 100644 --- a/src/libstore/windows/pathlocks.cc +++ b/src/libstore/windows/pathlocks.cc @@ -2,10 +2,12 @@ #include "pathlocks.hh" #include "signals.hh" #include "util.hh" -#include -#include -#include -#include "windows-error.hh" + +#ifdef WIN32 +# include +# include +# include +# include "windows-error.hh" namespace nix { @@ -154,3 +156,4 @@ FdLock::FdLock(Descriptor desc, LockType lockType, bool wait, std::string_view w } } +#endif diff --git a/src/libutil-c/build-utils-meson b/src/libutil-c/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libutil-c/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libutil-c/meson.build b/src/libutil-c/meson.build index d44453676..44cec1afc 100644 --- a/src/libutil-c/meson.build +++ b/src/libutil-c/meson.build @@ -14,7 +14,7 @@ project('nix-util-c', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -23,7 +23,7 @@ deps_private_maybe_subproject = [ ] deps_public_maybe_subproject = [ ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') # TODO rename, because it will conflict with downstream projects configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) @@ -45,7 +45,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'nix_api_util.cc', @@ -60,8 +60,8 @@ headers = [config_h] + files( # TODO don't install this once tests don't use it. headers += files('nix_api_util_internal.h') -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') this_library = library( 'nixutilc', @@ -77,4 +77,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libutil-c/nix-meson-build-support b/src/libutil-c/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libutil-c/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libutil-c/nix_api_util.h b/src/libutil-c/nix_api_util.h index 43f9fa9dc..5f42641d4 100644 --- a/src/libutil-c/nix_api_util.h +++ b/src/libutil-c/nix_api_util.h @@ -47,7 +47,7 @@ extern "C" { */ // Error codes /** - * @brief Type for error codes in the NIX system + * @brief Type for error codes in the Nix system * * This type can have one of several predefined constants: * - NIX_OK: No error occurred (0) diff --git a/src/libutil-c/package.nix b/src/libutil-c/package.nix index 35533f981..4caf96804 100644 --- a/src/libutil-c/package.nix +++ b/src/libutil-c/package.nix @@ -19,8 +19,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libutil-test-support/build-utils-meson b/src/libutil-test-support/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libutil-test-support/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libutil-test-support/meson.build b/src/libutil-test-support/meson.build index 4afed01ca..03ae63f1a 100644 --- a/src/libutil-test-support/meson.build +++ b/src/libutil-test-support/meson.build @@ -14,7 +14,7 @@ project('nix-util-test-support', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') deps_private_maybe_subproject = [ ] @@ -22,7 +22,7 @@ deps_public_maybe_subproject = [ dependency('nix-util'), dependency('nix-util-c'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') rapidcheck = dependency('rapidcheck') deps_public += rapidcheck @@ -34,7 +34,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'tests/hash.cc', @@ -51,8 +51,8 @@ headers = files( 'tests/string_callback.hh', ) -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') this_library = library( 'nix-util-test-support', @@ -70,4 +70,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libutil-test-support/nix-meson-build-support b/src/libutil-test-support/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libutil-test-support/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libutil-test-support/package.nix b/src/libutil-test-support/package.nix index c403e762c..19b5d6b77 100644 --- a/src/libutil-test-support/package.nix +++ b/src/libutil-test-support/package.nix @@ -22,8 +22,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libutil-tests/build-utils-meson b/src/libutil-tests/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libutil-tests/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libutil-tests/meson.build b/src/libutil-tests/meson.build index f59350774..83ac79e92 100644 --- a/src/libutil-tests/meson.build +++ b/src/libutil-tests/meson.build @@ -14,7 +14,7 @@ project('nix-util-tests', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') deps_private_maybe_subproject = [ dependency('nix-util'), @@ -23,10 +23,10 @@ deps_private_maybe_subproject = [ ] deps_public_maybe_subproject = [ ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') rapidcheck = dependency('rapidcheck') deps_private += rapidcheck @@ -42,7 +42,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'args.cc', diff --git a/src/libutil-tests/nix-meson-build-support b/src/libutil-tests/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libutil-tests/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libutil-tests/package.nix b/src/libutil-tests/package.nix index b099037ee..f06704e26 100644 --- a/src/libutil-tests/package.nix +++ b/src/libutil-tests/package.nix @@ -26,8 +26,8 @@ mkMesonExecutable (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libutil-tests/strings.cc b/src/libutil-tests/strings.cc index 8ceb16767..206890bcf 100644 --- a/src/libutil-tests/strings.cc +++ b/src/libutil-tests/strings.cc @@ -2,11 +2,10 @@ #include #include "strings.hh" +#include "error.hh" namespace nix { -using Strings = std::vector; - /* ---------------------------------------------------------------------------- * concatStringsSep * --------------------------------------------------------------------------*/ @@ -345,4 +344,108 @@ RC_GTEST_PROP(splitString, recoveredByConcatStringsSep, (const std::string & s)) RC_ASSERT(concatStringsSep("a", splitString(s, "a")) == s); } +/* ---------------------------------------------------------------------------- + * shellSplitString + * --------------------------------------------------------------------------*/ + +TEST(shellSplitString, empty) +{ + std::list expected = {}; + + ASSERT_EQ(shellSplitString(""), expected); +} + +TEST(shellSplitString, oneWord) +{ + std::list expected = {"foo"}; + + ASSERT_EQ(shellSplitString("foo"), expected); +} + +TEST(shellSplitString, oneWordQuotedWithSpaces) +{ + std::list expected = {"foo bar"}; + + ASSERT_EQ(shellSplitString("'foo bar'"), expected); +} + +TEST(shellSplitString, oneWordQuotedWithSpacesAndDoubleQuoteInSingleQuote) +{ + std::list expected = {"foo bar\""}; + + ASSERT_EQ(shellSplitString("'foo bar\"'"), expected); +} + +TEST(shellSplitString, oneWordQuotedWithDoubleQuotes) +{ + std::list expected = {"foo bar"}; + + ASSERT_EQ(shellSplitString("\"foo bar\""), expected); +} + +TEST(shellSplitString, twoWords) +{ + std::list expected = {"foo", "bar"}; + + ASSERT_EQ(shellSplitString("foo bar"), expected); +} + +TEST(shellSplitString, twoWordsWithSpacesAndQuotesQuoted) +{ + std::list expected = {"foo bar'", "baz\""}; + + ASSERT_EQ(shellSplitString("\"foo bar'\" 'baz\"'"), expected); +} + +TEST(shellSplitString, emptyArgumentsAreAllowedSingleQuotes) +{ + std::list expected = {"foo", "", "bar", "baz", ""}; + + ASSERT_EQ(shellSplitString("foo '' bar baz ''"), expected); +} + +TEST(shellSplitString, emptyArgumentsAreAllowedDoubleQuotes) +{ + std::list expected = {"foo", "", "bar", "baz", ""}; + + ASSERT_EQ(shellSplitString("foo \"\" bar baz \"\""), expected); +} + +TEST(shellSplitString, singleQuoteDoesNotUseEscapes) +{ + std::list expected = {"foo\\\"bar"}; + + ASSERT_EQ(shellSplitString("'foo\\\"bar'"), expected); +} + +TEST(shellSplitString, doubleQuoteDoesUseEscapes) +{ + std::list expected = {"foo\"bar"}; + + ASSERT_EQ(shellSplitString("\"foo\\\"bar\""), expected); +} + +TEST(shellSplitString, backslashEscapesSpaces) +{ + std::list expected = {"foo bar", "baz", "qux quux"}; + + ASSERT_EQ(shellSplitString("foo\\ bar baz qux\\ quux"), expected); +} + +TEST(shellSplitString, backslashEscapesQuotes) +{ + std::list expected = {"foo\"bar", "baz", "qux'quux"}; + + ASSERT_EQ(shellSplitString("foo\\\"bar baz qux\\'quux"), expected); +} + +TEST(shellSplitString, testUnbalancedQuotes) +{ + ASSERT_THROW(shellSplitString("foo'"), Error); + ASSERT_THROW(shellSplitString("foo\""), Error); + ASSERT_THROW(shellSplitString("foo'bar"), Error); + ASSERT_THROW(shellSplitString("foo\"bar"), Error); + ASSERT_THROW(shellSplitString("foo\"bar\\\""), Error); +} + } // namespace nix diff --git a/src/libutil/build-utils-meson b/src/libutil/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libutil/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libutil/callback.hh b/src/libutil/callback.hh index 26c386d80..c2cada2f6 100644 --- a/src/libutil/callback.hh +++ b/src/libutil/callback.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include #include #include diff --git a/src/libutil/config-impl.hh b/src/libutil/config-impl.hh index c3aa61ddb..94c2cb2e4 100644 --- a/src/libutil/config-impl.hh +++ b/src/libutil/config-impl.hh @@ -13,6 +13,7 @@ */ #include "config.hh" +#include "args.hh" namespace nix { diff --git a/src/libutil/executable-path.cc b/src/libutil/executable-path.cc index ebd522a41..8d665c7df 100644 --- a/src/libutil/executable-path.cc +++ b/src/libutil/executable-path.cc @@ -73,7 +73,7 @@ ExecutablePath::findName(const OsString & exe, std::function parseJSONMessage(const std::string & msg) +std::optional parseJSONMessage(const std::string & msg, std::string_view source) { if (!hasPrefix(msg, "@nix ")) return std::nullopt; try { return nlohmann::json::parse(std::string(msg, 5)); } catch (std::exception & e) { - printError("bad JSON log message from builder: %s", e.what()); + printError("bad JSON log message from %s: %s", + Uncolored(source), + e.what()); } return std::nullopt; } bool handleJSONLogMessage(nlohmann::json & json, const Activity & act, std::map & activities, - bool trusted) + std::string_view source, bool trusted) { - std::string action = json["action"]; + try { + std::string action = json["action"]; - if (action == "start") { - auto type = (ActivityType) json["type"]; - if (trusted || type == actFileTransfer) - activities.emplace(std::piecewise_construct, - std::forward_as_tuple(json["id"]), - std::forward_as_tuple(*logger, (Verbosity) json["level"], type, - json["text"], getFields(json["fields"]), act.id)); + if (action == "start") { + auto type = (ActivityType) json["type"]; + if (trusted || type == actFileTransfer) + activities.emplace(std::piecewise_construct, + std::forward_as_tuple(json["id"]), + std::forward_as_tuple(*logger, (Verbosity) json["level"], type, + json["text"], getFields(json["fields"]), act.id)); + } + + else if (action == "stop") + activities.erase((ActivityId) json["id"]); + + else if (action == "result") { + auto i = activities.find((ActivityId) json["id"]); + if (i != activities.end()) + i->second.result((ResultType) json["type"], getFields(json["fields"])); + } + + else if (action == "setPhase") { + std::string phase = json["phase"]; + act.result(resSetPhase, phase); + } + + else if (action == "msg") { + std::string msg = json["msg"]; + logger->log((Verbosity) json["level"], msg); + } + + return true; + } catch (const nlohmann::json::exception &e) { + warn( + "Unable to handle a JSON message from %s: %s", + Uncolored(source), + e.what() + ); + return false; } - - else if (action == "stop") - activities.erase((ActivityId) json["id"]); - - else if (action == "result") { - auto i = activities.find((ActivityId) json["id"]); - if (i != activities.end()) - i->second.result((ResultType) json["type"], getFields(json["fields"])); - } - - else if (action == "setPhase") { - std::string phase = json["phase"]; - act.result(resSetPhase, phase); - } - - else if (action == "msg") { - std::string msg = json["msg"]; - logger->log((Verbosity) json["level"], msg); - } - - return true; } bool handleJSONLogMessage(const std::string & msg, - const Activity & act, std::map & activities, bool trusted) + const Activity & act, std::map & activities, std::string_view source, bool trusted) { - auto json = parseJSONMessage(msg); + auto json = parseJSONMessage(msg, source); if (!json) return false; - return handleJSONLogMessage(*json, act, activities, trusted); + return handleJSONLogMessage(*json, act, activities, source, trusted); } Activity::~Activity() diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 250f92099..11e4033a5 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -185,14 +185,25 @@ Logger * makeSimpleLogger(bool printBuildLogs = true); Logger * makeJSONLogger(Logger & prevLogger); -std::optional parseJSONMessage(const std::string & msg); +/** + * @param source A noun phrase describing the source of the message, e.g. "the builder". + */ +std::optional parseJSONMessage(const std::string & msg, std::string_view source); +/** + * @param source A noun phrase describing the source of the message, e.g. "the builder". + */ bool handleJSONLogMessage(nlohmann::json & json, const Activity & act, std::map & activities, + std::string_view source, bool trusted); +/** + * @param source A noun phrase describing the source of the message, e.g. "the builder". + */ bool handleJSONLogMessage(const std::string & msg, const Activity & act, std::map & activities, + std::string_view source, bool trusted); /** diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 11b4ea592..bbe7872cf 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -14,7 +14,7 @@ project('nix-util', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -22,7 +22,7 @@ deps_private_maybe_subproject = [ ] deps_public_maybe_subproject = [ ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') # Check for each of these functions, and create a define like `#define # HAVE_LUTIMES 1`. The `#define` is unconditional, 0 for not found and 1 @@ -53,7 +53,7 @@ endforeach configdata.set('HAVE_DECL_AT_SYMLINK_NOFOLLOW', cxx.has_header_symbol('fcntl.h', 'AT_SYMLINK_NOFOLLOW').to_int()) -subdir('build-utils-meson/libatomic') +subdir('nix-meson-build-support/libatomic') if host_machine.system() == 'windows' socket = cxx.find_library('ws2_32') @@ -120,7 +120,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'archive.cc', @@ -256,8 +256,8 @@ else subdir('unix') endif -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') this_library = library( 'nixutil', @@ -278,4 +278,4 @@ if host_machine.system() == 'windows' libraries_private += ['-lws2_32'] endif -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libutil/nix-meson-build-support b/src/libutil/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libutil/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libutil/package.nix b/src/libutil/package.nix index 17a156740..69ebbf726 100644 --- a/src/libutil/package.nix +++ b/src/libutil/package.nix @@ -25,8 +25,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libutil/regex-combinators.hh b/src/libutil/regex-combinators.hh index 87d6aa678..75ccd4e6c 100644 --- a/src/libutil/regex-combinators.hh +++ b/src/libutil/regex-combinators.hh @@ -2,6 +2,8 @@ ///@file #include +#include +#include namespace nix::regex { @@ -10,22 +12,23 @@ namespace nix::regex { static inline std::string either(std::string_view a, std::string_view b) { - return std::string { a } + "|" + b; + std::stringstream ss; + ss << a << "|" << b; + return ss.str(); } static inline std::string group(std::string_view a) { - return std::string { "(" } + a + ")"; -} - -static inline std::string many(std::string_view a) -{ - return std::string { "(?:" } + a + ")*"; + std::stringstream ss; + ss << "(" << a << ")"; + return ss.str(); } static inline std::string list(std::string_view a) { - return std::string { a } + many(group("," + a)); + std::stringstream ss; + ss << a << "(," << a << ")*"; + return ss.str(); } } diff --git a/src/libutil/strings.cc b/src/libutil/strings.cc index c221a43c6..402b7ae98 100644 --- a/src/libutil/strings.cc +++ b/src/libutil/strings.cc @@ -4,6 +4,7 @@ #include "strings-inline.hh" #include "os-string.hh" +#include "error.hh" namespace nix { @@ -48,4 +49,107 @@ template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const s template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::set &); template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::vector &); +/** + * Shell split string: split a string into shell arguments, respecting quotes and backslashes. + * + * Used for NIX_SSHOPTS handling, which previously used `tokenizeString` and was broken by + * Arguments that need to be passed to ssh with spaces in them. + * + * Read https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html for the + * POSIX shell specification, which is technically what we are implementing here. + */ +std::list shellSplitString(std::string_view s) +{ + std::list result; + std::string current; + bool startedCurrent = false; + bool escaping = false; + + auto pushCurrent = [&]() { + if (startedCurrent) { + result.push_back(current); + current.clear(); + startedCurrent = false; + } + }; + + auto pushChar = [&](char c) { + current.push_back(c); + startedCurrent = true; + }; + + auto pop = [&]() { + auto c = s[0]; + s.remove_prefix(1); + return c; + }; + + auto inDoubleQuotes = [&]() { + startedCurrent = true; + // in double quotes, escaping with backslash is only effective for $, `, ", and backslash + while (!s.empty()) { + auto c = pop(); + if (escaping) { + switch (c) { + case '$': + case '`': + case '"': + case '\\': + pushChar(c); + break; + default: + pushChar('\\'); + pushChar(c); + break; + } + escaping = false; + } else if (c == '\\') { + escaping = true; + } else if (c == '"') { + return; + } else { + pushChar(c); + } + } + if (s.empty()) { + throw Error("unterminated double quote"); + } + }; + + auto inSingleQuotes = [&]() { + startedCurrent = true; + while (!s.empty()) { + auto c = pop(); + if (c == '\'') { + return; + } + pushChar(c); + } + if (s.empty()) { + throw Error("unterminated single quote"); + } + }; + + while (!s.empty()) { + auto c = pop(); + if (escaping) { + pushChar(c); + escaping = false; + } else if (c == '\\') { + escaping = true; + } else if (c == ' ' || c == '\t') { + pushCurrent(); + } else if (c == '"') { + inDoubleQuotes(); + } else if (c == '\'') { + inSingleQuotes(); + } else { + pushChar(c); + } + } + + pushCurrent(); + + return result; +} } // namespace nix diff --git a/src/libutil/strings.hh b/src/libutil/strings.hh index 533126be1..c4fd3daa1 100644 --- a/src/libutil/strings.hh +++ b/src/libutil/strings.hh @@ -71,4 +71,11 @@ extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::set &); extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::vector &); +/** + * Shell split string: split a string into shell arguments, respecting quotes and backslashes. + * + * Used for NIX_SSHOPTS handling, which previously used `tokenizeString` and was broken by + * Arguments that need to be passed to ssh with spaces in them. + */ +std::list shellSplitString(std::string_view s); } diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 325e3ea73..9f5c75827 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -43,9 +43,11 @@ template struct Explicit { T t; - bool operator ==(const Explicit & other) const + bool operator ==(const Explicit & other) const = default; + + bool operator <(const Explicit & other) const { - return t == other.t; + return t < other.t; } }; diff --git a/src/libutil/unix/signals-impl.hh b/src/libutil/unix/signals-impl.hh index 2193922be..037416e7d 100644 --- a/src/libutil/unix/signals-impl.hh +++ b/src/libutil/unix/signals-impl.hh @@ -14,6 +14,7 @@ #include "error.hh" #include "logging.hh" #include "ansicolor.hh" +#include "signals.hh" #include #include diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 0fb6ff837..4d5683e2b 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -338,7 +338,9 @@ std::string showBytes(uint64_t bytes); */ inline std::string operator + (const std::string & s1, std::string_view s2) { - auto s = s1; + std::string s; + s.reserve(s1.size() + s2.size()); + s.append(s1); s.append(s2); return s; } @@ -351,10 +353,11 @@ inline std::string operator + (std::string && s, std::string_view s2) inline std::string operator + (std::string_view s1, const char * s2) { + auto s2Size = strlen(s2); std::string s; - s.reserve(s1.size() + strlen(s2)); + s.reserve(s1.size() + s2Size); s.append(s1); - s.append(s2); + s.append(s2, s2Size); return s; } diff --git a/src/libutil/windows/environment-variables.cc b/src/libutil/windows/environment-variables.cc index 5ce8a1395..308a432e4 100644 --- a/src/libutil/windows/environment-variables.cc +++ b/src/libutil/windows/environment-variables.cc @@ -1,6 +1,7 @@ #include "environment-variables.hh" -#include "processenv.h" +#ifdef WIN32 +# include "processenv.h" namespace nix { @@ -43,3 +44,4 @@ int setEnvOs(const OsString & name, const OsString & value) } } +#endif diff --git a/src/libutil/windows/file-descriptor.cc b/src/libutil/windows/file-descriptor.cc index 7b8a712e8..71f53ccb8 100644 --- a/src/libutil/windows/file-descriptor.cc +++ b/src/libutil/windows/file-descriptor.cc @@ -5,6 +5,7 @@ #include "windows-error.hh" #include "file-path.hh" +#ifdef WIN32 #include #include #include @@ -152,3 +153,4 @@ Path windows::handleToPath(HANDLE handle) { #endif } +#endif diff --git a/src/libutil/windows/file-system.cc b/src/libutil/windows/file-system.cc index b15355efe..53271cef3 100644 --- a/src/libutil/windows/file-system.cc +++ b/src/libutil/windows/file-system.cc @@ -1,5 +1,6 @@ #include "file-system.hh" +#ifdef WIN32 namespace nix { Descriptor openDirectory(const std::filesystem::path & path) @@ -15,3 +16,4 @@ Descriptor openDirectory(const std::filesystem::path & path) } } +#endif diff --git a/src/libutil/windows/muxable-pipe.cc b/src/libutil/windows/muxable-pipe.cc index 91a321f7c..af7e987e9 100644 --- a/src/libutil/windows/muxable-pipe.cc +++ b/src/libutil/windows/muxable-pipe.cc @@ -1,9 +1,10 @@ -#include -#include "windows-error.hh" +#ifdef WIN32 +# include +# include "windows-error.hh" -#include "logging.hh" -#include "util.hh" -#include "muxable-pipe.hh" +# include "logging.hh" +# include "util.hh" +# include "muxable-pipe.hh" namespace nix { @@ -68,3 +69,4 @@ void MuxablePipePollState::iterate( } } +#endif diff --git a/src/libutil/windows/os-string.cc b/src/libutil/windows/os-string.cc index 7507f9030..26ad9cba0 100644 --- a/src/libutil/windows/os-string.cc +++ b/src/libutil/windows/os-string.cc @@ -7,6 +7,8 @@ #include "file-path-impl.hh" #include "util.hh" +#ifdef WIN32 + namespace nix { std::string os_string_to_string(PathViewNG::string_view path) @@ -22,3 +24,5 @@ std::filesystem::path::string_type string_to_os_string(std::string_view s) } } + +#endif diff --git a/src/libutil/windows/processes.cc b/src/libutil/windows/processes.cc index 7f34c5632..e69f1ed45 100644 --- a/src/libutil/windows/processes.cc +++ b/src/libutil/windows/processes.cc @@ -23,6 +23,8 @@ #include #include +#ifdef WIN32 + #define WIN32_LEAN_AND_MEAN #include @@ -386,3 +388,5 @@ int execvpe(const wchar_t * file0, const wchar_t * const argv[], const wchar_t * } } + +#endif diff --git a/src/libutil/windows/users.cc b/src/libutil/windows/users.cc index db6c42df3..2780e45f4 100644 --- a/src/libutil/windows/users.cc +++ b/src/libutil/windows/users.cc @@ -4,6 +4,7 @@ #include "file-system.hh" #include "windows-error.hh" +#ifdef WIN32 #define WIN32_LEAN_AND_MEAN #include @@ -50,3 +51,4 @@ bool isRootUser() { } } +#endif diff --git a/src/libutil/windows/windows-async-pipe.cc b/src/libutil/windows/windows-async-pipe.cc index 4fa57ca36..13b563510 100644 --- a/src/libutil/windows/windows-async-pipe.cc +++ b/src/libutil/windows/windows-async-pipe.cc @@ -1,6 +1,8 @@ #include "windows-async-pipe.hh" #include "windows-error.hh" +#ifdef WIN32 + namespace nix::windows { void AsyncPipe::createAsyncPipe(HANDLE iocp) @@ -47,3 +49,5 @@ void AsyncPipe::close() } } + +#endif diff --git a/src/libutil/windows/windows-async-pipe.hh b/src/libutil/windows/windows-async-pipe.hh index 8f554e403..277336ed7 100644 --- a/src/libutil/windows/windows-async-pipe.hh +++ b/src/libutil/windows/windows-async-pipe.hh @@ -2,6 +2,7 @@ ///@file #include "file-descriptor.hh" +#ifdef WIN32 namespace nix::windows { @@ -25,3 +26,4 @@ public: }; } +#endif diff --git a/src/libutil/windows/windows-error.cc b/src/libutil/windows/windows-error.cc index aead4af23..4cf4274da 100644 --- a/src/libutil/windows/windows-error.cc +++ b/src/libutil/windows/windows-error.cc @@ -1,5 +1,6 @@ #include "windows-error.hh" +#ifdef WIN32 #include #define WIN32_LEAN_AND_MEAN #include @@ -29,3 +30,4 @@ std::string WinError::renderError(DWORD lastError) } } +#endif diff --git a/src/libutil/windows/windows-error.hh b/src/libutil/windows/windows-error.hh index 624b4c4cb..4e48ee859 100644 --- a/src/libutil/windows/windows-error.hh +++ b/src/libutil/windows/windows-error.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#ifdef WIN32 #include #include "error.hh" @@ -49,3 +50,4 @@ private: }; } +#endif diff --git a/src/nix/build-utils-meson b/src/nix/build-utils-meson deleted file mode 120000 index 91937f183..000000000 --- a/src/nix/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson/ \ No newline at end of file diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 9f3584a11..55aa8971e 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -162,6 +162,7 @@ struct CmdFlakeLock : FlakeCommand settings.tarballTtl = 0; lockFlags.writeLockFile = true; + lockFlags.failOnUnlocked = true; lockFlags.applyNixConfig = true; lockFlake(); diff --git a/src/nix/flake.md b/src/nix/flake.md index 1e9895f6e..fa17375df 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -165,7 +165,8 @@ can occur in *locked* flake references and are available to Nix code: Currently the `type` attribute can be one of the following: -* `indirect`: *The default*. Indirection through the flake registry. +* `indirect`: *The default*. These are symbolic references to flakes + that are looked up in [the flake registries](./nix3-registry.md). These have the form ``` @@ -653,7 +654,7 @@ following fields: * `inputs`: The dependencies of this node, as a mapping from input names (e.g. `nixpkgs`) to node labels (e.g. `n2`). -* `original`: The original input specification from `flake.lock`, as a +* `original`: The original input specification from `flake.nix`, as a set of `builtins.fetchTree` arguments. * `locked`: The locked input specification, as a set of diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 2f9b3fe7c..416cd19b3 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -163,8 +163,11 @@ struct CmdToBase : Command HashFormat hashFormat; std::optional hashAlgo; std::vector args; + bool legacyCli; - CmdToBase(HashFormat hashFormat) : hashFormat(hashFormat) + CmdToBase(HashFormat hashFormat, bool legacyCli = false) + : hashFormat(hashFormat) + , legacyCli(legacyCli) { addFlag(flag::hashAlgoOpt("type", &hashAlgo)); expectArgs("strings", &args); @@ -181,7 +184,8 @@ struct CmdToBase : Command void run() override { - warn("The old format conversion sub commands of `nix hash` were deprecated in favor of `nix hash convert`."); + if (!legacyCli) + warn("The old format conversion subcommands of `nix hash` were deprecated in favor of `nix hash convert`."); for (const auto & s : args) logger->cout(Hash::parseAny(s, hashAlgo).to_string(hashFormat, hashFormat == HashFormat::SRI)); } @@ -222,11 +226,18 @@ struct CmdHashConvert : Command Category category() override { return catUtility; } void run() override { - for (const auto& s: hashStrings) { - Hash h = Hash::parseAny(s, algo); - if (from && h.to_string(*from, from == HashFormat::SRI) != s) { + for (const auto & s : hashStrings) { + Hash h = + from == HashFormat::SRI + ? Hash::parseSRI(s) + : Hash::parseAny(s, algo); + if (from + && from != HashFormat::SRI + && h.to_string(*from, false) != + (from == HashFormat::Base16 ? toLower(s) : s)) + { auto from_as_string = printHashFormat(*from); - throw BadHash("input hash '%s' does not have the expected format '--from %s'", s, from_as_string); + throw BadHash("input hash '%s' does not have the expected format for '--from %s'", s, from_as_string); } logger->cout(h.to_string(to, to == HashFormat::SRI)); } @@ -321,7 +332,7 @@ static int compatNixHash(int argc, char * * argv) } else { - CmdToBase cmd(hashFormat); + CmdToBase cmd(hashFormat, true); cmd.args = ss; if (hashAlgo.has_value()) cmd.hashAlgo = hashAlgo; cmd.run(); diff --git a/src/nix/meson.build b/src/nix/meson.build index 5c70c8216..1d4840b12 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -15,7 +15,7 @@ project('nix', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') nix_store = dependency('nix-store') @@ -30,10 +30,10 @@ deps_private_maybe_subproject = [ ] deps_public_maybe_subproject = [ ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') configdata = configuration_data() @@ -63,8 +63,8 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') -subdir('build-utils-meson/generate-header') +subdir('nix-meson-build-support/common') +subdir('nix-meson-build-support/generate-header') nix_sources = [config_h] + files( 'add-to-store.cc', diff --git a/src/nix/nix-meson-build-support b/src/nix/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/nix/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/nix/package.nix b/src/nix/package.nix index c7b24efce..9bc139c3b 100644 --- a/src/nix/package.nix +++ b/src/nix/package.nix @@ -22,8 +22,8 @@ mkMesonExecutable (finalAttrs: { workDir = ./.; fileset = fileset.unions ([ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/nix/registry.md b/src/nix/registry.md index bd3575d1b..d6f8af5e9 100644 --- a/src/nix/registry.md +++ b/src/nix/registry.md @@ -34,6 +34,8 @@ highest precedence: * Overrides specified on the command line using the option `--override-flake`. +Note that the system and user registries are not used to resolve flake references in `flake.nix`. They are only used to resolve flake references on the command line. + # Registry format A registry is a JSON file with the following format: diff --git a/src/nix/self-exe.hh b/src/nix/self-exe.hh index 3161553ec..91e260f0b 100644 --- a/src/nix/self-exe.hh +++ b/src/nix/self-exe.hh @@ -2,6 +2,8 @@ ///@file #include +#include +#include namespace nix { diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index f54cc59d0..77fb798a7 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -107,7 +107,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand auto whereOpt = ExecutablePath::load().findName(OS_STR("nix-env")); if (!whereOpt) throw Error("couldn't figure out how Nix is installed, so I can't upgrade it"); - auto & where = *whereOpt; + const auto & where = whereOpt->parent_path(); printInfo("found Nix in '%s'", where); @@ -126,7 +126,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand if (where.filename() != "bin" || !hasSuffix(userEnv, "user-environment")) - throw Error("directory '%s' does not appear to be part of a Nix profile", where); + throw Error("directory %s does not appear to be part of a Nix profile", where); if (!store->isValidPath(store->parseStorePath(userEnv))) throw Error("directory '%s' is not in the Nix store", userEnv); diff --git a/tests/functional/build-remote-trustless-should-fail-0.sh b/tests/functional/build-remote-trustless-should-fail-0.sh index 4eccb73e0..3401de1b0 100755 --- a/tests/functional/build-remote-trustless-should-fail-0.sh +++ b/tests/functional/build-remote-trustless-should-fail-0.sh @@ -8,6 +8,7 @@ TODO_NixOS restartDaemon requireSandboxSupport +requiresUnprivilegedUserNamespaces [[ $busybox =~ busybox ]] || skipTest "no busybox" unset NIX_STORE_DIR diff --git a/tests/functional/build-remote-trustless.sh b/tests/functional/build-remote-trustless.sh index c498d46c3..9f91a91a9 100644 --- a/tests/functional/build-remote-trustless.sh +++ b/tests/functional/build-remote-trustless.sh @@ -5,6 +5,7 @@ # shellcheck disable=SC2154 requireSandboxSupport +requiresUnprivilegedUserNamespaces [[ "$busybox" =~ busybox ]] || skipTest "no busybox" unset NIX_STORE_DIR diff --git a/tests/functional/build-remote.sh b/tests/functional/build-remote.sh index 1a5334577..3231341cb 100644 --- a/tests/functional/build-remote.sh +++ b/tests/functional/build-remote.sh @@ -3,6 +3,7 @@ : "${file?must be defined by caller (remote building test case using this)}" requireSandboxSupport +requiresUnprivilegedUserNamespaces [[ "${busybox-}" =~ busybox ]] || skipTest "no busybox" # Avoid store dir being inside sandbox build-dir @@ -27,6 +28,7 @@ builders=( chmod -R +w "$TEST_ROOT/machine"* || true rm -rf "$TEST_ROOT/machine"* || true + # Note: ssh://localhost bypasses ssh, directly invoking nix-store as a # child process. This allows us to test LegacySSHStore::buildDerivation(). # ssh-ng://... likewise allows us to test RemoteStore::buildDerivation(). diff --git a/tests/functional/chroot-store.sh b/tests/functional/chroot-store.sh index 46e91f0aa..ccde3e90b 100755 --- a/tests/functional/chroot-store.sh +++ b/tests/functional/chroot-store.sh @@ -40,6 +40,7 @@ EOF cp simple.nix shell.nix simple.builder.sh "${config_nix}" "$flakeDir/" TODO_NixOS + requiresUnprivilegedUserNamespaces outPath=$(nix build --print-out-paths --no-link --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' --store "$TEST_ROOT/x" path:"$flakeDir") diff --git a/tests/functional/common/functions.sh b/tests/functional/common/functions.sh index 7195149cb..bf3dd2ca8 100644 --- a/tests/functional/common/functions.sh +++ b/tests/functional/common/functions.sh @@ -345,4 +345,15 @@ count() { trap onError ERR +requiresUnprivilegedUserNamespaces() { + if [[ -f /proc/sys/kernel/apparmor_restrict_unprivileged_userns ]] && [[ $(< /proc/sys/kernel/apparmor_restrict_unprivileged_userns) -eq 1 ]]; then + skipTest "Unprivileged user namespaces are disabled. Run 'sudo sysctl -w /proc/sys/kernel/apparmor_restrict_unprivileged_userns=0' to allow, and run these tests." + fi +} + +execUnshare () { + requiresUnprivilegedUserNamespaces + exec unshare --mount --map-root-user "$SHELL" "$@" +} + fi # COMMON_FUNCTIONS_SH_SOURCED diff --git a/tests/functional/dependencies.nix b/tests/functional/dependencies.nix index be1a7ae9a..4ff29227f 100644 --- a/tests/functional/dependencies.nix +++ b/tests/functional/dependencies.nix @@ -1,7 +1,7 @@ { hashInvalidator ? "" }: with import ./config.nix; -let { +let input0 = mkDerivation { name = "dependencies-input-0"; @@ -33,16 +33,15 @@ let { outputHash = "1dq9p0hnm1y75q2x40fws5887bq1r840hzdxak0a9djbwvx0b16d"; }; - body = mkDerivation { - name = "dependencies-top"; - builder = ./dependencies.builder0.sh + "/FOOBAR/../."; - input1 = input1 + "/."; - input2 = "${input2}/."; - input1_drv = input1; - input2_drv = input2; - input0_drv = input0; - fod_input_drv = fod_input; - meta.description = "Random test package"; - }; - +in +mkDerivation { + name = "dependencies-top"; + builder = ./dependencies.builder0.sh + "/FOOBAR/../."; + input1 = input1 + "/."; + input2 = "${input2}/."; + input1_drv = input1; + input2_drv = input2; + input0_drv = input0; + fod_input_drv = fod_input; + meta.description = "Random test package"; } diff --git a/tests/functional/derivation-advanced-attributes.sh b/tests/functional/derivation-advanced-attributes.sh index 271f17dc6..6707b345c 100755 --- a/tests/functional/derivation-advanced-attributes.sh +++ b/tests/functional/derivation-advanced-attributes.sh @@ -13,7 +13,7 @@ badExitCode=0 store="$TEST_ROOT/store" for nixFile in derivation/*.nix; do - drvPath=$(nix-instantiate --store "$store" --pure-eval --expr "$(< "$nixFile")") + drvPath=$(env -u NIX_STORE nix-instantiate --store "$store" --pure-eval --expr "$(< "$nixFile")") testName=$(basename "$nixFile" .nix) got="${store}${drvPath}" expected="derivation/$testName.drv" diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index de37ae1b7..6c466a0c7 100755 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -77,6 +77,7 @@ hash1=$(echo "$json" | jq -r .revision) echo foo > "$flake1Dir/foo" git -C "$flake1Dir" add $flake1Dir/foo [[ $(nix flake metadata flake1 --json --refresh | jq -r .dirtyRevision) == "$hash1-dirty" ]] +[[ "$(nix flake metadata flake1 --json | jq -r .fingerprint)" != null ]] echo -n '# foo' >> "$flake1Dir/flake.nix" flake1OriginalCommit=$(git -C "$flake1Dir" rev-parse HEAD) diff --git a/tests/functional/flakes/unlocked-override.sh b/tests/functional/flakes/unlocked-override.sh index a17a0c2af..ebad332d0 100755 --- a/tests/functional/flakes/unlocked-override.sh +++ b/tests/functional/flakes/unlocked-override.sh @@ -30,3 +30,6 @@ git -C "$flake2Dir" add flake.nix echo 456 > "$flake1Dir"/x.nix [[ $(nix eval --json "$flake2Dir#x" --override-input flake1 "$TEST_ROOT/flake1") = 456 ]] + +expectStderr 1 nix flake lock "$flake2Dir" --override-input flake1 "$TEST_ROOT/flake1" | + grepQuiet "cannot write lock file.*because it has an unlocked input" diff --git a/tests/functional/hash-convert.sh b/tests/functional/hash-convert.sh index 3a099950f..c40cb469c 100755 --- a/tests/functional/hash-convert.sh +++ b/tests/functional/hash-convert.sh @@ -93,15 +93,19 @@ try3() { # Asserting input format fails. # - fail=$(nix hash convert --hash-algo "$1" --from nix32 "$2" 2>&1 || echo "exit: $?") - [[ "$fail" == *"error: input hash"*"exit: 1" ]] - fail=$(nix hash convert --hash-algo "$1" --from base16 "$3" 2>&1 || echo "exit: $?") - [[ "$fail" == *"error: input hash"*"exit: 1" ]] - fail=$(nix hash convert --hash-algo "$1" --from nix32 "$4" 2>&1 || echo "exit: $?") - [[ "$fail" == *"error: input hash"*"exit: 1" ]] + expectStderr 1 nix hash convert --hash-algo "$1" --from sri "$2" | grepQuiet "is not SRI" + expectStderr 1 nix hash convert --hash-algo "$1" --from nix32 "$2" | grepQuiet "input hash" + expectStderr 1 nix hash convert --hash-algo "$1" --from base16 "$3" | grepQuiet "input hash" + expectStderr 1 nix hash convert --hash-algo "$1" --from nix32 "$4" | grepQuiet "input hash" + # Base-16 hashes can be in uppercase. + nix hash convert --hash-algo "$1" --from base16 "$(echo $2 | tr [a-z] [A-Z])" } try3 sha1 "800d59cfcd3c05e900cb4e214be48f6b886a08df" "vw46m23bizj4n8afrc0fj19wrp7mj3c0" "gA1Zz808BekAy04hS+SPa4hqCN8=" try3 sha256 "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=" try3 sha512 "204a8fc6dda82f0a0ced7beb8e08a41657c16ef468b228a8279be331a703c33596fd15c13b1b07f9aa1d3bea57789ca031ad85c7a71dd70354ec631238ca3445" "12k9jiq29iyqm03swfsgiw5mlqs173qazm3n7daz43infy12pyrcdf30fkk3qwv4yl2ick8yipc2mqnlh48xsvvxl60lbx8vp38yji0" "IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ==" + +# Test SRI hashes that lack trailing '=' characters. These are incorrect but we need to support them for backward compatibility. +[[ $(nix hash convert --from sri "sha256-ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0") = sha256-ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0= ]] +[[ $(nix hash convert --from sri "sha512-IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ") = sha512-IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ== ]] diff --git a/tests/functional/import-from-derivation.nix b/tests/functional/import-from-derivation.nix index cc53451cf..770dd86cf 100644 --- a/tests/functional/import-from-derivation.nix +++ b/tests/functional/import-from-derivation.nix @@ -1,4 +1,4 @@ -with import ./config.nix; +with import ; rec { bar = mkDerivation { @@ -30,4 +30,23 @@ rec { echo -n BLA$(cat $src) > $out ''; }; + + step1 = mkDerivation { + name = "step1"; + buildCommand = '' + mkdir -p $out + echo 'foo' > $out/bla + ''; + }; + + addPathExpr = mkDerivation { + name = "add-path"; + inherit step1; + buildCommand = '' + mkdir -p $out + echo "builtins.path { path = \"$step1\"; sha256 = \"7ptL+pnrZXnSa5hwwB+2SXTLkcSb5264WGGokN8OXto=\"; }" > $out/default.nix + ''; + }; + + importAddPathExpr = import addPathExpr; } diff --git a/tests/functional/import-from-derivation.sh b/tests/functional/import-from-derivation.sh index 83ef92a6f..a00761235 100755 --- a/tests/functional/import-from-derivation.sh +++ b/tests/functional/import-from-derivation.sh @@ -6,6 +6,8 @@ TODO_NixOS clearStoreIfPossible +export NIX_PATH=config="${config_nix}" + if nix-instantiate --readonly-mode ./import-from-derivation.nix -A result; then echo "read-only evaluation of an imported derivation unexpectedly failed" exit 1 @@ -15,6 +17,9 @@ outPath=$(nix-build ./import-from-derivation.nix -A result --no-out-link) [ "$(cat "$outPath")" = FOO579 ] +# Check that we can have access to the entire closure of a derivation output. +nix build --no-link --restrict-eval -I src=. -f ./import-from-derivation.nix importAddPathExpr -v + # FIXME: the next tests are broken on CA. if [[ -n "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then exit 0 diff --git a/tests/functional/lang/eval-fail-fromJSON-keyWithNullByte.err.exp b/tests/functional/lang/eval-fail-fromJSON-keyWithNullByte.err.exp new file mode 100644 index 000000000..a16192c59 --- /dev/null +++ b/tests/functional/lang/eval-fail-fromJSON-keyWithNullByte.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fromJSON' builtin + at /pwd/lang/eval-fail-fromJSON-keyWithNullByte.nix:1:1: + 1| builtins.fromJSON ''{"a\u0000b": 1}'' + | ^ + 2| + + error: input string 'a␀b' cannot be represented as Nix string because it contains null bytes diff --git a/tests/functional/lang/eval-fail-fromJSON-keyWithNullByte.nix b/tests/functional/lang/eval-fail-fromJSON-keyWithNullByte.nix new file mode 100644 index 000000000..ffaa6a97d --- /dev/null +++ b/tests/functional/lang/eval-fail-fromJSON-keyWithNullByte.nix @@ -0,0 +1 @@ +builtins.fromJSON ''{"a\u0000b": 1}'' diff --git a/tests/functional/lang/eval-fail-fromJSON-valueWithNullByte.err.exp b/tests/functional/lang/eval-fail-fromJSON-valueWithNullByte.err.exp new file mode 100644 index 000000000..c5c08e6ff --- /dev/null +++ b/tests/functional/lang/eval-fail-fromJSON-valueWithNullByte.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fromJSON' builtin + at /pwd/lang/eval-fail-fromJSON-valueWithNullByte.nix:1:1: + 1| builtins.fromJSON ''"a\u0000b"'' + | ^ + 2| + + error: input string 'a␀b' cannot be represented as Nix string because it contains null bytes diff --git a/tests/functional/lang/eval-fail-fromJSON-valueWithNullByte.nix b/tests/functional/lang/eval-fail-fromJSON-valueWithNullByte.nix new file mode 100644 index 000000000..c71ab990d --- /dev/null +++ b/tests/functional/lang/eval-fail-fromJSON-valueWithNullByte.nix @@ -0,0 +1 @@ +builtins.fromJSON ''"a\u0000b"'' diff --git a/tests/functional/lang/eval-fail-fromTOML-keyWithNullByte.err.exp b/tests/functional/lang/eval-fail-fromTOML-keyWithNullByte.err.exp new file mode 100644 index 000000000..dc2180f0b --- /dev/null +++ b/tests/functional/lang/eval-fail-fromTOML-keyWithNullByte.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fromTOML' builtin + at /pwd/lang/eval-fail-fromTOML-keyWithNullByte.nix:1:1: + 1| builtins.fromTOML ''"a\u0000b" = 1'' + | ^ + 2| + + error: while parsing TOML: error: input string 'a␀b' cannot be represented as Nix string because it contains null bytes diff --git a/tests/functional/lang/eval-fail-fromTOML-keyWithNullByte.nix b/tests/functional/lang/eval-fail-fromTOML-keyWithNullByte.nix new file mode 100644 index 000000000..b622dc4dc --- /dev/null +++ b/tests/functional/lang/eval-fail-fromTOML-keyWithNullByte.nix @@ -0,0 +1 @@ +builtins.fromTOML ''"a\u0000b" = 1'' diff --git a/tests/functional/lang/eval-fail-fromTOML-valueWithNullByte.err.exp b/tests/functional/lang/eval-fail-fromTOML-valueWithNullByte.err.exp new file mode 100644 index 000000000..0235692a8 --- /dev/null +++ b/tests/functional/lang/eval-fail-fromTOML-valueWithNullByte.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fromTOML' builtin + at /pwd/lang/eval-fail-fromTOML-valueWithNullByte.nix:1:1: + 1| builtins.fromTOML ''k = "a\u0000b"'' + | ^ + 2| + + error: while parsing TOML: error: input string 'a␀b' cannot be represented as Nix string because it contains null bytes diff --git a/tests/functional/lang/eval-fail-fromTOML-valueWithNullByte.nix b/tests/functional/lang/eval-fail-fromTOML-valueWithNullByte.nix new file mode 100644 index 000000000..183cab6b3 --- /dev/null +++ b/tests/functional/lang/eval-fail-fromTOML-valueWithNullByte.nix @@ -0,0 +1 @@ +builtins.fromTOML ''k = "a\u0000b"'' diff --git a/tests/functional/linux-sandbox.sh b/tests/functional/linux-sandbox.sh index 81ef36237..abb635f11 100755 --- a/tests/functional/linux-sandbox.sh +++ b/tests/functional/linux-sandbox.sh @@ -9,6 +9,7 @@ TODO_NixOS clearStore requireSandboxSupport +requiresUnprivilegedUserNamespaces # Note: we need to bind-mount $SHELL into the chroot. Currently we # only support the case where $SHELL is in the Nix store, because diff --git a/tests/functional/local-overlay-store/bad-uris.sh b/tests/functional/local-overlay-store/bad-uris.sh index b7930e32e..f0c6a151c 100644 --- a/tests/functional/local-overlay-store/bad-uris.sh +++ b/tests/functional/local-overlay-store/bad-uris.sh @@ -19,7 +19,7 @@ TODO_NixOS for i in "${storesBad[@]}"; do echo $i - unshare --mount --map-root-user bash <&$NIX_LOG_FD + touch $out + ''; +} diff --git a/tests/functional/nested-sandboxing.sh b/tests/functional/nested-sandboxing.sh index 7462d2968..4d4cf125e 100755 --- a/tests/functional/nested-sandboxing.sh +++ b/tests/functional/nested-sandboxing.sh @@ -7,6 +7,7 @@ source common.sh TODO_NixOS requireSandboxSupport +requiresUnprivilegedUserNamespaces start="$TEST_ROOT/start" mkdir -p "$start" diff --git a/tests/functional/nested-sandboxing/command.sh b/tests/functional/nested-sandboxing/command.sh index e9c40a5d9..7c04e82f5 100644 --- a/tests/functional/nested-sandboxing/command.sh +++ b/tests/functional/nested-sandboxing/command.sh @@ -18,6 +18,7 @@ goodStoreUrl () { # whether this test is being run in a derivation as part of the nix build or # being manually run by a developer outside a derivation runNixBuild () { + local storeFun=$1 local altitude=$2 nix-build \ diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index 706e0f5db..59d1f1be0 100755 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -309,7 +309,7 @@ runRepl () { for test in $(cd "$testDir/repl"; echo *.in); do test="$(basename "$test" .in)" in="$testDir/repl/$test.in" - actual="$testDir/repl/$test.actual" + actual="$TEST_ROOT/$test.actual" expected="$testDir/repl/$test.expected" (cd "$testDir/repl"; set +x; runRepl 2>&1) < "$in" > "$actual" || { echo "FAIL: $test (exit code $?)" >&2 diff --git a/tests/functional/shell.sh b/tests/functional/shell.sh index cfc8e4102..51032ff1b 100755 --- a/tests/functional/shell.sh +++ b/tests/functional/shell.sh @@ -52,6 +52,7 @@ if isDaemonNewer "2.20.0pre20231220"; then fi requireSandboxSupport +requiresUnprivilegedUserNamespaces chmod -R u+w "$TEST_ROOT/store0" || true rm -rf "$TEST_ROOT/store0" diff --git a/tests/functional/supplementary-groups.sh b/tests/functional/supplementary-groups.sh index 50259a3e1..400333f7d 100755 --- a/tests/functional/supplementary-groups.sh +++ b/tests/functional/supplementary-groups.sh @@ -9,7 +9,7 @@ needLocalStore "The test uses --store always so we would just be bypassing the d TODO_NixOS -unshare --mount --map-root-user -- bash -e -x <